├── .travis.yml
├── LICENSE
├── README.md
├── misc
    └── thmap_lookup_80_64bit_keys_intel_4980hq.svg
├── pkg
    ├── Makefile
    ├── SPECS
    │   └── libthmap.spec
    ├── debian
    │   ├── changelog
    │   ├── compat
    │   ├── control
    │   ├── libthmap-dev.install
    │   ├── libthmap-dev.manpages
    │   ├── libthmap1.install
    │   ├── rules
    │   └── source
    │   │   └── format
    └── version.txt
└── src
    ├── Makefile
    ├── murmurhash.c
    ├── t_stress.c
    ├── t_thmap.c
    ├── thmap.3
    ├── thmap.c
    ├── thmap.h
    └── utils.h


/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: c
 2 | 
 3 | compiler:
 4 |   - gcc
 5 |   - clang
 6 | 
 7 | dist: bionic
 8 | 
 9 | matrix:
10 |   include:
11 |     - os: linux
12 |       arch: amd64
13 |     - os: linux
14 |       arch: arm64
15 |     - os: linux
16 |       arch: ppc64le
17 | 
18 | addons:
19 |   apt:
20 |     update: true
21 |     packages:
22 |     - build-essential
23 |     - fakeroot
24 |     - debhelper
25 |     - libtool
26 |     - libtool-bin
27 | 
28 | script:
29 |   # Build the package.
30 |   - (cd pkg && make deb)
31 |   # Run the unit tests.
32 |   - (cd src && make clean && make tests)
33 |   # Run the stress test (non-DEBUG for high concurrency).
34 |   - (cd src && make clean && make stress)
35 |   # Run the stress test (DEBUG with ASAN enabled).
36 |   - (cd src && make clean && DEBUG=1 make stress)
37 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | /*-
 2 |  * Copyright (c) 2018 Mindaugas Rasiukevicius <rmind at noxt eu>
 3 |  * All rights reserved.
 4 |  *
 5 |  * Redistribution and use in source and binary forms, with or without
 6 |  * modification, are permitted provided that the following conditions
 7 |  * are met:
 8 |  * 1. Redistributions of source code must retain the above copyright
 9 |  *    notice, this list of conditions and the following disclaimer.
10 |  * 2. Redistributions in binary form must reproduce the above copyright
11 |  *    notice, this list of conditions and the following disclaimer in the
12 |  *    documentation and/or other materials provided with the distribution.
13 |  *
14 |  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 |  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 |  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 |  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 |  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 |  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 |  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 |  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 |  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 |  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 |  * SUCH DAMAGE.
25 |  */
26 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Concurrent trie-hash map
  2 | 
  3 | [![Build Status](https://travis-ci.com/rmind/thmap.svg?branch=master)](https://travis-ci.com/rmind/thmap)
  4 | 
  5 | Concurrent trie-hash map library -- a general purpose associative array,
  6 | combining the elements of hashing and radix trie.  Highlights:
  7 | - Very competitive performance, with logarithmic time complexity on average.
  8 | - Lookups are lock-free and inserts/deletes are using fine-grained locking.
  9 | - Incremental growth of the data structure (no large resizing/rehashing).
 10 | - Optional support for use with shared memory, e.g. memory-mapped file.
 11 | 
 12 | The implementation is written in C11 and distributed under the 2-clause
 13 | BSD license.
 14 | 
 15 | NOTE: Delete operations (the key/data destruction) must be synchronised with
 16 | the readers using some reclamation mechanism.  You can use the Epoch-based
 17 | Reclamation (EBR) library provided [HERE](https://github.com/rmind/libqsbr).
 18 | 
 19 | References (some, but not all, key ideas are based on these papers):
 20 | 
 21 | - [W. Litwin. Trie Hashing. Proceedings of the 1981 ACM SIGMOD, p. 19-29
 22 | ](https://dl.acm.org/citation.cfm?id=582322)
 23 | 
 24 | - [P. L. Lehman and S. B. Yao.
 25 | Efficient locking for concurrent operations on B-trees.
 26 | ACM TODS, 6(4):650-670, 1981
 27 | ](https://www.csd.uoc.gr/~hy460/pdf/p650-lehman.pdf)
 28 | 
 29 | ## API
 30 | 
 31 | * `thmap_t *thmap_create(uintptr_t baseptr, const thmap_ops_t *ops, unsigned flags)`
 32 |   * Construct a new trie-hash map.  The optional `ops` parameter can
 33 |   used to set the custom allocate/free operations (see the description
 34 |   of `thmap_ops_t` below).  In such case, the `baseptr` is the base (start)
 35 |   address of the address space mapping (it must be word-aligned).  If `ops`
 36 |   is set to `NULL`, then _malloc(3)_ and _free(3)_ will be used as the
 37 |   default operations and `baseptr` should be
 38 |   set to zero.  Currently, the supported `flags` are:
 39 |     * `THMAP_NOCOPY`: the keys on insert will not be copied and the given
 40 |     pointers to them will be expected to be valid and the values constant
 41 |     until the key is deleted; by default, the put operation will make a
 42 |     copy of the key.
 43 |     * `THMAP_SETROOT`: indicate that the root of the map will be manually
 44 |     set using the `thmap_setroot` routine; by default, the map is initialised
 45 |     and the root node is set on `thmap_create`.
 46 | 
 47 | * `void thmap_destroy(thmap_t *hmap)`
 48 |   * Destroy the map, freeing the memory it uses.
 49 | 
 50 | * `void *thmap_get(thmap_t *hmap, const void *key, size_t len)`
 51 |   * Lookup the key (of a given length) and return the value associated with it.
 52 |   Return `NULL` if the key is not found (see the caveats section).
 53 | 
 54 | * `void *thmap_put(thmap_t *hmap, const void *key, size_t len, void *val)`
 55 |   * Insert the key with an arbitrary value.  If the key is already present,
 56 |   return the already existing associated value without changing it.
 57 |   Otherwise, on a successful insert, return the given value.  Just compare
 58 |   the result against `val` to test whether the insert was successful.
 59 | 
 60 | * `void *thmap_del(thmap_t *hmap, const void *key, size_t len)`
 61 |   * Remove the given key.  If the key was present, return the associated
 62 |   value; otherwise return `NULL`.  The memory associated with the entry is
 63 |   not released immediately, because in the concurrent environment (e.g.
 64 |   multi-threaded application) the caller may need to ensure it is safe to
 65 |   do so.  It is managed using the `thmap_stage_gc` and `thmap_gc` routines.
 66 | 
 67 | * `void *thmap_stage_gc(thmap_t *hmap)`
 68 |   * Stage the currently pending entries (the memory not yet released after
 69 |   the deletion) for reclamation (G/C).  This operation should be called
 70 |   **before** the synchronisation barrier.
 71 |   * Returns a reference which must be passed to `thmap_gc`.  Not calling the
 72 |   G/C function for the returned reference would result in a memory leak.
 73 | 
 74 | * `void thmap_gc(thmap_t *hmap, void *ref)`
 75 |   * Reclaim (G/C) the staged entries i.e. release any memory associated
 76 |   with the deleted keys.  The reference must be the value returned by the
 77 |   call to `thmap_stage_gc`.
 78 |   * This function must be called **after** the synchronisation barrier which
 79 |   guarantees that there are no active readers referencing the staged entries.
 80 | 
 81 | If the map is created using the `THMAP_SETROOT` flag, then the following
 82 | functions are applicable:
 83 | 
 84 | * `void thmap_setroot(thmap_t *thmap, uintptr_t root_offset)`
 85 |   * Set the root node.  The address must be relative to the base address,
 86 |   as if allocated by the `thmap_ops_t::alloc` routine.  Return 0 on success
 87 |   and -1 on failure (if already set).
 88 | 
 89 | * `uintptr_t thmap_getroot(const thmap_t *thmap)`
 90 |   * Get the root node address.  The returned address will be relative to
 91 |   the base address.
 92 | 
 93 | The `thmap_ops_t` structure has the following members:
 94 | * `uintptr_t (*alloc)(size_t len)`
 95 |   * Function to allocate the memory.  Must return an address to the
 96 |   memory area of the size `len`.  The address must be relative to the
 97 |   base address specified during map creation and must be word-aligned.
 98 | * `void (*free)(uintptr_t addr, size_t len)`
 99 |   * Function to release the memory.  Must take a previously allocated
100 |   address (relative to the base) and release the memory area.  The `len`
101 |   is guaranteed to match the original allocation length.
102 | 
103 | ## Notes
104 | 
105 | Internally, offsets from the base pointer are used to organise the access
106 | to the data structure.  This allows user to store the data structure in the
107 | shared memory, using the allocation/free functions.  The keys will also be
108 | copied using the custom functions; if `THMAP_NOCOPY` is set, then the keys
109 | must belong to the same shared memory object.
110 | 
111 | The implementation was extensively tested on a 24-core x86 machine,
112 | see [the stress test](src/t_stress.c) for the details on the technique.
113 | 
114 | ## Caveats
115 | 
116 | * The implementation uses pointer tagging and atomic operations.  This
117 | requires the base address and the allocations to provide at least word
118 | alignment.
119 | 
120 | * While the `NULL` values may be inserted, `thmap_get` and `thmap_del`
121 | cannot indicate whether the key was not found or a key with a NULL value
122 | was found.  If the caller needs to indicate an "empty" value, it can use a
123 | special pointer value, such as `(void *)(uintptr_t)0x1`.
124 | 
125 | ## Performance
126 | 
127 | The library has been benchmarked using different key profiles (8 to 256
128 | bytes), set sizes (hundreds, thousands, millions) and ratio between readers
129 | and writers (from 60:40 to 90:10).  In all cases it demonstrated nearly
130 | linear scalability (up to the number of cores).  Here is an example result
131 | when matched with the C++ libcuckoo library:
132 | 
133 | ![](misc/thmap_lookup_80_64bit_keys_intel_4980hq.svg)
134 | 
135 | Disclaimer: benchmark results, however, depend on many aspects (workload,
136 | hardware characteristics, methodology, etc).  Ultimately, readers are
137 | encouraged to perform their own benchmarks.
138 | 
139 | ## Example
140 | 
141 | Simple case backed by _malloc(3)_, which could be used in multi-threaded
142 | environment:
143 | ```c
144 | thmap_t *kvmap;
145 | struct obj *obj;
146 | 
147 | kvmap = thmap_create(0, NULL);
148 | assert(kvmap != NULL);
149 | ...
150 | obj = obj_create();
151 | thmap_put(kvmap, "test", sizeof("test") - 1, obj);
152 | ...
153 | obj = thmap_get(kvmap, "test", sizeof("test") - 1);
154 | ...
155 | thmap_destroy(kvmap);
156 | ```
157 | 
158 | ## Packages
159 | 
160 | Just build the package, install it and link the library using the
161 | `-lthmap` flag.
162 | * RPM (tested on RHEL/CentOS 7): `cd pkg && make rpm`
163 | * DEB (tested on Debian 9): `cd pkg && make deb`
164 | 


--------------------------------------------------------------------------------
/misc/thmap_lookup_80_64bit_keys_intel_4980hq.svg:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="utf-8"  standalone="no"?>
  2 | <svg 
  3 |  width="1024" height="768"
  4 |  viewBox="0 0 1024 768"
  5 |  xmlns="http://www.w3.org/2000/svg"
  6 |  xmlns:xlink="http://www.w3.org/1999/xlink"
  7 | >
  8 | 
  9 | <title>Gnuplot</title>
 10 | <desc>Produced by GNUPLOT 5.2 patchlevel 0 </desc>
 11 | 
 12 | <g id="gnuplot_canvas">
 13 | 
 14 | <rect x="0" y="0" width="1024" height="768" fill="none"/>
 15 | <defs>
 16 | 
 17 | 	<circle id='gpDot' r='0.5' stroke-width='0.5'/>
 18 | 	<path id='gpPt0' stroke-width='0.222' stroke='currentColor' d='M-1,0 h2 M0,-1 v2'/>
 19 | 	<path id='gpPt1' stroke-width='0.222' stroke='currentColor' d='M-1,-1 L1,1 M1,-1 L-1,1'/>
 20 | 	<path id='gpPt2' stroke-width='0.222' stroke='currentColor' d='M-1,0 L1,0 M0,-1 L0,1 M-1,-1 L1,1 M-1,1 L1,-1'/>
 21 | 	<rect id='gpPt3' stroke-width='0.222' stroke='currentColor' x='-1' y='-1' width='2' height='2'/>
 22 | 	<rect id='gpPt4' stroke-width='0.222' stroke='currentColor' fill='currentColor' x='-1' y='-1' width='2' height='2'/>
 23 | 	<circle id='gpPt5' stroke-width='0.222' stroke='currentColor' cx='0' cy='0' r='1'/>
 24 | 	<use xlink:href='#gpPt5' id='gpPt6' fill='currentColor' stroke='none'/>
 25 | 	<path id='gpPt7' stroke-width='0.222' stroke='currentColor' d='M0,-1.33 L-1.33,0.67 L1.33,0.67 z'/>
 26 | 	<use xlink:href='#gpPt7' id='gpPt8' fill='currentColor' stroke='none'/>
 27 | 	<use xlink:href='#gpPt7' id='gpPt9' stroke='currentColor' transform='rotate(180)'/>
 28 | 	<use xlink:href='#gpPt9' id='gpPt10' fill='currentColor' stroke='none'/>
 29 | 	<use xlink:href='#gpPt3' id='gpPt11' stroke='currentColor' transform='rotate(45)'/>
 30 | 	<use xlink:href='#gpPt11' id='gpPt12' fill='currentColor' stroke='none'/>
 31 | 	<path id='gpPt13' stroke-width='0.222' stroke='currentColor' d='M0,1.330 L1.265,0.411 L0.782,-1.067 L-0.782,-1.076 L-1.265,0.411 z'/>
 32 | 	<use xlink:href='#gpPt13' id='gpPt14' fill='currentColor' stroke='none'/>
 33 | 	<filter id='textbox' filterUnits='objectBoundingBox' x='0' y='0' height='1' width='1'>
 34 | 	  <feFlood flood-color='white' flood-opacity='1' result='bgnd'/>
 35 | 	  <feComposite in='SourceGraphic' in2='bgnd' operator='atop'/>
 36 | 	</filter>
 37 | 	<filter id='greybox' filterUnits='objectBoundingBox' x='0' y='0' height='1' width='1'>
 38 | 	  <feFlood flood-color='lightgrey' flood-opacity='1' result='grey'/>
 39 | 	  <feComposite in='SourceGraphic' in2='grey' operator='atop'/>
 40 | 	</filter>
 41 | </defs>
 42 | <g fill="none" color="white" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
 43 | </g>
 44 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
 45 | 	<path stroke='black'  d='M80.2,710.4 L89.2,710.4 M999.0,710.4 L990.0,710.4  '/>	<g transform="translate(71.9,714.3)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
 46 | 		<text><tspan font-family="Arial" >0.0 </tspan></text>
 47 | 	</g>
 48 | </g>
 49 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
 50 | 	<path stroke='black'  d='M80.2,621.8 L89.2,621.8 M999.0,621.8 L990.0,621.8  '/>	<g transform="translate(71.9,625.7)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
 51 | 		<text><tspan font-family="Arial" >5.0M</tspan></text>
 52 | 	</g>
 53 | </g>
 54 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
 55 | 	<path stroke='black'  d='M80.2,533.2 L89.2,533.2 M999.0,533.2 L990.0,533.2  '/>	<g transform="translate(71.9,537.1)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
 56 | 		<text><tspan font-family="Arial" >10.0M</tspan></text>
 57 | 	</g>
 58 | </g>
 59 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
 60 | 	<path stroke='black'  d='M80.2,444.6 L89.2,444.6 M999.0,444.6 L990.0,444.6  '/>	<g transform="translate(71.9,448.5)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
 61 | 		<text><tspan font-family="Arial" >15.0M</tspan></text>
 62 | 	</g>
 63 | </g>
 64 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
 65 | 	<path stroke='black'  d='M80.2,355.9 L89.2,355.9 M999.0,355.9 L990.0,355.9  '/>	<g transform="translate(71.9,359.8)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
 66 | 		<text><tspan font-family="Arial" >20.0M</tspan></text>
 67 | 	</g>
 68 | </g>
 69 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
 70 | 	<path stroke='black'  d='M80.2,267.3 L89.2,267.3 M999.0,267.3 L990.0,267.3  '/>	<g transform="translate(71.9,271.2)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
 71 | 		<text><tspan font-family="Arial" >25.0M</tspan></text>
 72 | 	</g>
 73 | </g>
 74 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
 75 | 	<path stroke='black'  d='M80.2,178.7 L89.2,178.7 M999.0,178.7 L990.0,178.7  '/>	<g transform="translate(71.9,182.6)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
 76 | 		<text><tspan font-family="Arial" >30.0M</tspan></text>
 77 | 	</g>
 78 | </g>
 79 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
 80 | 	<path stroke='black'  d='M80.2,90.1 L89.2,90.1 M999.0,90.1 L990.0,90.1  '/>	<g transform="translate(71.9,94.0)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
 81 | 		<text><tspan font-family="Arial" >35.0M</tspan></text>
 82 | 	</g>
 83 | </g>
 84 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
 85 | 	<path stroke='black'  d='M80.2,710.4 L80.2,701.4 M80.2,90.1 L80.2,99.1  '/>	<g transform="translate(80.2,732.3)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
 86 | 		<text><tspan font-family="Arial" > 1</tspan></text>
 87 | 	</g>
 88 | </g>
 89 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
 90 | 	<path stroke='black'  d='M211.5,710.4 L211.5,701.4 M211.5,90.1 L211.5,99.1  '/>	<g transform="translate(211.5,732.3)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
 91 | 		<text><tspan font-family="Arial" > 2</tspan></text>
 92 | 	</g>
 93 | </g>
 94 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
 95 | 	<path stroke='black'  d='M342.7,710.4 L342.7,701.4 M342.7,90.1 L342.7,99.1  '/>	<g transform="translate(342.7,732.3)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
 96 | 		<text><tspan font-family="Arial" > 3</tspan></text>
 97 | 	</g>
 98 | </g>
 99 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
100 | 	<path stroke='black'  d='M474.0,710.4 L474.0,701.4 M474.0,90.1 L474.0,99.1  '/>	<g transform="translate(474.0,732.3)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
101 | 		<text><tspan font-family="Arial" > 4</tspan></text>
102 | 	</g>
103 | </g>
104 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
105 | 	<path stroke='black'  d='M605.2,710.4 L605.2,701.4 M605.2,90.1 L605.2,99.1  '/>	<g transform="translate(605.2,732.3)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
106 | 		<text><tspan font-family="Arial" > 5</tspan></text>
107 | 	</g>
108 | </g>
109 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
110 | 	<path stroke='black'  d='M736.5,710.4 L736.5,701.4 M736.5,90.1 L736.5,99.1  '/>	<g transform="translate(736.5,732.3)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
111 | 		<text><tspan font-family="Arial" > 6</tspan></text>
112 | 	</g>
113 | </g>
114 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
115 | 	<path stroke='black'  d='M867.7,710.4 L867.7,701.4 M867.7,90.1 L867.7,99.1  '/>	<g transform="translate(867.7,732.3)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
116 | 		<text><tspan font-family="Arial" > 7</tspan></text>
117 | 	</g>
118 | </g>
119 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
120 | 	<path stroke='black'  d='M999.0,710.4 L999.0,701.4 M999.0,90.1 L999.0,99.1  '/>	<g transform="translate(999.0,732.3)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
121 | 		<text><tspan font-family="Arial" > 8</tspan></text>
122 | 	</g>
123 | </g>
124 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
125 | </g>
126 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
127 | 	<path stroke='black'  d='M80.2,90.1 L80.2,710.4 L999.0,710.4 L999.0,90.1 L80.2,90.1 Z  '/></g>
128 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
129 | 	<g transform="translate(16.3,400.3) rotate(270)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
130 | 		<text><tspan font-family="Arial" >Operations / second</tspan></text>
131 | 	</g>
132 | </g>
133 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
134 | 	<g transform="translate(539.6,759.3)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
135 | 		<text><tspan font-family="Arial" >Threads</tspan></text>
136 | 	</g>
137 | </g>
138 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
139 | 	<g transform="translate(539.6,31.0)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
140 | 		<text><tspan font-family="Arial" >CONCURRENT LOOKUP (keys: 20k; key size: 8 bytes; ratio: 80% lookup vs 10%/10% insert/delete)</tspan></text>
141 | 	</g>
142 | 	<g transform="translate(539.6,49.0)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
143 | 		<text><tspan font-family="Arial" >	Intel(R) Core(TM) i7-4980HQ CPU - 2.80GHz</tspan></text>
144 | 	</g>
145 | 	<g transform="translate(539.6,67.0)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
146 | 		<text><tspan font-family="Arial" > (higher is better)</tspan></text>
147 | 	</g>
148 | </g>
149 | 	<g id="gnuplot_plot_1" ><title>thmap</title>
150 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
151 | 	<g transform="translate(931.9,112.0)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
152 | 		<text><tspan font-family="Arial" >thmap</tspan></text>
153 | 	</g>
154 | </g>
155 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
156 | 	<path stroke='rgb(  0,  66, 183)'  d='M940.2,108.1 L982.4,108.1 M80.2,585.3 L211.5,482.6 L342.7,382.7 L474.0,288.4 L605.2,245.8 L736.5,218.2
157 | 		L867.7,194.9 L999.0,162.6  '/>	<use xlink:href='#gpPt0' transform='translate(80.2,585.3) scale(4.50)' color='rgb(  0,  66, 183)'/>
158 | 	<use xlink:href='#gpPt0' transform='translate(211.5,482.6) scale(4.50)' color='rgb(  0,  66, 183)'/>
159 | 	<use xlink:href='#gpPt0' transform='translate(342.7,382.7) scale(4.50)' color='rgb(  0,  66, 183)'/>
160 | 	<use xlink:href='#gpPt0' transform='translate(474.0,288.4) scale(4.50)' color='rgb(  0,  66, 183)'/>
161 | 	<use xlink:href='#gpPt0' transform='translate(605.2,245.8) scale(4.50)' color='rgb(  0,  66, 183)'/>
162 | 	<use xlink:href='#gpPt0' transform='translate(736.5,218.2) scale(4.50)' color='rgb(  0,  66, 183)'/>
163 | 	<use xlink:href='#gpPt0' transform='translate(867.7,194.9) scale(4.50)' color='rgb(  0,  66, 183)'/>
164 | 	<use xlink:href='#gpPt0' transform='translate(999.0,162.6) scale(4.50)' color='rgb(  0,  66, 183)'/>
165 | 	<use xlink:href='#gpPt0' transform='translate(961.3,108.1) scale(4.50)' color='rgb(  0,  66, 183)'/>
166 | </g>
167 | 	</g>
168 | 	<g id="gnuplot_plot_2" ><title>libcuckoo</title>
169 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
170 | 	<g transform="translate(931.9,130.0)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
171 | 		<text><tspan font-family="Arial" >libcuckoo</tspan></text>
172 | 	</g>
173 | </g>
174 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
175 | 	<path stroke='rgb(  0, 158, 115)'  d='M940.2,126.1 L982.4,126.1 M80.2,670.2 L211.5,632.9 L342.7,600.8 L474.0,579.4 L605.2,585.8 L736.5,581.3
176 | 		L867.7,578.2 L999.0,575.3  '/>	<use xlink:href='#gpPt1' transform='translate(80.2,670.2) scale(4.50)' color='rgb(  0, 158, 115)'/>
177 | 	<use xlink:href='#gpPt1' transform='translate(211.5,632.9) scale(4.50)' color='rgb(  0, 158, 115)'/>
178 | 	<use xlink:href='#gpPt1' transform='translate(342.7,600.8) scale(4.50)' color='rgb(  0, 158, 115)'/>
179 | 	<use xlink:href='#gpPt1' transform='translate(474.0,579.4) scale(4.50)' color='rgb(  0, 158, 115)'/>
180 | 	<use xlink:href='#gpPt1' transform='translate(605.2,585.8) scale(4.50)' color='rgb(  0, 158, 115)'/>
181 | 	<use xlink:href='#gpPt1' transform='translate(736.5,581.3) scale(4.50)' color='rgb(  0, 158, 115)'/>
182 | 	<use xlink:href='#gpPt1' transform='translate(867.7,578.2) scale(4.50)' color='rgb(  0, 158, 115)'/>
183 | 	<use xlink:href='#gpPt1' transform='translate(999.0,575.3) scale(4.50)' color='rgb(  0, 158, 115)'/>
184 | 	<use xlink:href='#gpPt1' transform='translate(961.3,126.1) scale(4.50)' color='rgb(  0, 158, 115)'/>
185 | </g>
186 | 	</g>
187 | 	<g id="gnuplot_plot_3" ><title>rbtree + RW-lock</title>
188 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
189 | 	<g transform="translate(931.9,148.0)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
190 | 		<text><tspan font-family="Arial" >rbtree + RW-lock</tspan></text>
191 | 	</g>
192 | </g>
193 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
194 | 	<path stroke='rgb(154,   0,  46)'  d='M940.2,144.1 L982.4,144.1 M80.2,633.7 L211.5,694.9 L342.7,696.2 L474.0,697.3 L605.2,698.9 L736.5,699.6
195 | 		L867.7,700.5 L999.0,701.3  '/>	<use xlink:href='#gpPt2' transform='translate(80.2,633.7) scale(4.50)' color='rgb(154,   0,  46)'/>
196 | 	<use xlink:href='#gpPt2' transform='translate(211.5,694.9) scale(4.50)' color='rgb(154,   0,  46)'/>
197 | 	<use xlink:href='#gpPt2' transform='translate(342.7,696.2) scale(4.50)' color='rgb(154,   0,  46)'/>
198 | 	<use xlink:href='#gpPt2' transform='translate(474.0,697.3) scale(4.50)' color='rgb(154,   0,  46)'/>
199 | 	<use xlink:href='#gpPt2' transform='translate(605.2,698.9) scale(4.50)' color='rgb(154,   0,  46)'/>
200 | 	<use xlink:href='#gpPt2' transform='translate(736.5,699.6) scale(4.50)' color='rgb(154,   0,  46)'/>
201 | 	<use xlink:href='#gpPt2' transform='translate(867.7,700.5) scale(4.50)' color='rgb(154,   0,  46)'/>
202 | 	<use xlink:href='#gpPt2' transform='translate(999.0,701.3) scale(4.50)' color='rgb(154,   0,  46)'/>
203 | 	<use xlink:href='#gpPt2' transform='translate(961.3,144.1) scale(4.50)' color='rgb(154,   0,  46)'/>
204 | </g>
205 | 	</g>
206 | <g fill="none" color="white" stroke="rgb(154,   0,  46)" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
207 | </g>
208 | <g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
209 | </g>
210 | <g fill="none" color="black" stroke="black" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
211 | </g>
212 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
213 | 	<path stroke='black'  d='M80.2,90.1 L80.2,710.4 L999.0,710.4 L999.0,90.1 L80.2,90.1 Z  '/></g>
214 | <g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
215 | </g>
216 | </g>
217 | </svg>
218 | 
219 | 


--------------------------------------------------------------------------------
/pkg/Makefile:
--------------------------------------------------------------------------------
 1 | PROJ=	libthmap
 2 | 
 3 | all:
 4 | 	@ echo "targets"
 5 | 	@ echo "	make rpm"
 6 | 	@ echo "	make deb"
 7 | 
 8 | rpm:
 9 | 	mkdir -p SOURCES && tar czpvf SOURCES/$(PROJ).tar.gz ../src
10 | 	rpmbuild -ba -v --define "_topdir ${PWD}" SPECS/$(PROJ).spec
11 | 	@ echo && printf "\x1B[32mRPM packages:\033[0m\n" && ls -1 RPMS/*
12 | 
13 | deb:
14 | 	cp -R ../src ./SOURCES
15 | 	dpkg-buildpackage -rfakeroot -us -uc -b
16 | 	@ echo && printf "\x1B[32mDEB packages:\033[0m\n" && ls -1 ../*.deb
17 | 
18 | clean:
19 | 	rm -rf BUILD BUILDROOT RPMS SOURCES SRPMS
20 | 
21 | .PHONY: all rpm deb clean
22 | 


--------------------------------------------------------------------------------
/pkg/SPECS/libthmap.spec:
--------------------------------------------------------------------------------
 1 | %define version	%(cat %{_topdir}/version.txt)
 2 | 
 3 | Name:		libthmap
 4 | Version:	%{version}
 5 | Release:	1%{?dist}
 6 | Summary:	Concurrent trie-hash map library
 7 | Group:		System Environment/Libraries
 8 | License:	BSD
 9 | URL:		https://github.com/rmind/thmap
10 | Source0:	libthmap.tar.gz
11 | 
12 | BuildRequires:	make
13 | BuildRequires:	libtool
14 | 
15 | %description
16 | 
17 | Concurrent trie-hash map library -- a general purpose hash map, combining
18 | the elements of hashing and radix trie.  The implementation is written in
19 | C11 and distributed under the 2-clause BSD license.
20 | 
21 | %prep
22 | %setup -q -n src
23 | 
24 | %check
25 | make tests
26 | 
27 | %build
28 | make %{?_smp_mflags} lib \
29 |     LIBDIR=%{_libdir}
30 | 
31 | %install
32 | make install \
33 |     DESTDIR=%{buildroot} \
34 |     LIBDIR=%{_libdir} \
35 |     INCDIR=%{_includedir} \
36 |     MANDIR=%{_mandir}
37 | 
38 | %files
39 | %{_libdir}/*
40 | %{_includedir}/*
41 | %{_mandir}/*
42 | 
43 | %changelog
44 | 


--------------------------------------------------------------------------------
/pkg/debian/changelog:
--------------------------------------------------------------------------------
1 | thmap (0.0.1) unstable; urgency=medium
2 | 
3 |   * Initial release.
4 | 
5 |  -- rmind <rmind@noxt.eu>  Fri, 26 Aug 2018 18:00:00 +0100
6 | 


--------------------------------------------------------------------------------
/pkg/debian/compat:
--------------------------------------------------------------------------------
1 | 10
2 | 


--------------------------------------------------------------------------------
/pkg/debian/control:
--------------------------------------------------------------------------------
 1 | Source: thmap
 2 | Priority: extra
 3 | Maintainer: https://github.com/rmind
 4 | Build-Depends:
 5 |  debhelper (>= 9),
 6 |  make,
 7 |  libtool
 8 | Standards-Version: 3.9.1
 9 | Homepage: https://github.com/rmind/thmap
10 | License: BSD-2-clause
11 | 
12 | Package: libthmap1
13 | Section: lib
14 | Architecture: any
15 | Depends: ${shlibs:Depends}, ${misc:Depends}
16 | Description: Concurrent trie-hash map library
17 |  Concurrent trie-hash map library -- a general purpose hash map, combining
18 |  the elements of hashing and radix trie.  The implementation is written in
19 |  C11 and distributed under the 2-clause BSD license.
20 | 
21 | Package: libthmap1-dbg
22 | Section: debug
23 | Architecture: any
24 | Depends: ${misc:Depends}, libthmap1 (= ${binary:Version})
25 | Description: Debug symbols for libthmap1
26 |  Debug symbols for libthmap1.
27 | 
28 | Package: libthmap-dev
29 | Section: libdevel
30 | Architecture: any
31 | Depends: ${shlibs:Depends}, ${misc:Depends}, libthmap1 (= ${binary:Version})
32 | Description: Development files for libthmap1
33 |  Development files for libthmap1.
34 | 


--------------------------------------------------------------------------------
/pkg/debian/libthmap-dev.install:
--------------------------------------------------------------------------------
1 | usr/include/*
2 | usr/lib/*/lib*.a
3 | usr/lib/*/lib*.so
4 | 


--------------------------------------------------------------------------------
/pkg/debian/libthmap-dev.manpages:
--------------------------------------------------------------------------------
1 | debian/tmp/man3/thmap.3
2 | 


--------------------------------------------------------------------------------
/pkg/debian/libthmap1.install:
--------------------------------------------------------------------------------
1 | usr/lib/*/lib*.so.*
2 | 


--------------------------------------------------------------------------------
/pkg/debian/rules:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/make -f
 2 | 
 3 | #export DH_VERBOSE = 1
 4 | 
 5 | PKGVERSION:=$(shell cat version.txt)
 6 | DEB_HOST_MULTIARCH?=$(shell dpkg-architecture -qDEB_HOST_MULTIARCH)
 7 | LIBDIR:=/usr/lib/$(DEB_HOST_MULTIARCH)
 8 | INCDIR:=/usr/include
 9 | 
10 | %:
11 | 	dh $@ --sourcedirectory=SOURCES --parallel
12 | 
13 | override_dh_auto_test:
14 | 	dh_auto_test tests
15 | 
16 | override_dh_auto_install:
17 | 	dh_auto_install -- LIBDIR=$(LIBDIR) INCDIR=$(INCDIR)
18 | 
19 | override_dh_strip:
20 | 	dh_strip -p libthmap1 --dbg-package=libthmap1-dbg
21 | 	dh_strip -a --remaining-packages
22 | 
23 | override_dh_gencontrol:
24 | 	dh_gencontrol -- -v$(PKGVERSION)
25 | 


--------------------------------------------------------------------------------
/pkg/debian/source/format:
--------------------------------------------------------------------------------
1 | 3.0 (quilt)
2 | 


--------------------------------------------------------------------------------
/pkg/version.txt:
--------------------------------------------------------------------------------
1 | 0.1.0
2 | 


--------------------------------------------------------------------------------
/src/Makefile:
--------------------------------------------------------------------------------
  1 | #
  2 | # This file is in the Public Domain.
  3 | #
  4 | 
  5 | PROJ=		thmap
  6 | LIB=		lib$(PROJ)
  7 | INCS=		thmap.h
  8 | MANS=		thmap.3
  9 | 
 10 | SYSNAME:=	$(shell uname -s)
 11 | SYSARCH:=	$(shell uname -m)
 12 | 
 13 | ifeq ($(MAKECMDGOALS),tests)
 14 | DEBUG=		1
 15 | endif
 16 | 
 17 | #
 18 | # Common C compiler flags.
 19 | #
 20 | CFLAGS+=	-std=c11 -O2 -g -Wall -Wextra -Werror
 21 | 
 22 | #
 23 | # Extended warning flags.
 24 | #
 25 | CFLAGS+=	-Wno-unknown-warning-option # gcc vs clang
 26 | 
 27 | CFLAGS+=	-Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith
 28 | CFLAGS+=	-Wmissing-declarations -Wredundant-decls -Wnested-externs
 29 | CFLAGS+=	-Wshadow -Wcast-qual -Wcast-align -Wwrite-strings
 30 | CFLAGS+=	-Wold-style-definition
 31 | CFLAGS+=	-Wsuggest-attribute=noreturn -Wjump-misses-init
 32 | CFLAGS+=	-Wduplicated-cond -Wmisleading-indentation -Wnull-dereference
 33 | CFLAGS+=	-Wduplicated-branches -Wrestrict
 34 | 
 35 | #
 36 | # System-specific or compiler-specific flags.
 37 | #
 38 | 
 39 | ifeq ($(SYSNAME),Linux)
 40 | CFLAGS+=	-D_POSIX_C_SOURCE=200809L
 41 | CFLAGS+=	-D_GNU_SOURCE -D_DEFAULT_SOURCE
 42 | endif
 43 | 
 44 | #
 45 | # Standard vs debug build flags.
 46 | #
 47 | ifeq ($(DEBUG),1)
 48 | CFLAGS+=	-Og -DDEBUG -fno-omit-frame-pointer
 49 | ifeq ($(SYSARCH),x86_64)
 50 | CFLAGS+=	-fsanitize=address -fsanitize=undefined
 51 | LDFLAGS+=	-fsanitize=address -fsanitize=undefined
 52 | endif
 53 | else
 54 | CFLAGS+=	-DNDEBUG
 55 | endif
 56 | 
 57 | #
 58 | # Source and targets.
 59 | #
 60 | 
 61 | OBJS=		thmap.o
 62 | OBJS+=		murmurhash.o
 63 | 
 64 | $(LIB).la:	LDFLAGS+=	-rpath $(LIBDIR) -version-info 1:0:0
 65 | install/%.la:	ILIBDIR=	$(DESTDIR)/$(LIBDIR)
 66 | install:	IINCDIR=	$(DESTDIR)/$(INCDIR)/
 67 | install:	IMANDIR=	$(DESTDIR)/$(MANDIR)/man3/
 68 | 
 69 | obj: $(OBJS)
 70 | 
 71 | lib: $(LIB).la
 72 | 
 73 | %.lo: %.c
 74 | 	libtool --mode=compile --tag CC $(CC) $(CFLAGS) -c $<
 75 | 
 76 | $(LIB).la: $(shell echo $(OBJS) | sed 's/\.o/\.lo/g')
 77 | 	libtool --mode=link --tag CC $(CC) $(LDFLAGS) -o $@ $(notdir $^)
 78 | 
 79 | install/%.la: %.la
 80 | 	mkdir -p $(ILIBDIR)
 81 | 	libtool --mode=install install -c $(notdir $@) $(ILIBDIR)/$(notdir $@)
 82 | 
 83 | install: $(addprefix install/,$(LIB).la)
 84 | 	libtool --mode=finish $(LIBDIR)
 85 | 	mkdir -p $(IINCDIR) && install -c $(INCS) $(IINCDIR)
 86 | 	mkdir -p $(IMANDIR) && install -c $(MANS) $(IMANDIR)
 87 | 
 88 | tests: $(OBJS) t_$(PROJ).o
 89 | 	$(CC) $(CFLAGS) $^ -o t_$(PROJ)
 90 | 	MALLOC_CHECK_=3 ./t_$(PROJ)
 91 | 
 92 | stress: $(OBJS) t_stress.o murmurhash.o
 93 | 	$(CC) $(CFLAGS) $^ -o t_stress -lpthread
 94 | 	./t_stress
 95 | 
 96 | clean:
 97 | 	libtool --mode=clean rm
 98 | 	rm -rf .libs *.o *.lo *.la t_thmap t_stress
 99 | 
100 | .PHONY: all obj lib install tests stress clean
101 | 


--------------------------------------------------------------------------------
/src/murmurhash.c:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * murmurhash3 -- from the original code:
 3 |  *
 4 |  * "MurmurHash3 was written by Austin Appleby, and is placed in the public
 5 |  * domain. The author hereby disclaims copyright to this source code."
 6 |  *
 7 |  * References:
 8 |  *	https://github.com/aappleby/smhasher/
 9 |  */
10 | 
11 | #include <inttypes.h>
12 | #include <endian.h>
13 | 
14 | #include "utils.h"
15 | 
16 | uint32_t
17 | murmurhash3(const void *key, size_t len, uint32_t seed)
18 | {
19 | 	const uint8_t *data = key;
20 | 	const size_t orig_len = len;
21 | 	uint32_t h = seed;
22 | 
23 | 	if (__predict_true(((uintptr_t)key & 3) == 0)) {
24 | 		while (len >= sizeof(uint32_t)) {
25 | 			uint32_t k = *(const uint32_t *)(const void *)data;
26 | 
27 | 			k = htole32(k);
28 | 
29 | 			k *= 0xcc9e2d51;
30 | 			k = (k << 15) | (k >> 17);
31 | 			k *= 0x1b873593;
32 | 
33 | 			h ^= k;
34 | 			h = (h << 13) | (h >> 19);
35 | 			h = h * 5 + 0xe6546b64;
36 | 
37 | 			data += sizeof(uint32_t);
38 | 			len -= sizeof(uint32_t);
39 | 		}
40 | 	} else {
41 | 		while (len >= sizeof(uint32_t)) {
42 | 			uint32_t k;
43 | 
44 | 			k  = data[0];
45 | 			k |= data[1] << 8;
46 | 			k |= data[2] << 16;
47 | 			k |= data[3] << 24;
48 | 
49 | 			k *= 0xcc9e2d51;
50 | 			k = (k << 15) | (k >> 17);
51 | 			k *= 0x1b873593;
52 | 
53 | 			h ^= k;
54 | 			h = (h << 13) | (h >> 19);
55 | 			h = h * 5 + 0xe6546b64;
56 | 
57 | 			data += sizeof(uint32_t);
58 | 			len -= sizeof(uint32_t);
59 | 		}
60 | 	}
61 | 
62 | 	/*
63 | 	 * Handle the last few bytes of the input array.
64 | 	 */
65 | 	uint32_t k = 0;
66 | 
67 | 	switch (len) {
68 | 	case 3:
69 | 		k ^= data[2] << 16;
70 | 		/* FALLTHROUGH */
71 | 	case 2:
72 | 		k ^= data[1] << 8;
73 | 		/* FALLTHROUGH */
74 | 	case 1:
75 | 		k ^= data[0];
76 | 		k *= 0xcc9e2d51;
77 | 		k = (k << 15) | (k >> 17);
78 | 		k *= 0x1b873593;
79 | 		h ^= k;
80 | 	}
81 | 
82 | 	/*
83 | 	 * Finalisation mix: force all bits of a hash block to avalanche.
84 | 	 */
85 | 	h ^= orig_len;
86 | 	h ^= h >> 16;
87 | 	h *= 0x85ebca6b;
88 | 	h ^= h >> 13;
89 | 	h *= 0xc2b2ae35;
90 | 	h ^= h >> 16;
91 | 
92 | 	return h;
93 | }
94 | 


--------------------------------------------------------------------------------
/src/t_stress.c:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2017-2018 Mindaugas Rasiukevicius <rmind at noxt eu>
  3 |  * All rights reserved.
  4 |  *
  5 |  * Use is subject to license terms, as specified in the LICENSE file.
  6 |  */
  7 | 
  8 | #include <stdio.h>
  9 | #include <stdlib.h>
 10 | #include <inttypes.h>
 11 | #include <unistd.h>
 12 | #include <pthread.h>
 13 | #include <time.h>
 14 | #include <limits.h>
 15 | #include <errno.h>
 16 | #include <err.h>
 17 | 
 18 | #include "thmap.h"
 19 | #include "utils.h"
 20 | 
 21 | #define	CHECK_TRUE(x)	\
 22 |     if (!(x)) { printf("FAIL: %s line %d\n", __func__, __LINE__); abort(); }
 23 | 
 24 | static thmap_t *		map;
 25 | static pthread_barrier_t	barrier;
 26 | static unsigned			nworkers;
 27 | 
 28 | static uint64_t			c_keys[4];
 29 | static unsigned			thmap_alloc_count;
 30 | 
 31 | static uintptr_t
 32 | alloc_test_wrapper(size_t len)
 33 | {
 34 | 	thmap_alloc_count++; // count the allocation
 35 | 	return (uintptr_t)malloc(len);
 36 | }
 37 | 
 38 | static void
 39 | free_test_wrapper(uintptr_t addr, size_t len)
 40 | {
 41 | 	free((void *)addr); (void)len;
 42 | }
 43 | 
 44 | static void
 45 | del_collision_keys(thmap_t *m)
 46 | {
 47 | 	thmap_del(m, &c_keys[0], sizeof(uint64_t));
 48 | 	thmap_del(m, &c_keys[1], sizeof(uint64_t));
 49 | 	thmap_del(m, &c_keys[2], sizeof(uint64_t));
 50 | 	thmap_del(m, &c_keys[3], sizeof(uint64_t));
 51 | }
 52 | 
 53 | static void
 54 | prepare_collisions(void)
 55 | {
 56 | 	static const thmap_ops_t thmap_test_ops = {
 57 | 		.alloc = alloc_test_wrapper,
 58 | 		.free = free_test_wrapper
 59 | 	};
 60 | 	void *val, *keyval = (void *)(uintptr_t)0xdeadbeef;
 61 | 
 62 | 	/*
 63 | 	 * Pre-calculated collisions.  Note: the brute-force conditions
 64 | 	 * on the murmurhash3() values:
 65 | 	 *
 66 | 	 *	((h0 >> 26) ^ 8) != ((h1 >> 26) ^ 8) || (h0 & 3) == (h1 & 3)
 67 | 	 *	((h0 >> 26) ^ 8) != ((h1 >> 26) ^ 8) || (h0 & 3) != (h1 & 3)
 68 | 	 *	h0 != h3
 69 | 	 */
 70 | 	c_keys[0] = 0x8000100000080001;
 71 | 	c_keys[1] = 0x80001000000800fa;
 72 | 	c_keys[2] = 0x80001000000800df;
 73 | 	c_keys[3] = 0x800010012e04d085;
 74 | 
 75 | 	/*
 76 | 	 * Validate check root-level collision.
 77 | 	 */
 78 | 	map = thmap_create(0, &thmap_test_ops, THMAP_NOCOPY);
 79 | 	thmap_alloc_count = 0;
 80 | 
 81 | 	val = thmap_put(map, &c_keys[0], sizeof(uint64_t), keyval);
 82 | 	CHECK_TRUE(val && thmap_alloc_count == 2); // leaf + internode
 83 | 
 84 | 	val = thmap_put(map, &c_keys[1], sizeof(uint64_t), keyval);
 85 | 	CHECK_TRUE(val && thmap_alloc_count == 3); // just leaf
 86 | 
 87 | 	del_collision_keys(map);
 88 | 	thmap_destroy(map);
 89 | 
 90 | 	/*
 91 | 	 * Validate check first-level (L0) collision.
 92 | 	 */
 93 | 	map = thmap_create(0, &thmap_test_ops, THMAP_NOCOPY);
 94 | 	(void)thmap_put(map, &c_keys[0], sizeof(uint64_t), keyval);
 95 | 
 96 | 	thmap_alloc_count = 0;
 97 | 	val = thmap_put(map, &c_keys[2], sizeof(uint64_t), keyval);
 98 | 	CHECK_TRUE(val && thmap_alloc_count == 2); // leaf + internode
 99 | 
100 | 	del_collision_keys(map);
101 | 	thmap_destroy(map);
102 | 
103 | 	/*
104 | 	 * Validate the full 32-bit collision.
105 | 	 */
106 | 	map = thmap_create(0, &thmap_test_ops, THMAP_NOCOPY);
107 | 	(void)thmap_put(map, &c_keys[0], sizeof(uint64_t), keyval);
108 | 
109 | 	thmap_alloc_count = 0;
110 | 	val = thmap_put(map, &c_keys[3], sizeof(uint64_t), keyval);
111 | 	CHECK_TRUE(val && thmap_alloc_count == 1 + 8); // leaf + 8 levels
112 | 
113 | 	del_collision_keys(map);
114 | 	thmap_destroy(map);
115 | }
116 | 
117 | /*
118 |  * Simple xorshift; random() causes huge lock contention on Linux/glibc,
119 |  * which would "hide" the possible race conditions.
120 |  */
121 | static unsigned long
122 | fast_random(void)
123 | {
124 | 	static __thread uint32_t fast_random_seed = 5381;
125 | 	uint32_t x = fast_random_seed;
126 | 	x ^= x << 13;
127 | 	x ^= x >> 17;
128 | 	x ^= x << 5;
129 | 	fast_random_seed = x;
130 | 	return x;
131 | }
132 | 
133 | static void *
134 | fuzz_collision(void *arg, unsigned range_mask)
135 | {
136 | 	const unsigned id = (uintptr_t)arg;
137 | 	unsigned n = 1 * 1000 * 1000;
138 | 
139 | 	pthread_barrier_wait(&barrier);
140 | 	while (n--) {
141 | 		uint64_t key = c_keys[fast_random() & range_mask];
142 | 		void *keyval = (void *)(uintptr_t)key;
143 | 		void *val;
144 | 
145 | 		switch (fast_random() & 3) {
146 | 		case 0:
147 | 		case 1: // ~50% lookups
148 | 			val = thmap_get(map, &key, sizeof(key));
149 | 			CHECK_TRUE(!val || val == keyval);
150 | 			break;
151 | 		case 2:
152 | 			val = thmap_put(map, &key, sizeof(key), keyval);
153 | 			CHECK_TRUE(val == keyval);
154 | 			break;
155 | 		case 3:
156 | 			val = thmap_del(map, &key, sizeof(key));
157 | 			CHECK_TRUE(!val || val == keyval);
158 | 			break;
159 | 		}
160 | 	}
161 | 	pthread_barrier_wait(&barrier);
162 | 
163 | 	/* The primary thread performs the clean-up. */
164 | 	if (id == 0) {
165 | 		thmap_del(map, &c_keys[0], sizeof(uint64_t));
166 | 		thmap_del(map, &c_keys[1], sizeof(uint64_t));
167 | 		thmap_del(map, &c_keys[2], sizeof(uint64_t));
168 | 		thmap_del(map, &c_keys[3], sizeof(uint64_t));
169 | 	}
170 | 	pthread_exit(NULL);
171 | 	return NULL;
172 | }
173 | 
174 | static void *
175 | fuzz_root_collision(void *arg)
176 | {
177 | 	/* Root-level collision: c_keys[0] vs c_keys[1]. */
178 | 	return fuzz_collision(arg, 0x1);
179 | }
180 | 
181 | static void *
182 | fuzz_l0_collision(void *arg)
183 | {
184 | 	/* First-level collision: c_keys[0] vs c_keys[2]. */
185 | 	return fuzz_collision(arg, 0x2);
186 | }
187 | 
188 | static void *
189 | fuzz_multi_collision(void *arg)
190 | {
191 | 	/* Root-level collision: c_keys vs c_keys. */
192 | 	return fuzz_collision(arg, 0x3);
193 | }
194 | 
195 | static void *
196 | fuzz_multi(void *arg, uint64_t range_mask)
197 | {
198 | 	const unsigned id = (uintptr_t)arg;
199 | 	unsigned n = 1 * 1000 * 1000;
200 | 
201 | 	pthread_barrier_wait(&barrier);
202 | 	while (n--) {
203 | 		uint64_t key = fast_random() & range_mask;
204 | 		void *keyval = (void *)(uintptr_t)key;
205 | 		void *val;
206 | 
207 | 		switch (fast_random() & 3) {
208 | 		case 0:
209 | 		case 1: // ~50% lookups
210 | 			val = thmap_get(map, &key, sizeof(key));
211 | 			CHECK_TRUE(!val || val == keyval);
212 | 			break;
213 | 		case 2:
214 | 			val = thmap_put(map, &key, sizeof(key), keyval);
215 | 			CHECK_TRUE(val == keyval);
216 | 			break;
217 | 		case 3:
218 | 			val = thmap_del(map, &key, sizeof(key));
219 | 			CHECK_TRUE(!val || val == keyval);
220 | 			break;
221 | 		}
222 | 	}
223 | 	pthread_barrier_wait(&barrier);
224 | 
225 | 	if (id == 0) for (uint64_t key = 0; key <= range_mask; key++) {
226 | 		thmap_del(map, &key, sizeof(key));
227 | 	}
228 | 	pthread_exit(NULL);
229 | 	return NULL;
230 | }
231 | 
232 | static void *
233 | fuzz_multi_128(void *arg)
234 | {
235 | 	/*
236 | 	 * Key range of 128 values to trigger contended
237 | 	 * expand/collapse cycles mostly within two levels.
238 | 	 */
239 | 	return fuzz_multi(arg, 0x1f);
240 | }
241 | 
242 | static void *
243 | fuzz_multi_512(void *arg)
244 | {
245 | 	/*
246 | 	 * Key range of 512 ought to create multiple levels.
247 | 	 */
248 | 	return fuzz_multi(arg, 0x1ff);
249 | }
250 | 
251 | static void
252 | run_test(void *func(void *))
253 | {
254 | 	pthread_t *thr;
255 | 
256 | 	puts(".");
257 | 	map = thmap_create(0, NULL, 0);
258 | 	nworkers = sysconf(_SC_NPROCESSORS_CONF) + 1;
259 | 
260 | 	thr = malloc(sizeof(pthread_t) * nworkers);
261 | 	pthread_barrier_init(&barrier, NULL, nworkers);
262 | 
263 | 	for (unsigned i = 0; i < nworkers; i++) {
264 | 		if ((errno = pthread_create(&thr[i], NULL,
265 | 		    func, (void *)(uintptr_t)i)) != 0) {
266 | 			err(EXIT_FAILURE, "pthread_create");
267 | 		}
268 | 	}
269 | 	for (unsigned i = 0; i < nworkers; i++) {
270 | 		pthread_join(thr[i], NULL);
271 | 	}
272 | 	pthread_barrier_destroy(&barrier);
273 | 	thmap_destroy(map);
274 | 	free(thr);
275 | }
276 | 
277 | int
278 | main(void)
279 | {
280 | 	prepare_collisions();
281 | 	run_test(fuzz_root_collision);
282 | 	run_test(fuzz_l0_collision);
283 | 	run_test(fuzz_multi_collision);
284 | 	run_test(fuzz_multi_128);
285 | 	run_test(fuzz_multi_512);
286 | 	puts("ok");
287 | 	return 0;
288 | }
289 | 


--------------------------------------------------------------------------------
/src/t_thmap.c:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2017 Mindaugas Rasiukevicius <rmind at noxt eu>
  3 |  * All rights reserved.
  4 |  *
  5 |  * Use is subject to license terms, as specified in the LICENSE file.
  6 |  */
  7 | 
  8 | #include <stdio.h>
  9 | #include <stdlib.h>
 10 | #include <string.h>
 11 | #include <inttypes.h>
 12 | #include <assert.h>
 13 | 
 14 | #include "utils.h"
 15 | #include "thmap.h"
 16 | 
 17 | #define	NUM2PTR(x)	((void *)(uintptr_t)(x))
 18 | 
 19 | static unsigned		space_allocated = 0;
 20 | static const unsigned	space_off = 4;
 21 | static unsigned char	space[42500] __aligned(4);
 22 | 
 23 | static void
 24 | test_basic(void)
 25 | {
 26 | 	thmap_t *hmap;
 27 | 	void *ret;
 28 | 
 29 | 	hmap = thmap_create(0, NULL, 0);
 30 | 	assert(hmap != NULL);
 31 | 
 32 | 	ret = thmap_get(hmap, "test", 4);
 33 | 	assert(ret == NULL);
 34 | 
 35 | 	ret = thmap_put(hmap, "test", 4, NUM2PTR(0x55));
 36 | 	assert(ret == NUM2PTR(0x55));
 37 | 
 38 | 	ret = thmap_put(hmap, "test", 4, NUM2PTR(0x01));
 39 | 	assert(ret == NUM2PTR(0x55));
 40 | 
 41 | 	ret = thmap_get(hmap, "test", 4);
 42 | 	assert(ret == NUM2PTR(0x55));
 43 | 
 44 | 	ret = thmap_del(hmap, "test", 4);
 45 | 	assert(ret == NUM2PTR(0x55));
 46 | 
 47 | 	ret = thmap_get(hmap, "test", 4);
 48 | 	assert(ret == NULL);
 49 | 
 50 | 	ret = thmap_del(hmap, "test", 4);
 51 | 	assert(ret == NULL);
 52 | 
 53 | 	thmap_destroy(hmap);
 54 | }
 55 | 
 56 | static void
 57 | test_large(void)
 58 | {
 59 | 	const unsigned nitems = 1024 * 1024;
 60 | 	thmap_t *hmap;
 61 | 	void *ret;
 62 | 
 63 | 	hmap = thmap_create(0, NULL, 0);
 64 | 	assert(hmap != NULL);
 65 | 
 66 | 	for (unsigned i = 0; i < nitems; i++) {
 67 | 		ret = thmap_put(hmap, &i, sizeof(int), NUM2PTR(i));
 68 | 		assert(ret == NUM2PTR(i));
 69 | 
 70 | 		ret = thmap_get(hmap, &i, sizeof(int));
 71 | 		assert(ret == NUM2PTR(i));
 72 | 	}
 73 | 
 74 | 	for (unsigned i = 0; i < nitems; i++) {
 75 | 		ret = thmap_get(hmap, &i, sizeof(int));
 76 | 		assert(ret == NUM2PTR(i));
 77 | 	}
 78 | 
 79 | 	for (unsigned i = 0; i < nitems; i++) {
 80 | 		ret = thmap_del(hmap, &i, sizeof(int));
 81 | 		assert(ret == NUM2PTR(i));
 82 | 
 83 | 		ret = thmap_get(hmap, &i, sizeof(int));
 84 | 		assert(ret == NULL);
 85 | 	}
 86 | 
 87 | 	thmap_destroy(hmap);
 88 | }
 89 | 
 90 | static void
 91 | test_delete(void)
 92 | {
 93 | 	const unsigned nitems = 300;
 94 | 	thmap_t *hmap;
 95 | 	uint64_t *keys;
 96 | 	void *ret;
 97 | 
 98 | 	hmap = thmap_create(0, NULL, 0);
 99 | 	assert(hmap != NULL);
100 | 
101 | 	keys = calloc(nitems, sizeof(uint64_t));
102 | 	assert(keys != NULL);
103 | 
104 | 	for (unsigned i = 0; i < nitems; i++) {
105 | 		keys[i] = random() + 1;
106 | 		ret = thmap_put(hmap, &keys[i], sizeof(uint64_t), NUM2PTR(i));
107 | 		assert(ret == NUM2PTR(i));
108 | 	}
109 | 
110 | 	for (unsigned i = 0; i < nitems; i++) {
111 | 		/* Delete a key. */
112 | 		ret = thmap_del(hmap, &keys[i], sizeof(uint64_t));
113 | 		assert(ret == NUM2PTR(i));
114 | 
115 | 		/* Check the remaining keys. */
116 | 		for (unsigned j = i + 1; j < nitems; j++) {
117 | 			ret = thmap_get(hmap, &keys[j], sizeof(uint64_t));
118 | 			assert(ret == NUM2PTR(j));
119 | 		}
120 | 		thmap_gc(hmap, thmap_stage_gc(hmap));
121 | 	}
122 | 	thmap_destroy(hmap);
123 | 	free(keys);
124 | }
125 | 
126 | static void
127 | test_longkey(void)
128 | {
129 | 	thmap_t *hmap;
130 | 	void *buf, *ret;
131 | 
132 | 	hmap = thmap_create(0, NULL, 0);
133 | 	assert(hmap != NULL);
134 | 
135 | 	buf = malloc(32 * 1024);
136 | 	assert(buf != NULL);
137 | 	memset(buf, 0x11, 32 * 1024);
138 | 
139 | 	for (unsigned i = 1; i < 32; i++) {
140 | 		ret = thmap_put(hmap, buf, i * 1024, NUM2PTR(i));
141 | 		assert(ret == NUM2PTR(i));
142 | 	}
143 | 	for (unsigned i = 1; i < 32; i++) {
144 | 		ret = thmap_get(hmap, buf, i * 1024);
145 | 		assert(ret == NUM2PTR(i));
146 | 	}
147 | 	for (unsigned i = 1; i < 32; i++) {
148 | 		ret = thmap_del(hmap, buf, i * 1024);
149 | 		assert(ret == NUM2PTR(i));
150 | 	}
151 | 
152 | 	thmap_destroy(hmap);
153 | 	free(buf);
154 | }
155 | 
156 | static void *
157 | generate_unique_key(unsigned idx, int *rlen)
158 | {
159 | 	const unsigned rndlen = random() % 32;
160 | 	const unsigned len = rndlen + sizeof(idx);
161 | 	unsigned char *key = malloc(len);
162 | 
163 | 	for (unsigned i = 0; i < rndlen; i++) {
164 | 		key[i] = random() % 0xff;
165 | 	}
166 | 	memcpy(&key[rndlen], &idx, sizeof(idx));
167 | 	*rlen = len;
168 | 	return key;
169 | }
170 | 
171 | #define	KEY_MAGIC_VAL(k)	NUM2PTR(((unsigned char *)(k))[0] ^ 0x55)
172 | 
173 | static void
174 | test_random(void)
175 | {
176 | 	const unsigned nitems = 300;
177 | 	unsigned n = 10 * 1000 * 1000;
178 | 	thmap_t *hmap;
179 | 	unsigned char **keys;
180 | 	int *lens;
181 | 
182 | 	hmap = thmap_create(0, NULL, 0);
183 | 	assert(hmap != NULL);
184 | 
185 | 	keys = calloc(nitems, sizeof(unsigned char *));
186 | 	assert(keys != NULL);
187 | 
188 | 	lens = calloc(nitems, sizeof(int));
189 | 	assert(lens != NULL);
190 | 
191 | 	srandom(1);
192 | 	while (n--) {
193 | 		const unsigned i = random() % nitems;
194 | 		void *val = keys[i] ? KEY_MAGIC_VAL(keys[i]) : NULL;
195 | 		void *ret;
196 | 
197 | 		switch (random() % 3) {
198 | 		case 0:
199 | 			/* Create a unique random key. */
200 | 			if (keys[i] == NULL) {
201 | 				keys[i] = generate_unique_key(i, &lens[i]);
202 | 				val = KEY_MAGIC_VAL(keys[i]);
203 | 				ret = thmap_put(hmap, keys[i], lens[i], val);
204 | 				assert(ret == val);
205 | 			}
206 | 			break;
207 | 		case 1:
208 | 			/* Lookup a key. */
209 | 			if (keys[i]) {
210 | 				ret = thmap_get(hmap, keys[i], lens[i]);
211 | 				assert(ret == val);
212 | 			}
213 | 			break;
214 | 		case 2:
215 | 			/* Delete a key. */
216 | 			if (keys[i]) {
217 | 				ret = thmap_del(hmap, keys[i], lens[i]);
218 | 				assert(ret == val);
219 | 				free(keys[i]);
220 | 				keys[i] = NULL;
221 | 			}
222 | 			break;
223 | 		}
224 | 		thmap_gc(hmap, thmap_stage_gc(hmap));
225 | 	}
226 | 
227 | 	for (unsigned i = 0; i < nitems; i++) {
228 | 		if (keys[i]) {
229 | 			void *ret = thmap_del(hmap, keys[i], lens[i]);
230 | 			assert(KEY_MAGIC_VAL(keys[i]) == ret);
231 | 			free(keys[i]);
232 | 		}
233 | 	}
234 | 
235 | 	thmap_destroy(hmap);
236 | 	free(keys);
237 | 	free(lens);
238 | }
239 | 
240 | static uintptr_t
241 | alloc_test_wrapper(size_t len)
242 | {
243 | 	uintptr_t p = space_allocated;
244 | 	space_allocated += roundup2(len, sizeof(void *));
245 | 	assert(space_allocated <= sizeof(space));
246 | 	return p + space_off;
247 | }
248 | 
249 | static void
250 | free_test_wrapper(uintptr_t addr, size_t len)
251 | {
252 | 	assert(addr - space_off < sizeof(space));
253 | 	assert(len < sizeof(space));
254 | 
255 | 	space_allocated -= roundup2(len, sizeof(void *));
256 | 	assert(space_allocated < sizeof(space));
257 | }
258 | 
259 | static const thmap_ops_t thmap_test_ops = {
260 | 	.alloc = alloc_test_wrapper,
261 | 	.free = free_test_wrapper
262 | };
263 | 
264 | static void
265 | test_mem(void)
266 | {
267 | 	uintptr_t baseptr = (uintptr_t)(void *)space - space_off;
268 | 	const unsigned nitems = 512;
269 | 	thmap_t *hmap;
270 | 	void *ret;
271 | 
272 | 	hmap = thmap_create(baseptr, &thmap_test_ops, 0);
273 | 	assert(hmap != NULL);
274 | 
275 | 	for (unsigned i = 0; i < nitems; i++) {
276 | 		ret = thmap_put(hmap, &i, sizeof(int), NUM2PTR(i));
277 | 		assert(ret == NUM2PTR(i));
278 | 	}
279 | 	for (unsigned i = 0; i < nitems; i++) {
280 | 		ret = thmap_get(hmap, &i, sizeof(int));
281 | 		assert(ret == NUM2PTR(i));
282 | 	}
283 | 	assert(space_allocated > 0);
284 | 
285 | 	for (unsigned i = 0; i < nitems; i++) {
286 | 		ret = thmap_del(hmap, &i, sizeof(int));
287 | 		assert(ret == NUM2PTR(i));
288 | 	}
289 | 	thmap_destroy(hmap);
290 | 
291 | 	/* All space must be freed. */
292 | 	assert(space_allocated == 0);
293 | }
294 | 
295 | int
296 | main(void)
297 | {
298 | 	test_basic();
299 | 	test_large();
300 | 	test_delete();
301 | 	test_longkey();
302 | 	test_random();
303 | 	test_mem();
304 | 	puts("ok");
305 | 	return 0;
306 | }
307 | 


--------------------------------------------------------------------------------
/src/thmap.3:
--------------------------------------------------------------------------------
  1 | .\"
  2 | .\" Copyright (c) 2018 Mindaugas Rasiukevicius <rmind at noxt eu>
  3 | .\" All rights reserved.
  4 | .\"
  5 | .\" Redistribution and use in source and binary forms, with or without
  6 | .\" modification, are permitted provided that the following conditions
  7 | .\" are met:
  8 | .\" 1. Redistributions of source code must retain the above copyright
  9 | .\"    notice, this list of conditions and the following disclaimer.
 10 | .\" 2. Redistributions in binary form must reproduce the above copyright
 11 | .\"    notice, this list of conditions and the following disclaimer in the
 12 | .\"    documentation and/or other materials provided with the distribution.
 13 | .\"
 14 | .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 15 | .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 16 | .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 17 | .\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 18 | .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 19 | .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 20 | .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 21 | .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 22 | .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 23 | .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 24 | .\" SUCH DAMAGE.
 25 | .\"
 26 | .Dd December 11, 2018
 27 | .Dt THMAP 3
 28 | .Os
 29 | .Sh NAME
 30 | .Nm thmap
 31 | .Nd concurrent trie-hash map
 32 | .Sh SYNOPSIS
 33 | .In thmap.h
 34 | .\" -----
 35 | .Ft thmap_t *
 36 | .Fn thmap_create "uintptr_t baseptr" "const thmap_ops_t *ops" "unsigned flags"
 37 | .Ft void
 38 | .Fn thmap_destroy "thmap_t *hmap"
 39 | .Ft void *
 40 | .Fn thmap_get "thmap_t *hmap" "const void *key" "size_t len"
 41 | .Ft void *
 42 | .Fn thmap_put "thmap_t *hmap" "const void *key" "size_t len" "void *val"
 43 | .Ft void *
 44 | .Fn thmap_del "thmap_t *hmap" "const void *key" "size_t len"
 45 | .Ft void *
 46 | .Fn thmap_stage_gc "thmap_t *hmap"
 47 | .Ft void
 48 | .Fn thmap_gc "thmap_t *hmap" "void *ref"
 49 | .Ft void
 50 | .Fn thmap_setroot "thmap_t *thmap" "uintptr_t root_offset"
 51 | .Ft uintptr_t
 52 | .Fn thmap_getroot "const thmap_t *thmap"
 53 | .\" -----
 54 | .Sh DESCRIPTION
 55 | Concurrent trie-hash map \(em a general purpose associative array,
 56 | combining the elements of hashing and radix trie.
 57 | Highlights:
 58 | .Pp
 59 | .Bl -hyphen -compact
 60 | .It
 61 | Very competitive performance, with logarithmic time complexity on average.
 62 | .It
 63 | Lookups are lock-free and inserts/deletes are using fine-grained locking.
 64 | .It
 65 | Incremental growth of the data structure (no large resizing/rehashing).
 66 | .It
 67 | Optional support for use with shared memory, e.g. memory-mapped file.
 68 | .El
 69 | .Pp
 70 | Delete operations (the key/data destruction) must be synchronized with
 71 | the readers using some reclamation mechanism.
 72 | .\" -----
 73 | .Sh FUNCTIONS
 74 | .Bl -tag -width thmap_create
 75 | .It Fn thmap_create
 76 | Construct a new trie-hash map.
 77 | The optional
 78 | .Fa ops
 79 | parameter can
 80 | used to set the custom allocate/free operations (see the description of
 81 | .Vt thmap_ops_t
 82 | below).
 83 | In such case, the
 84 | .Fa baseptr
 85 | is the base (start) address of the address space mapping (it must be
 86 | word-aligned).
 87 | If
 88 | .Fa ops
 89 | is set to
 90 | .Dv NULL ,
 91 | then
 92 | .Xr malloc 3
 93 | and
 94 | .Xr free 3
 95 | will be used as the default operations and
 96 | .Fa baseptr
 97 | should be set to zero.
 98 | Currently, the supported
 99 | .Fa flags
100 | are:
101 | .Bl -tag -width THMAP_NOCOPY
102 | .It Dv THMAP_NOCOPY
103 | The keys on insert will not be copied and the given pointers to them will
104 | be expected to be valid and the values constant until the key is deleted;
105 | by default, the put operation will make a copy of the key.
106 | .It Dv THMAP_SETROOT
107 | Indicate that the root of the map will be manually set using the
108 | .Fn thmap_setroot
109 | routine;
110 | by default, the map is initialized and the root node is set on
111 | .Fn thmap_create .
112 | .El
113 | .\" ---
114 | .It Fn thmap_destroy
115 | Destroy the map, freeing the memory it uses.
116 | .\" ---
117 | .It Fn thmap_get
118 | Lookup the key (of a given length) and return the value associated with it.
119 | Return
120 | .Dv NULL
121 | if the key is not found (see the
122 | .Sx CAVEATS
123 | section).
124 | .\" ---
125 | .It Fn thmap_put
126 | Insert the key with an arbitrary value.
127 | If the key is already present, return the already existing associated value
128 | without changing it.
129 | Otherwise, on a successful insert, return the given value.
130 | Just compare the result against
131 | .Fa val
132 | to test whether the insert was successful.
133 | .\" ---
134 | .It Fn thmap_del
135 | Remove the given key.
136 | If the key was present, return the associated value;
137 | otherwise return
138 | .Dv NULL .
139 | The memory associated with the entry is not released immediately, because
140 | in the concurrent environment (e.g., multi-threaded application) the caller
141 | may need to ensure it is safe to do so.
142 | It is managed using the
143 | .Fn thmap_stage_gc
144 | and
145 | .Fn thmap_gc
146 | routines.
147 | .\" ---
148 | .It Fn thmap_stage_gc
149 | Stage the currently pending entries (the memory not yet released after
150 | the deletion) for reclamation (G/C).
151 | This operation should be called
152 | .Em before
153 | the synchronization barrier.
154 | .Pp
155 | Returns a reference which must be passed to
156 | .Fn thmap_gc .
157 | Not calling the G/C function for the returned reference would result in
158 | a memory leak.
159 | .\" ---
160 | .It Fn thmap_gc
161 | Reclaim (G/C) the staged entries i.e. release any memory associated
162 | with the deleted keys.
163 | The reference must be the value returned by the call to
164 | .Fn thmap_stage_gc .
165 | .Pp
166 | This function must be called
167 | .Em after
168 | the synchronization barrier which guarantees that there are no active
169 | readers referencing the staged entries.
170 | .\" ---
171 | .El
172 | .Pp
173 | If the map is created using the
174 | .Fa THMAP_SETROOT
175 | flag, then the following functions are applicable:
176 | .\" ---
177 | .Bl -tag -width thmap_setroot
178 | .It Fn thmap_setroot
179 | Set the root node.
180 | The address must be relative to the base address, as if allocated by the
181 | .Fn thmap_ops_t::alloc
182 | routine.
183 | Return 0 on success and \-1 on failure (if already set).
184 | .It Fn thmap_getroot
185 | Get the root node address.
186 | The returned address will be relative to the base address.
187 | .El
188 | .\" ---
189 | .Pp
190 | Members of
191 | .Vt thmap_ops_t
192 | are
193 | .Bd -literal
194 |         uintptr_t (*alloc)(size_t len);
195 |         void      (*free)(uintptr_t addr, size_t len);
196 | .Ed
197 | .\" -----
198 | .Sh CAVEATS
199 | The implementation uses pointer tagging and atomic operations.
200 | This requires the base address and the allocations to provide at least word
201 | alignment.
202 | .Pp
203 | While the
204 | .Dv NULL
205 | values may be inserted,
206 | .Fn thmap_get
207 | and
208 | .Fn thmap_del
209 | cannot indicate whether the key was not found or a key with a
210 | .Dv NULL
211 | value was found.
212 | If the caller needs to indicate an "empty" value, it can use a
213 | special pointer value, such as
214 | .Li (void *)(uintptr_t)0x1 .
215 | .\" -----
216 | .Sh EXAMPLES
217 | Simple case backed by
218 | .Xr malloc 3 ,
219 | which could be used in multi-threaded environment:
220 | .Bd -literal
221 | 	thmap_t *kvmap;
222 | 	struct obj *obj;
223 | 
224 | 	kvmap = thmap_create(0, NULL);
225 | 	assert(kvmap != NULL);
226 | 	...
227 | 	obj = obj_create();
228 | 	thmap_put(kvmap, "test", sizeof("test") - 1, obj);
229 | 	...
230 | 	obj = thmap_get(kvmap, "test", sizeof("test") - 1);
231 | 	...
232 | 	thmap_destroy(kvmap);
233 | .Ed
234 | .\" -----
235 | .Sh AUTHORS
236 | .An Mindaugas Rasiukevicius Aq Mt rmind@noxt.eu
237 | 


--------------------------------------------------------------------------------
/src/thmap.c:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2018 Mindaugas Rasiukevicius <rmind at noxt eu>
  3 |  * All rights reserved.
  4 |  *
  5 |  * Use is subject to license terms, as specified in the LICENSE file.
  6 |  */
  7 | 
  8 | /*
  9 |  * Concurrent trie-hash map.
 10 |  *
 11 |  * The data structure is conceptually a radix trie on hashed keys.
 12 |  * Keys are hashed using a 32-bit function.  The root level is a special
 13 |  * case: it is managed using the compare-and-swap (CAS) atomic operation
 14 |  * and has a fanout of 64.  The subsequent levels are constructed using
 15 |  * intermediate nodes with a fanout of 16 (using 4 bits).  As more levels
 16 |  * are created, more blocks of the 32-bit hash value might be generated
 17 |  * by incrementing the seed parameter of the hash function.
 18 |  *
 19 |  * Concurrency
 20 |  *
 21 |  * - READERS: Descending is simply walking through the slot values of
 22 |  *   the intermediate nodes.  It is lock-free as there is no intermediate
 23 |  *   state: the slot is either empty or has a pointer to the child node.
 24 |  *   The main assumptions here are the following:
 25 |  *
 26 |  *   i) modifications must preserve consistency with the respect to the
 27 |  *   readers i.e. the readers can only see the valid node values;
 28 |  *
 29 |  *   ii) any invalid view must "fail" the reads, e.g. by making them
 30 |  *   re-try from the root; this is a case for deletions and is achieved
 31 |  *   using the NODE_DELETED flag.
 32 |  *
 33 |  *   iii) the node destruction must be synchronized with the readers,
 34 |  *   e.g. by using the Epoch-based reclamation or other techniques.
 35 |  *
 36 |  * - WRITERS AND LOCKING: Each intermediate node has a spin-lock (which
 37 |  *   is implemented using the NODE_LOCKED bit) -- it provides mutual
 38 |  *   exclusion amongst concurrent writers.  The lock order for the nodes
 39 |  *   is "bottom-up" i.e. they are locked as we ascend the trie.  A key
 40 |  *   constraint here is that parent pointer never changes.
 41 |  *
 42 |  * - DELETES: In addition to writer's locking, the deletion keeps the
 43 |  *   intermediate nodes in a valid state and sets the NODE_DELETED flag,
 44 |  *   to indicate that the readers must re-start the walk from the root.
 45 |  *   As the levels are collapsed, NODE_DELETED gets propagated up-tree.
 46 |  *   The leaf nodes just stay as-is until they are reclaimed.
 47 |  *
 48 |  * - ROOT LEVEL: The root level is a special case, as it is implemented
 49 |  *   as an array (rather than intermediate node).  The root-level slot can
 50 |  *   only be set using CAS and it can only be set to a valid intermediate
 51 |  *   node.  The root-level slot can only be cleared when the node it points
 52 |  *   at becomes empty, is locked and marked as NODE_DELETED (this causes
 53 |  *   the insert/delete operations to re-try until the slot is set to NULL).
 54 |  *
 55 |  * References:
 56 |  *
 57 |  *	W. Litwin, 1981, Trie Hashing.
 58 |  *	Proceedings of the 1981 ACM SIGMOD, p. 19-29
 59 |  *	https://dl.acm.org/citation.cfm?id=582322
 60 |  *
 61 |  *	P. L. Lehman and S. B. Yao.
 62 |  *	Efficient locking for concurrent operations on B-trees.
 63 |  *	ACM TODS, 6(4):650-670, 1981
 64 |  *	https://www.csd.uoc.gr/~hy460/pdf/p650-lehman.pdf
 65 |  */
 66 | 
 67 | #include <stdio.h>
 68 | #include <stdlib.h>
 69 | #include <stdbool.h>
 70 | #include <stddef.h>
 71 | #include <inttypes.h>
 72 | #include <string.h>
 73 | #include <limits.h>
 74 | 
 75 | #include "thmap.h"
 76 | #include "utils.h"
 77 | 
 78 | /*
 79 |  * The root level fanout is 64 (indexed by the last 6 bits of the hash
 80 |  * value XORed with the length).  Each subsequent level, represented by
 81 |  * intermediate nodes, has a fanout of 16 (using 4 bits).
 82 |  *
 83 |  * The hash function produces 32-bit values.
 84 |  */
 85 | 
 86 | #define	HASHVAL_BITS	(32)
 87 | #define	HASHVAL_MOD	(HASHVAL_BITS - 1)
 88 | #define	HASHVAL_SHIFT	(5)
 89 | 
 90 | #define	ROOT_BITS	(6)
 91 | #define	ROOT_SIZE	(1 << ROOT_BITS)
 92 | #define	ROOT_MASK	(ROOT_SIZE - 1)
 93 | #define	ROOT_MSBITS	(HASHVAL_BITS - ROOT_BITS)
 94 | 
 95 | #define	LEVEL_BITS	(4)
 96 | #define	LEVEL_SIZE	(1 << LEVEL_BITS)
 97 | #define	LEVEL_MASK	(LEVEL_SIZE - 1)
 98 | 
 99 | /*
100 |  * Instead of raw pointers, we use offsets from the base address.
101 |  * This accommodates the use of this data structure in shared memory,
102 |  * where mappings can be in different address spaces.
103 |  *
104 |  * The pointers must be aligned, since pointer tagging is used to
105 |  * differentiate the intermediate nodes from leaves.  We reserve the
106 |  * least significant bit.
107 |  */
108 | typedef uintptr_t thmap_ptr_t;
109 | typedef atomic_uintptr_t atomic_thmap_ptr_t;
110 | 
111 | #define	THMAP_NULL		((thmap_ptr_t)0)
112 | 
113 | #define	THMAP_LEAF_BIT		(0x1)
114 | 
115 | #define	THMAP_ALIGNED_P(p)	(((uintptr_t)(p) & 3) == 0)
116 | #define	THMAP_ALIGN(p)		((uintptr_t)(p) & ~(uintptr_t)3)
117 | #define	THMAP_INODE_P(p)	(((uintptr_t)(p) & THMAP_LEAF_BIT) == 0)
118 | 
119 | #define	THMAP_GETPTR(th, p)	((void *)((th)->baseptr + (uintptr_t)(p)))
120 | #define	THMAP_GETOFF(th, p)	((thmap_ptr_t)((uintptr_t)(p) - (th)->baseptr))
121 | #define	THMAP_NODE(th, p)	THMAP_GETPTR(th, THMAP_ALIGN(p))
122 | 
123 | /*
124 |  * State field.
125 |  */
126 | 
127 | #define	NODE_LOCKED		(1U << 31)		// lock (writers)
128 | #define	NODE_DELETED		(1U << 30)		// node deleted
129 | #define	NODE_COUNT(s)		((s) & 0x3fffffff)	// slot count mask
130 | 
131 | /*
132 |  * There are two types of nodes:
133 |  * - Intermediate nodes -- arrays pointing to another level or a leaf;
134 |  * - Leaves, which store a key-value pair.
135 |  */
136 | 
137 | typedef struct {
138 | 	atomic_uint_least32_t	state;
139 | 	thmap_ptr_t		parent;
140 | 	atomic_thmap_ptr_t	slots[LEVEL_SIZE];
141 | } thmap_inode_t;
142 | 
143 | #define	THMAP_INODE_LEN	sizeof(thmap_inode_t)
144 | 
145 | typedef struct {
146 | 	thmap_ptr_t	key;
147 | 	size_t		len;
148 | 	void *		val;
149 | } thmap_leaf_t;
150 | 
151 | typedef struct {
152 | 	unsigned	rslot;		// root-level slot index
153 | 	unsigned	level;		// current level in the tree
154 | 	unsigned	hashidx;	// current hash index (block of bits)
155 | 	uint32_t	hashval;	// current hash value
156 | } thmap_query_t;
157 | 
158 | typedef struct {
159 | 	uintptr_t	addr;
160 | 	size_t		len;
161 | 	void *		next;
162 | } thmap_gc_t;
163 | 
164 | #define	THMAP_ROOT_LEN	(sizeof(thmap_ptr_t) * ROOT_SIZE)
165 | 
166 | struct thmap {
167 | 	uintptr_t		baseptr;
168 | 	atomic_thmap_ptr_t *	root;
169 | 	unsigned		flags;
170 | 	const thmap_ops_t *	ops;
171 | 	thmap_gc_t *_Atomic	gc_list;
172 | };
173 | 
174 | static void	stage_mem_gc(thmap_t *, uintptr_t, size_t);
175 | 
176 | /*
177 |  * A few low-level helper routines.
178 |  */
179 | 
180 | static uintptr_t
181 | alloc_wrapper(size_t len)
182 | {
183 | 	return (uintptr_t)malloc(len);
184 | }
185 | 
186 | static void
187 | free_wrapper(uintptr_t addr, size_t len)
188 | {
189 | 	free((void *)addr); (void)len;
190 | }
191 | 
192 | static const thmap_ops_t thmap_default_ops = {
193 | 	.alloc = alloc_wrapper,
194 | 	.free = free_wrapper
195 | };
196 | 
197 | /*
198 |  * NODE LOCKING.
199 |  */
200 | 
201 | #ifdef DEBUG
202 | static inline bool
203 | node_locked_p(thmap_inode_t *node)
204 | {
205 | 	return (atomic_load_relaxed(&node->state) & NODE_LOCKED) != 0;
206 | }
207 | #endif
208 | 
209 | static void
210 | lock_node(thmap_inode_t *node)
211 | {
212 | 	unsigned bcount = SPINLOCK_BACKOFF_MIN;
213 | 	uint32_t s;
214 | again:
215 | 	s = atomic_load_relaxed(&node->state);
216 | 	if (s & NODE_LOCKED) {
217 | 		SPINLOCK_BACKOFF(bcount);
218 | 		goto again;
219 | 	}
220 | 	/* Acquire from prior release in unlock_node.() */
221 | 	if (!atomic_compare_exchange_weak_explicit(&node->state,
222 | 	    &s, s | NODE_LOCKED, memory_order_acquire, memory_order_relaxed)) {
223 | 		bcount = SPINLOCK_BACKOFF_MIN;
224 | 		goto again;
225 | 	}
226 | }
227 | 
228 | static void
229 | unlock_node(thmap_inode_t *node)
230 | {
231 | 	uint32_t s = atomic_load_relaxed(&node->state) & ~NODE_LOCKED;
232 | 
233 | 	ASSERT(node_locked_p(node));
234 | 	/* Release to subsequent acquire in lock_node(). */
235 | 	atomic_store_release(&node->state, s);
236 | }
237 | 
238 | /*
239 |  * HASH VALUE AND KEY OPERATIONS.
240 |  */
241 | 
242 | static inline void
243 | hashval_init(thmap_query_t *query, const void * restrict key, size_t len)
244 | {
245 | 	const uint32_t hashval = murmurhash3(key, len, 0);
246 | 
247 | 	query->rslot = ((hashval >> ROOT_MSBITS) ^ len) & ROOT_MASK;
248 | 	query->level = 0;
249 | 	query->hashval = hashval;
250 | 	query->hashidx = 0;
251 | }
252 | 
253 | /*
254 |  * hashval_getslot: given the key, compute the hash (if not already cached)
255 |  * and return the offset for the current level.
256 |  */
257 | static unsigned
258 | hashval_getslot(thmap_query_t *query, const void * restrict key, size_t len)
259 | {
260 | 	const unsigned offset = query->level * LEVEL_BITS;
261 | 	const unsigned shift = offset & HASHVAL_MOD;
262 | 	const unsigned i = offset >> HASHVAL_SHIFT;
263 | 
264 | 	if (query->hashidx != i) {
265 | 		/* Generate a hash value for a required range. */
266 | 		query->hashval = murmurhash3(key, len, i);
267 | 		query->hashidx = i;
268 | 	}
269 | 	return (query->hashval >> shift) & LEVEL_MASK;
270 | }
271 | 
272 | static unsigned
273 | hashval_getleafslot(const thmap_t *thmap,
274 |     const thmap_leaf_t *leaf, unsigned level)
275 | {
276 | 	const void *key = THMAP_GETPTR(thmap, leaf->key);
277 | 	const unsigned offset = level * LEVEL_BITS;
278 | 	const unsigned shift = offset & HASHVAL_MOD;
279 | 	const unsigned i = offset >> HASHVAL_SHIFT;
280 | 
281 | 	return (murmurhash3(key, leaf->len, i) >> shift) & LEVEL_MASK;
282 | }
283 | 
284 | static inline unsigned
285 | hashval_getl0slot(const thmap_t *thmap, const thmap_query_t *query,
286 |     const thmap_leaf_t *leaf)
287 | {
288 | 	if (__predict_true(query->hashidx == 0)) {
289 | 		return query->hashval & LEVEL_MASK;
290 | 	}
291 | 	return hashval_getleafslot(thmap, leaf, 0);
292 | }
293 | 
294 | static bool
295 | key_cmp_p(const thmap_t *thmap, const thmap_leaf_t *leaf,
296 |     const void * restrict key, size_t len)
297 | {
298 | 	const void *leafkey = THMAP_GETPTR(thmap, leaf->key);
299 | 	return len == leaf->len && memcmp(key, leafkey, len) == 0;
300 | }
301 | 
302 | /*
303 |  * INTER-NODE OPERATIONS.
304 |  */
305 | 
306 | static thmap_inode_t *
307 | node_create(thmap_t *thmap, thmap_inode_t *parent)
308 | {
309 | 	thmap_inode_t *node;
310 | 	uintptr_t p;
311 | 
312 | 	p = thmap->ops->alloc(THMAP_INODE_LEN);
313 | 	if (!p) {
314 | 		return NULL;
315 | 	}
316 | 	node = THMAP_GETPTR(thmap, p);
317 | 	ASSERT(THMAP_ALIGNED_P(node));
318 | 
319 | 	memset(node, 0, THMAP_INODE_LEN);
320 | 	if (parent) {
321 | 		/* Not yet published, no need for ordering. */
322 | 		atomic_store_relaxed(&node->state, NODE_LOCKED);
323 | 		node->parent = THMAP_GETOFF(thmap, parent);
324 | 	}
325 | 	return node;
326 | }
327 | 
328 | static void
329 | node_insert(thmap_inode_t *node, unsigned slot, thmap_ptr_t child)
330 | {
331 | 	ASSERT(node_locked_p(node) || node->parent == THMAP_NULL);
332 | 	ASSERT((atomic_load_relaxed(&node->state) & NODE_DELETED) == 0);
333 | 	ASSERT(atomic_load_relaxed(&node->slots[slot]) == THMAP_NULL);
334 | 
335 | 	ASSERT(NODE_COUNT(atomic_load_relaxed(&node->state)) < LEVEL_SIZE);
336 | 
337 | 	/*
338 | 	 * If node is public already, caller is responsible for issuing
339 | 	 * release fence; if node is not public, no ordering is needed.
340 | 	 * Hence relaxed ordering.
341 | 	 */
342 | 	atomic_store_relaxed(&node->slots[slot], child);
343 | 	atomic_store_relaxed(&node->state,
344 | 	    atomic_load_relaxed(&node->state) + 1);
345 | }
346 | 
347 | static void
348 | node_remove(thmap_inode_t *node, unsigned slot)
349 | {
350 | 	const uint32_t state = atomic_load_relaxed(&node->state);
351 | 
352 | 	ASSERT(node_locked_p(node));
353 | 	ASSERT((state & NODE_DELETED) == 0);
354 | 	ASSERT(atomic_load_relaxed(&node->slots[slot]) != THMAP_NULL);
355 | 
356 | 	ASSERT(NODE_COUNT(state) > 0);
357 | 	ASSERT(NODE_COUNT(state) <= LEVEL_SIZE);
358 | 
359 | 	/* Element will be GC-ed later; no need for ordering here. */
360 | 	atomic_store_relaxed(&node->slots[slot], THMAP_NULL);
361 | 	atomic_store_relaxed(&node->state, state - 1);
362 | }
363 | 
364 | /*
365 |  * LEAF OPERATIONS.
366 |  */
367 | 
368 | static thmap_leaf_t *
369 | leaf_create(const thmap_t *thmap, const void *key, size_t len, void *val)
370 | {
371 | 	thmap_leaf_t *leaf;
372 | 	uintptr_t leaf_off, key_off;
373 | 
374 | 	leaf_off = thmap->ops->alloc(sizeof(thmap_leaf_t));
375 | 	if (!leaf_off) {
376 | 		return NULL;
377 | 	}
378 | 	leaf = THMAP_GETPTR(thmap, leaf_off);
379 | 	ASSERT(THMAP_ALIGNED_P(leaf));
380 | 
381 | 	if ((thmap->flags & THMAP_NOCOPY) == 0) {
382 | 		/*
383 | 		 * Copy the key.
384 | 		 */
385 | 		key_off = thmap->ops->alloc(len);
386 | 		if (!key_off) {
387 | 			thmap->ops->free(leaf_off, sizeof(thmap_leaf_t));
388 | 			return NULL;
389 | 		}
390 | 		memcpy(THMAP_GETPTR(thmap, key_off), key, len);
391 | 		leaf->key = key_off;
392 | 	} else {
393 | 		/* Otherwise, we use a reference. */
394 | 		leaf->key = (uintptr_t)key;
395 | 	}
396 | 	leaf->len = len;
397 | 	leaf->val = val;
398 | 	return leaf;
399 | }
400 | 
401 | static void
402 | leaf_free(const thmap_t *thmap, thmap_leaf_t *leaf)
403 | {
404 | 	if ((thmap->flags & THMAP_NOCOPY) == 0) {
405 | 		thmap->ops->free(leaf->key, leaf->len);
406 | 	}
407 | 	thmap->ops->free(THMAP_GETOFF(thmap, leaf), sizeof(thmap_leaf_t));
408 | }
409 | 
410 | static thmap_leaf_t *
411 | get_leaf(const thmap_t *thmap, thmap_inode_t *parent, unsigned slot)
412 | {
413 | 	thmap_ptr_t node;
414 | 
415 | 	/* Consume from prior release in thmap_put(). */
416 | 	node = atomic_load_consume(&parent->slots[slot]);
417 | 	if (THMAP_INODE_P(node)) {
418 | 		return NULL;
419 | 	}
420 | 	return THMAP_NODE(thmap, node);
421 | }
422 | 
423 | /*
424 |  * ROOT OPERATIONS.
425 |  */
426 | 
427 | /*
428 |  * root_try_put: Try to set a root pointer at query->rslot.
429 |  *
430 |  * => Implies release operation on success.
431 |  * => Implies no ordering on failure.
432 |  */
433 | static inline bool
434 | root_try_put(thmap_t *thmap, const thmap_query_t *query, thmap_leaf_t *leaf)
435 | {
436 | 	thmap_ptr_t expected;
437 | 	const unsigned i = query->rslot;
438 | 	thmap_inode_t *node;
439 | 	thmap_ptr_t nptr;
440 | 	unsigned slot;
441 | 
442 | 	/*
443 | 	 * Must pre-check first.  No ordering required because we will
444 | 	 * check again before taking any actions, and start over if
445 | 	 * this changes from null.
446 | 	 */
447 | 	if (atomic_load_relaxed(&thmap->root[i])) {
448 | 		return false;
449 | 	}
450 | 
451 | 	/*
452 | 	 * Create an intermediate node.  Since there is no parent set,
453 | 	 * it will be created unlocked and the CAS operation will
454 | 	 * release it to readers.
455 | 	 */
456 | 	node = node_create(thmap, NULL);
457 | 	slot = hashval_getl0slot(thmap, query, leaf);
458 | 	node_insert(node, slot, THMAP_GETOFF(thmap, leaf) | THMAP_LEAF_BIT);
459 | 	nptr = THMAP_GETOFF(thmap, node);
460 | again:
461 | 	if (atomic_load_relaxed(&thmap->root[i])) {
462 | 		thmap->ops->free(nptr, THMAP_INODE_LEN);
463 | 		return false;
464 | 	}
465 | 	/* Release to subsequent consume in find_edge_node(). */
466 | 	expected = THMAP_NULL;
467 | 	if (!atomic_compare_exchange_weak_explicit(&thmap->root[i], &expected,
468 | 	    nptr, memory_order_release, memory_order_relaxed)) {
469 | 		goto again;
470 | 	}
471 | 	return true;
472 | }
473 | 
474 | /*
475 |  * find_edge_node: given the hash, traverse the tree to find the edge node.
476 |  *
477 |  * => Returns an aligned (clean) pointer to the parent node.
478 |  * => Returns the slot number and sets current level.
479 |  */
480 | static thmap_inode_t *
481 | find_edge_node(const thmap_t *thmap, thmap_query_t *query,
482 |     const void * restrict key, size_t len, unsigned *slot)
483 | {
484 | 	thmap_ptr_t root_slot;
485 | 	thmap_inode_t *parent;
486 | 	thmap_ptr_t node;
487 | 	unsigned off;
488 | 
489 | 	ASSERT(query->level == 0);
490 | 
491 | 	/* Consume from prior release in root_try_put(). */
492 | 	root_slot = atomic_load_consume(&thmap->root[query->rslot]);
493 | 	parent = THMAP_NODE(thmap, root_slot);
494 | 	if (!parent) {
495 | 		return NULL;
496 | 	}
497 | descend:
498 | 	off = hashval_getslot(query, key, len);
499 | 	/* Consume from prior release in thmap_put(). */
500 | 	node = atomic_load_consume(&parent->slots[off]);
501 | 
502 | 	/* Descend the tree until we find a leaf or empty slot. */
503 | 	if (node && THMAP_INODE_P(node)) {
504 | 		parent = THMAP_NODE(thmap, node);
505 | 		query->level++;
506 | 		goto descend;
507 | 	}
508 | 	/*
509 | 	 * NODE_DELETED does not become stale until GC runs, which
510 | 	 * cannot happen while we are in the middle of an operation,
511 | 	 * hence relaxed ordering.
512 | 	 */
513 | 	if (atomic_load_relaxed(&parent->state) & NODE_DELETED) {
514 | 		return NULL;
515 | 	}
516 | 	*slot = off;
517 | 	return parent;
518 | }
519 | 
520 | /*
521 |  * find_edge_node_locked: traverse the tree, like find_edge_node(),
522 |  * but attempt to lock the edge node.
523 |  *
524 |  * => Returns NULL if the deleted node is found.  This indicates that
525 |  *    the caller must re-try from the root, as the root slot might have
526 |  *    changed too.
527 |  */
528 | static thmap_inode_t *
529 | find_edge_node_locked(const thmap_t *thmap, thmap_query_t *query,
530 |     const void * restrict key, size_t len, unsigned *slot)
531 | {
532 | 	thmap_inode_t *node;
533 | 	thmap_ptr_t target;
534 | retry:
535 | 	/*
536 | 	 * Find the edge node and lock it!  Re-check the state since
537 | 	 * the tree might change by the time we acquire the lock.
538 | 	 */
539 | 	node = find_edge_node(thmap, query, key, len, slot);
540 | 	if (!node) {
541 | 		/* The root slot is empty -- let the caller decide. */
542 | 		query->level = 0;
543 | 		return NULL;
544 | 	}
545 | 	lock_node(node);
546 | 	if (__predict_false(atomic_load_relaxed(&node->state) & NODE_DELETED)) {
547 | 		/*
548 | 		 * The node has been deleted.  The tree might have a new
549 | 		 * shape now, therefore we must re-start from the root.
550 | 		 */
551 | 		unlock_node(node);
552 | 		query->level = 0;
553 | 		return NULL;
554 | 	}
555 | 	target = atomic_load_relaxed(&node->slots[*slot]);
556 | 	if (__predict_false(target && THMAP_INODE_P(target))) {
557 | 		/*
558 | 		 * The target slot has been changed and it is now an
559 | 		 * intermediate node.  Re-start from the top internode.
560 | 		 */
561 | 		unlock_node(node);
562 | 		query->level = 0;
563 | 		goto retry;
564 | 	}
565 | 	return node;
566 | }
567 | 
568 | /*
569 |  * thmap_get: lookup a value given the key.
570 |  */
571 | void *
572 | thmap_get(thmap_t *thmap, const void *key, size_t len)
573 | {
574 | 	thmap_query_t query;
575 | 	thmap_inode_t *parent;
576 | 	thmap_leaf_t *leaf;
577 | 	unsigned slot;
578 | 
579 | 	hashval_init(&query, key, len);
580 | 	parent = find_edge_node(thmap, &query, key, len, &slot);
581 | 	if (!parent) {
582 | 		return NULL;
583 | 	}
584 | 	leaf = get_leaf(thmap, parent, slot);
585 | 	if (!leaf) {
586 | 		return NULL;
587 | 	}
588 | 	if (!key_cmp_p(thmap, leaf, key, len)) {
589 | 		return NULL;
590 | 	}
591 | 	return leaf->val;
592 | }
593 | 
594 | /*
595 |  * thmap_put: insert a value given the key.
596 |  *
597 |  * => If the key is already present, return the associated value.
598 |  * => Otherwise, on successful insert, return the given value.
599 |  */
600 | void *
601 | thmap_put(thmap_t *thmap, const void *key, size_t len, void *val)
602 | {
603 | 	thmap_query_t query;
604 | 	thmap_leaf_t *leaf, *other;
605 | 	thmap_inode_t *parent, *child;
606 | 	unsigned slot, other_slot;
607 | 	thmap_ptr_t target;
608 | 
609 | 	/*
610 | 	 * First, pre-allocate and initialize the leaf node.
611 | 	 */
612 | 	leaf = leaf_create(thmap, key, len, val);
613 | 	if (__predict_false(!leaf)) {
614 | 		return NULL;
615 | 	}
616 | 	hashval_init(&query, key, len);
617 | retry:
618 | 	/*
619 | 	 * Try to insert into the root first, if its slot is empty.
620 | 	 */
621 | 	if (root_try_put(thmap, &query, leaf)) {
622 | 		/* Success: the leaf was inserted; no locking involved. */
623 | 		return val;
624 | 	}
625 | 
626 | 	/*
627 | 	 * Release node via store in node_insert (*) to subsequent
628 | 	 * consume in get_leaf() or find_edge_node().
629 | 	 */
630 | 	atomic_thread_fence(memory_order_release);
631 | 
632 | 	/*
633 | 	 * Find the edge node and the target slot.
634 | 	 */
635 | 	parent = find_edge_node_locked(thmap, &query, key, len, &slot);
636 | 	if (!parent) {
637 | 		goto retry;
638 | 	}
639 | 	target = atomic_load_relaxed(&parent->slots[slot]); // tagged offset
640 | 	if (THMAP_INODE_P(target)) {
641 | 		/*
642 | 		 * Empty slot: simply insert the new leaf.  The release
643 | 		 * fence is already issued for us.
644 | 		 */
645 | 		target = THMAP_GETOFF(thmap, leaf) | THMAP_LEAF_BIT;
646 | 		node_insert(parent, slot, target); /* (*) */
647 | 		goto out;
648 | 	}
649 | 
650 | 	/*
651 | 	 * Collision or duplicate.
652 | 	 */
653 | 	other = THMAP_NODE(thmap, target);
654 | 	if (key_cmp_p(thmap, other, key, len)) {
655 | 		/*
656 | 		 * Duplicate.  Free the pre-allocated leaf and
657 | 		 * return the present value.
658 | 		 */
659 | 		leaf_free(thmap, leaf);
660 | 		val = other->val;
661 | 		goto out;
662 | 	}
663 | descend:
664 | 	/*
665 | 	 * Collision -- expand the tree.  Create an intermediate node
666 | 	 * which will be locked (NODE_LOCKED) for us.  At this point,
667 | 	 * we advance to the next level.
668 | 	 */
669 | 	child = node_create(thmap, parent);
670 | 	if (__predict_false(!child)) {
671 | 		leaf_free(thmap, leaf);
672 | 		val = NULL;
673 | 		goto out;
674 | 	}
675 | 	query.level++;
676 | 
677 | 	/*
678 | 	 * Insert the other (colliding) leaf first.  The new child is
679 | 	 * not yet published, so memory order is relaxed.
680 | 	 */
681 | 	other_slot = hashval_getleafslot(thmap, other, query.level);
682 | 	target = THMAP_GETOFF(thmap, other) | THMAP_LEAF_BIT;
683 | 	node_insert(child, other_slot, target);
684 | 
685 | 	/*
686 | 	 * Insert the intermediate node into the parent node.
687 | 	 * It becomes the new parent for the our new leaf.
688 | 	 *
689 | 	 * Ensure that stores to the child (and leaf) reach global
690 | 	 * visibility before it gets inserted to the parent, as
691 | 	 * consumed by get_leaf() or find_edge_node().
692 | 	 */
693 | 	atomic_store_release(&parent->slots[slot], THMAP_GETOFF(thmap, child));
694 | 
695 | 	unlock_node(parent);
696 | 	ASSERT(node_locked_p(child));
697 | 	parent = child;
698 | 
699 | 	/*
700 | 	 * Get the new slot and check for another collision
701 | 	 * at the next level.
702 | 	 */
703 | 	slot = hashval_getslot(&query, key, len);
704 | 	if (slot == other_slot) {
705 | 		/* Another collision -- descend and expand again. */
706 | 		goto descend;
707 | 	}
708 | 
709 | 	/*
710 | 	 * Insert our new leaf once we expanded enough.  The release
711 | 	 * fence is already issued for us.
712 | 	 */
713 | 	target = THMAP_GETOFF(thmap, leaf) | THMAP_LEAF_BIT;
714 | 	node_insert(parent, slot, target); /* (*) */
715 | out:
716 | 	unlock_node(parent);
717 | 	return val;
718 | }
719 | 
720 | /*
721 |  * thmap_del: remove the entry given the key.
722 |  */
723 | void *
724 | thmap_del(thmap_t *thmap, const void *key, size_t len)
725 | {
726 | 	thmap_query_t query;
727 | 	thmap_leaf_t *leaf;
728 | 	thmap_inode_t *parent;
729 | 	unsigned slot;
730 | 	void *val;
731 | 
732 | 	hashval_init(&query, key, len);
733 | 	parent = find_edge_node_locked(thmap, &query, key, len, &slot);
734 | 	if (!parent) {
735 | 		/* Root slot empty: not found. */
736 | 		return NULL;
737 | 	}
738 | 	leaf = get_leaf(thmap, parent, slot);
739 | 	if (!leaf || !key_cmp_p(thmap, leaf, key, len)) {
740 | 		/* Not found. */
741 | 		unlock_node(parent);
742 | 		return NULL;
743 | 	}
744 | 
745 | 	/* Remove the leaf. */
746 | 	ASSERT(THMAP_NODE(thmap, atomic_load_relaxed(&parent->slots[slot]))
747 | 	    == leaf);
748 | 	node_remove(parent, slot);
749 | 
750 | 	/*
751 | 	 * Collapse the levels if removing the last item.
752 | 	 */
753 | 	while (query.level &&
754 | 	    NODE_COUNT(atomic_load_relaxed(&parent->state)) == 0) {
755 | 		thmap_inode_t *node = parent;
756 | 
757 | 		ASSERT(atomic_load_relaxed(&node->state) == NODE_LOCKED);
758 | 
759 | 		/*
760 | 		 * Ascend one level up.
761 | 		 * => Mark our current parent as deleted.
762 | 		 * => Lock the parent one level up.
763 | 		 */
764 | 		query.level--;
765 | 		slot = hashval_getslot(&query, key, len);
766 | 		parent = THMAP_NODE(thmap, node->parent);
767 | 		ASSERT(parent != NULL);
768 | 
769 | 		lock_node(parent);
770 | 		ASSERT((atomic_load_relaxed(&parent->state) & NODE_DELETED)
771 | 		    == 0);
772 | 
773 | 		/*
774 | 		 * Lock is exclusive, so nobody else can be writing at
775 | 		 * the same time, and no need for atomic R/M/W, but
776 | 		 * readers may read without the lock and so need atomic
777 | 		 * load/store.  No ordering here needed because the
778 | 		 * entry itself stays valid until GC.
779 | 		 */
780 | 		atomic_store_relaxed(&node->state,
781 | 		    atomic_load_relaxed(&node->state) | NODE_DELETED);
782 | 		unlock_node(node); // memory_order_release
783 | 
784 | 		ASSERT(THMAP_NODE(thmap,
785 | 		    atomic_load_relaxed(&parent->slots[slot])) == node);
786 | 		node_remove(parent, slot);
787 | 
788 | 		/* Stage the removed node for G/C. */
789 | 		stage_mem_gc(thmap, THMAP_GETOFF(thmap, node), THMAP_INODE_LEN);
790 | 	}
791 | 
792 | 	/*
793 | 	 * If the top node is empty, then we need to remove it from the
794 | 	 * root level.  Mark the node as deleted and clear the slot.
795 | 	 *
796 | 	 * Note: acquiring the lock on the top node effectively prevents
797 | 	 * the root slot from changing.
798 | 	 */
799 | 	if (NODE_COUNT(atomic_load_relaxed(&parent->state)) == 0) {
800 | 		const unsigned rslot = query.rslot;
801 | 		const thmap_ptr_t nptr =
802 | 		    atomic_load_relaxed(&thmap->root[rslot]);
803 | 
804 | 		ASSERT(query.level == 0);
805 | 		ASSERT(parent->parent == THMAP_NULL);
806 | 		ASSERT(THMAP_GETOFF(thmap, parent) == nptr);
807 | 
808 | 		/* Mark as deleted and remove from the root-level slot. */
809 | 		atomic_store_relaxed(&parent->state,
810 | 		    atomic_load_relaxed(&parent->state) | NODE_DELETED);
811 | 		atomic_store_relaxed(&thmap->root[rslot], THMAP_NULL);
812 | 
813 | 		stage_mem_gc(thmap, nptr, THMAP_INODE_LEN);
814 | 	}
815 | 	unlock_node(parent);
816 | 
817 | 	/*
818 | 	 * Save the value and stage the leaf for G/C.
819 | 	 */
820 | 	val = leaf->val;
821 | 	if ((thmap->flags & THMAP_NOCOPY) == 0) {
822 | 		stage_mem_gc(thmap, leaf->key, leaf->len);
823 | 	}
824 | 	stage_mem_gc(thmap, THMAP_GETOFF(thmap, leaf), sizeof(thmap_leaf_t));
825 | 	return val;
826 | }
827 | 
828 | /*
829 |  * G/C routines.
830 |  */
831 | 
832 | static void
833 | stage_mem_gc(thmap_t *thmap, uintptr_t addr, size_t len)
834 | {
835 | 	thmap_gc_t *head, *gc;
836 | 
837 | 	gc = malloc(sizeof(thmap_gc_t));
838 | 	gc->addr = addr;
839 | 	gc->len = len;
840 | retry:
841 | 	head = atomic_load_relaxed(&thmap->gc_list);
842 | 	gc->next = head; // not yet published
843 | 
844 | 	/* Release to subsequent acquire in thmap_stage_gc(). */
845 | 	if (!atomic_compare_exchange_weak_explicit(&thmap->gc_list, &head, gc,
846 | 	    memory_order_release, memory_order_relaxed)) {
847 | 		goto retry;
848 | 	}
849 | }
850 | 
851 | void *
852 | thmap_stage_gc(thmap_t *thmap)
853 | {
854 | 	/* Acquire from prior release in stage_mem_gc(). */
855 | 	return atomic_exchange_explicit(&thmap->gc_list, NULL,
856 | 	    memory_order_acquire);
857 | }
858 | 
859 | void
860 | thmap_gc(thmap_t *thmap, void *ref)
861 | {
862 | 	thmap_gc_t *gc = ref;
863 | 
864 | 	while (gc) {
865 | 		thmap_gc_t *next = gc->next;
866 | 		thmap->ops->free(gc->addr, gc->len);
867 | 		free(gc);
868 | 		gc = next;
869 | 	}
870 | }
871 | 
872 | /*
873 |  * thmap_create: construct a new trie-hash map object.
874 |  */
875 | thmap_t *
876 | thmap_create(uintptr_t baseptr, const thmap_ops_t *ops, unsigned flags)
877 | {
878 | 	thmap_t *thmap;
879 | 	uintptr_t root;
880 | 
881 | 	/*
882 | 	 * Setup the map object.
883 | 	 */
884 | 	if (!THMAP_ALIGNED_P(baseptr)) {
885 | 		return NULL;
886 | 	}
887 | 	thmap = calloc(1, sizeof(thmap_t));
888 | 	if (!thmap) {
889 | 		return NULL;
890 | 	}
891 | 	thmap->baseptr = baseptr;
892 | 	thmap->ops = ops ? ops : &thmap_default_ops;
893 | 	thmap->flags = flags;
894 | 
895 | 	if ((thmap->flags & THMAP_SETROOT) == 0) {
896 | 		/* Allocate the root level. */
897 | 		root = thmap->ops->alloc(THMAP_ROOT_LEN);
898 | 		if (!root) {
899 | 			free(thmap);
900 | 			return NULL;
901 | 		}
902 | 		thmap->root = THMAP_GETPTR(thmap, root);
903 | 		memset(thmap->root, 0, THMAP_ROOT_LEN);
904 | 		atomic_thread_fence(memory_order_release); /* XXX */
905 | 	}
906 | 	return thmap;
907 | }
908 | 
909 | int
910 | thmap_setroot(thmap_t *thmap, uintptr_t root_off)
911 | {
912 | 	if (thmap->root) {
913 | 		return -1;
914 | 	}
915 | 	thmap->root = THMAP_GETPTR(thmap, root_off);
916 | 	atomic_thread_fence(memory_order_release); /* XXX */
917 | 	return 0;
918 | }
919 | 
920 | uintptr_t
921 | thmap_getroot(const thmap_t *thmap)
922 | {
923 | 	return THMAP_GETOFF(thmap, thmap->root);
924 | }
925 | 
926 | void
927 | thmap_destroy(thmap_t *thmap)
928 | {
929 | 	uintptr_t root = THMAP_GETOFF(thmap, thmap->root);
930 | 	void *ref;
931 | 
932 | 	ref = thmap_stage_gc(thmap);
933 | 	thmap_gc(thmap, ref);
934 | 
935 | 	if ((thmap->flags & THMAP_SETROOT) == 0) {
936 | 		thmap->ops->free(root, THMAP_ROOT_LEN);
937 | 	}
938 | 	free(thmap);
939 | }
940 | 


--------------------------------------------------------------------------------
/src/thmap.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (c) 2018 Mindaugas Rasiukevicius <rmind at noxt eu>
 3 |  * All rights reserved.
 4 |  *
 5 |  * Use is subject to license terms, as specified in the LICENSE file.
 6 |  */
 7 | 
 8 | #ifndef _THMAP_H_
 9 | #define _THMAP_H_
10 | 
11 | __BEGIN_DECLS
12 | 
13 | struct thmap;
14 | typedef struct thmap thmap_t;
15 | 
16 | #define	THMAP_NOCOPY	0x01
17 | #define	THMAP_SETROOT	0x02
18 | 
19 | typedef struct {
20 | 	uintptr_t	(*alloc)(size_t);
21 | 	void		(*free)(uintptr_t, size_t);
22 | } thmap_ops_t;
23 | 
24 | thmap_t *	thmap_create(uintptr_t, const thmap_ops_t *, unsigned);
25 | void		thmap_destroy(thmap_t *);
26 | 
27 | void *		thmap_get(thmap_t *, const void *, size_t);
28 | void *		thmap_put(thmap_t *, const void *, size_t, void *);
29 | void *		thmap_del(thmap_t *, const void *, size_t);
30 | 
31 | void *		thmap_stage_gc(thmap_t *);
32 | void		thmap_gc(thmap_t *, void *);
33 | 
34 | int		thmap_setroot(thmap_t *, uintptr_t);
35 | uintptr_t	thmap_getroot(const thmap_t *);
36 | 
37 | __END_DECLS
38 | 
39 | #endif
40 | 


--------------------------------------------------------------------------------
/src/utils.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2017 Mindaugas Rasiukevicius <rmind at noxt eu>
  3 |  * All rights reserved.
  4 |  *
  5 |  * Use is subject to license terms, as specified in the LICENSE file.
  6 |  */
  7 | 
  8 | #ifndef	_UTILS_H_
  9 | #define	_UTILS_H_
 10 | 
 11 | #include <stdatomic.h>
 12 | #include <stddef.h>
 13 | #include <inttypes.h>
 14 | #include <limits.h>
 15 | #include <assert.h>
 16 | 
 17 | /*
 18 |  * A regular assert (debug/diagnostic only).
 19 |  */
 20 | 
 21 | #if defined(DEBUG)
 22 | #define	ASSERT		assert
 23 | #else
 24 | #define	ASSERT(x)
 25 | #endif
 26 | 
 27 | /*
 28 |  * Branch prediction macros.
 29 |  */
 30 | 
 31 | #ifndef __predict_true
 32 | #define	__predict_true(x)	__builtin_expect((x) != 0, 1)
 33 | #define	__predict_false(x)	__builtin_expect((x) != 0, 0)
 34 | #endif
 35 | 
 36 | /*
 37 |  * Various C helpers and attribute macros.
 38 |  */
 39 | 
 40 | #ifndef __aligned
 41 | #define	__aligned(x)		__attribute__((__aligned__(x)))
 42 | #endif
 43 | 
 44 | /*
 45 |  * Minimum, maximum and rounding macros.
 46 |  */
 47 | 
 48 | #ifndef MIN
 49 | #define	MIN(x, y)	((x) < (y) ? (x) : (y))
 50 | #endif
 51 | 
 52 | #ifndef MAX
 53 | #define	MAX(x, y)	((x) > (y) ? (x) : (y))
 54 | #endif
 55 | 
 56 | #ifndef roundup2
 57 | #define	roundup2(x,m)	((((x) - 1) | ((m) - 1)) + 1)
 58 | #endif
 59 | 
 60 | /*
 61 |  * Atomic operations and memory barriers.  If C11 API is not available,
 62 |  * then wrap the GCC builtin routines.
 63 |  *
 64 |  * Note: This atomic_compare_exchange_weak does not do the C11 thing of
 65 |  * filling *(expected) with the actual value, because we don't need
 66 |  * that here.
 67 |  */
 68 | #ifndef atomic_compare_exchange_weak
 69 | #define	atomic_compare_exchange_weak(ptr, expected, desired) \
 70 |     __sync_bool_compare_and_swap(ptr, *(expected), desired)
 71 | #endif
 72 | #ifndef atomic_compare_exchange_weak_explicit
 73 | #define	atomic_compare_exchange_weak_explicit(ptr, expected, desired, s, f) \
 74 |     atomic_compare_exchange_weak(ptr, expected, desired)
 75 | #endif
 76 | 
 77 | #ifndef atomic_exchange
 78 | static inline void *
 79 | atomic_exchange(volatile void *ptr, void *newval)
 80 | {
 81 | 	void * volatile *ptrp = (void * volatile *)ptr;
 82 | 	void *oldval;
 83 | again:
 84 | 	oldval = *ptrp;
 85 | 	if (!__sync_bool_compare_and_swap(ptrp, oldval, newval)) {
 86 | 		goto again;
 87 | 	}
 88 | 	return oldval;
 89 | }
 90 | #define atomic_exchange_explicit(ptr, newval, o) atomic_exchange(ptr, newval)
 91 | #endif
 92 | 
 93 | #ifndef atomic_thread_fence
 94 | #define	memory_order_relaxed	__ATOMIC_RELAXED
 95 | #define	memory_order_acquire	__ATOMIC_ACQUIRE
 96 | #define	memory_order_consume	__ATOMIC_CONSUME
 97 | #define	memory_order_release	__ATOMIC_RELEASE
 98 | #define	memory_order_seq_cst	__ATOMIC_SEQ_CST
 99 | #define	atomic_thread_fence(m)	__atomic_thread_fence(m)
100 | #endif
101 | #ifndef atomic_store_explicit
102 | #define	atomic_store_explicit	__atomic_store_n
103 | #endif
104 | #ifndef atomic_load_explicit
105 | #define	atomic_load_explicit	__atomic_load_n
106 | #endif
107 | 
108 | /*
109 |  * Conciser convenience wrappers.
110 |  */
111 | 
112 | #define	atomic_load_relaxed(p)	atomic_load_explicit(p, memory_order_relaxed)
113 | #define	atomic_load_acquire(p)	atomic_load_explicit(p, memory_order_acquire)
114 | #define	atomic_load_consume(p)	atomic_load_explicit(p, memory_order_consume)
115 | 
116 | #define	atomic_store_release(p,v) \
117 |     atomic_store_explicit(p, v, memory_order_release)
118 | #define	atomic_store_relaxed(p,v) \
119 |     atomic_store_explicit(p, v, memory_order_relaxed)
120 | 
121 | /*
122 |  * Exponential back-off for the spinning paths.
123 |  */
124 | #define	SPINLOCK_BACKOFF_MIN	4
125 | #define	SPINLOCK_BACKOFF_MAX	128
126 | #if defined(__x86_64__) || defined(__i386__)
127 | #define SPINLOCK_BACKOFF_HOOK	__asm volatile("pause" ::: "memory")
128 | #else
129 | #define SPINLOCK_BACKOFF_HOOK
130 | #endif
131 | #define	SPINLOCK_BACKOFF(count)					\
132 | do {								\
133 | 	int __i;						\
134 | 	for (__i = (count); __i != 0; __i--) {			\
135 | 		SPINLOCK_BACKOFF_HOOK;				\
136 | 	}							\
137 | 	if ((count) < SPINLOCK_BACKOFF_MAX)			\
138 | 		(count) += (count);				\
139 | } while (/* CONSTCOND */ 0);
140 | 
141 | /*
142 |  * Cache line size - a reasonable upper bound.
143 |  */
144 | #define	CACHE_LINE_SIZE		64
145 | 
146 | /*
147 |  * Hash functions.
148 |  */
149 | uint32_t	murmurhash3(const void *, size_t, uint32_t);
150 | 
151 | #endif
152 | 


--------------------------------------------------------------------------------