├── .travis.yml
├── LICENSE
├── README.md
├── pkg
    ├── Makefile
    ├── SPECS
    │   └── libqsbr.spec
    ├── debian
    │   ├── changelog
    │   ├── compat
    │   ├── control
    │   ├── libqsbr-dev.install
    │   ├── libqsbr1.install
    │   ├── rules
    │   └── source
    │   │   └── format
    └── version.txt
└── src
    ├── Makefile
    ├── ebr.c
    ├── ebr.h
    ├── gc.c
    ├── gc.h
    ├── qsbr.c
    ├── qsbr.h
    ├── t_gc.c
    ├── t_stress.c
    └── utils.h


/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: c
 2 | 
 3 | compiler:
 4 |   - gcc
 5 |   - clang
 6 | 
 7 | dist: bionic
 8 | 
 9 | matrix:
10 |   include:
11 |     - os: linux
12 |       arch: amd64
13 |     - os: linux
14 |       arch: arm64
15 |     - os: linux
16 |       arch: ppc64le
17 | 
18 | addons:
19 |   apt:
20 |     update: true
21 |     packages:
22 |     - build-essential
23 |     - fakeroot
24 |     - debhelper
25 |     - libtool
26 |     - libtool-bin
27 | 
28 | script:
29 |   # Build the package.
30 |   - (cd pkg && make deb)
31 |   # Run the unit and stress tests.
32 |   - (cd src && make tests)
33 |   - (cd src && make clean && make stress)
34 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | /*-
 2 |  * Copyright (c) 2015-2018 Mindaugas Rasiukevicius <rmind at noxt eu>
 3 |  * All rights reserved.
 4 |  *
 5 |  * Redistribution and use in source and binary forms, with or without
 6 |  * modification, are permitted provided that the following conditions
 7 |  * are met:
 8 |  * 1. Redistributions of source code must retain the above copyright
 9 |  *    notice, this list of conditions and the following disclaimer.
10 |  * 2. Redistributions in binary form must reproduce the above copyright
11 |  *    notice, this list of conditions and the following disclaimer in the
12 |  *    documentation and/or other materials provided with the distribution.
13 |  *
14 |  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 |  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 |  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 |  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 |  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 |  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 |  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 |  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 |  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 |  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 |  * SUCH DAMAGE.
25 |  */
26 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Quiescent-State and Epoch based reclamation
  2 | 
  3 | [![Build Status](https://travis-ci.org/rmind/libqsbr.svg?branch=master)](https://travis-ci.org/rmind/libqsbr)
  4 | 
  5 | Epoch-Based Reclamation (EBR) and Quiescent-State-Based Reclamation (QSBR)
  6 | are synchronisation mechanisms which can be used for efficient memory/object
  7 | reclamation (garbage collection) in concurrent environment.  Conceptually
  8 | they are very similar to the read-copy-update (RCU) mechanism.
  9 | 
 10 | EBR and QSBR are simpler, more lightweight and often faster than RCU.
 11 | However, each thread must register itself when using these mechanisms.
 12 | EBR allows user to mark the critical code paths without the need to
 13 | periodically indicate the quiescent state.  It is slightly slower than
 14 | QSBR due to the need to issue a memory barrier on the reader side.
 15 | QSBR is more lightweight, but each thread must manually indicate the
 16 | quiescent state i.e. threads must periodically pass a checkpoint where
 17 | they call a dedicated function.  In many applications, such approach
 18 | can be practical.
 19 | 
 20 | A typical use case of the EBR or QSBR would be together with lock-free
 21 | data structures.  This library provides raw EBR and QSBR mechanisms as
 22 | well as a higher level garbage collection (GC) interface based on EBR.
 23 | 
 24 | The implementation is written in C11 and distributed under the
 25 | 2-clause BSD license.
 26 | 
 27 | References:
 28 | 
 29 | 	K. Fraser, Practical lock-freedom,
 30 | 	Technical Report UCAM-CL-TR-579, February 2004
 31 | 	https://www.cl.cam.ac.uk/techreports/UCAM-CL-TR-579.pdf
 32 | 
 33 | 	T. E. Hart, P. E. McKenney, A.D. Brown,
 34 | 	Making Lockless Synchronization Fast: Performance Implications of Memory Reclamation.
 35 | 	Parallel and Distributed Processing Symposium, April 2006.
 36 | 	http://csng.cs.toronto.edu/publication_files/0000/0165/ipdps06.pdf
 37 | 
 38 | ## EBR API
 39 | 
 40 | * `ebr_t *ebr_create(void)`
 41 |   * Construct a new EBR object.
 42 | 
 43 | * `void ebr_destroy(ebr_t *ebr)`
 44 |   * Destroy the EBR object.
 45 | 
 46 | * `int ebr_register(ebr_t *ebr)`
 47 |   * Register the current thread for EBR synchronisation.  Returns 0 on
 48 |   success and -1 on failure.  Note: each reader thread (i.e. callers of
 49 |   `ebr_enter/ebr_exit`) **must** register.
 50 | 
 51 | * `void ebr_unregister(ebr_t *ebr)`
 52 |   * Remove the current thread from the EBR synchronisation list.  Each
 53 |   registered thread must leave the list before the exit (this may be not
 54 |   necessary if all threads exit together).  It is the caller's responsibility
 55 |   to synchronise the thread exit, if needed.
 56 | 
 57 | * `void ebr_enter(ebr_t *ebr)`
 58 |   * Mark the entrance to the critical path.  Typically, this would be
 59 |   used by the readers when accessing some shared data; reclamation of
 60 |   objects is guaranteed to not occur in the critical path.
 61 |   * Note: the EBR mechanism is not limited to the concept of "objects".
 62 |   It can be any form of reference to the globally shared data.
 63 | 
 64 | * `void ebr_exit(ebr_t *ebr)`
 65 |   * Mark the exit of the critical path.  Reclamation of the shared data
 66 |   may occur after this point.
 67 | 
 68 | * `bool ebr_sync(ebr_t *ebr, unsigned *gc_epoch)`
 69 |   * Attempt to synchronise and announce a new epoch.  Returns `true` if
 70 |   a new epoch is announced and `false` otherwise.  In either case, the
 71 |   _epoch_ available for reclamation is returned.  The number of epochs
 72 |   is defined by the `EBR_EPOCHS` constant and the epoch value is
 73 |   `0 <= epoch < EBR_EPOCHS`.
 74 |   * The synchronisation points must be serialised (e.g. if there are
 75 |   multiple G/C workers or other writers).  Generally, calls to
 76 |   `ebr_staging_epoch` and `ebr_gc_epoch` would be a part of the same
 77 |   serialised path.
 78 | 
 79 | * `unsigned ebr_staging_epoch(ebr_t *ebr)`
 80 |   * Returns an _epoch_ where objects can be staged for reclamation.
 81 |   This can be used as a reference value for the pending queue/tag, used
 82 |   to postpone the reclamation until this epoch becomes available for G/C.
 83 |   Note that this function would normally be serialised together with
 84 |   the `ebr_sync` calls.
 85 | 
 86 | * `unsigned ebr_gc_epoch(ebr_t *ebr)`
 87 |   * Returns the _epoch_ available for reclamation, i.e. the epoch where
 88 |   it is guaranteed that the objects are safe to be reclaimed/destroyed.
 89 |   The _epoch_ value will be the same as returned by the last successful
 90 |   `ebr_sync` call.  Note that these two functions would require the same
 91 |   form of serialisation.
 92 | 
 93 | * `void ebr_full_sync(ebr_t *ebr, unsigned msec_retry)`
 94 |   * Perform full synchronisation ensuring that all objects which are no
 95 |   longer globally visible (and potentially staged for reclamation) at the
 96 |   time of calling this routine will be safe to reclaim/destroy after this
 97 |   synchronisation routine completes and returns.  Note: the synchronisation
 98 |   may take across multiple epochs.
 99 |   * This function will block for `msec_retry` milliseconds before trying
100 |   again if there are objects which cannot be reclaimed immediately.  If
101 |   this value is zero, then it will invoke `sched_yield(2)` before retrying.
102 | 
103 | * `bool ebr_incrit_p(ebr_t *ebr)`
104 |   * Returns `true` if the current worker is in the critical path, i.e.
105 |   called `ebr_enter()`; otherwise, returns `false`.  This routine should
106 |   generally only be used for diagnostic asserts.
107 | 
108 | 
109 | ## G/C API
110 | 
111 | * `gc_t *gc_create(unsigned entry_off, gc_func_t reclaim, void *arg)`
112 |   * Construct a new G/C management object.  The `entry_off` argument is
113 |   an offset of the `gc_entry_t` structure, which must be embedded in the
114 |   object; typically, this value would be `offsetof(struct obj, gc_entry)`.
115 |   The entry structure may also be embedded at the beginning of the object
116 |   structure (offset being zero), should the caller need to support
117 |   different object types.
118 |   * A custom reclamation function can be used for object destruction.
119 |   This function must process a list of objects, since a chain of objects
120 |   may be passed for reclamation; the user can iterate the chain using
121 |   the `gc_entry_t::next` member.  If _reclaim_ is NULL, then the default
122 |   logic invoked by the G/C mechanism will be calling the system `free(3)`
123 |   for each object.  An arbitrary user pointer, specified by `arg`, can
124 |   be passed to the reclamation function.
125 | 
126 | * `void gc_destroy(gc_t *gc)`
127 |   * Destroy the G/C management object.
128 | 
129 | * `void gc_register(gc_t *gc)`
130 |   * Register the current thread as a user of the G/C mechanism.
131 |   All threads having critical paths to reference the objects must register.
132 | 
133 | * `void gc_crit_enter(gc_t *gc)`
134 |   * Enter the critical path where objects may be actively referenced.
135 |   This prevents the G/C mechanism from reclaiming (destroying) the object.
136 | 
137 | * `void gc_crit_exit(gc_t *gc)`
138 |   * Exit the critical path, indicating that the target objects no
139 |   longer have active references and the G/C mechanism may consider
140 |   them for reclamation.
141 | 
142 | * `void gc_limbo(gc_t *gc, void *obj)`
143 |   * Insert the object into a "limbo" list, staging it for reclamation
144 |   (destruction).  This is a request to reclaim the object once it is
145 |   guaranteed that there are no threads referencing it in the critical path.
146 | 
147 | * `void gc_cycle(gc_t *gc)`
148 |   * Run a G/C cycle attempting to reclaim some objects which were
149 |   added to the limbo list.  The objects which are no longer referenced
150 |   are not guaranteed to be reclaimed immediately after one cycle.  This
151 |   function does not block and is expected to be called periodically for
152 |   an incremental object reclamation.
153 | 
154 | * `void gc_full(gc_t *gc, unsigned msec_retry)`
155 |   * Run a full G/C in order to ensure that all staged objects have been
156 |   reclaimed.  This function will block for `msec_retry` milliseconds before
157 |   trying again, if there are objects which cannot be reclaimed immediately.
158 | 
159 | ## Notes
160 | 
161 | The implementation was extensively tested on a 24-core x86 machine,
162 | see [the stress test](src/t_stress.c) for the details on the technique.
163 | 
164 | ## Examples
165 | 
166 | ### G/C API example
167 | 
168 | The G/C mechanism should be created by some master thread.
169 | ```c
170 | typedef struct {
171 | 	...
172 | 	gc_entry_t	gc_entry;
173 | } obj_t;
174 | 
175 | static gc_t *	gc;
176 | 
177 | void
178 | some_sysinit(void)
179 | {
180 | 	gc = gc_create(offsetof(obj_t, gc_entry), NULL, NULL);
181 | 	assert(gc != NULL);
182 | 	...
183 | }
184 | ```
185 | 
186 | An example code fragment of a reader thread:
187 | ```c
188 | 	gc_register(gc);
189 | 
190 | 	while (!exit) {
191 | 		/*
192 | 		 * Some processing which references the object(s).
193 | 		 * The readers must indicate the critical path where
194 | 		 * they actively reference objects.
195 | 		 */
196 | 		gc_crit_enter(gc);
197 | 		obj = object_lookup();
198 | 		process_object(obj);
199 | 		gc_crit_exit(gc);
200 | 	}
201 | ```
202 | 
203 | Here is an example code fragment in a writer thread which illustrates
204 | how the object would be staged for destruction (reclamation):
205 | ```c
206 | 	/*
207 | 	 * Remove the object from the lock-free container.  The
208 | 	 * object is no longer globally visible.  Not it can be
209 | 	 * staged for destruction -- add it to the limbo list.
210 | 	 */
211 | 	obj = lockfree_remove(container, key);
212 | 	gc_limbo(gc, obj);
213 | 	...
214 | 
215 | 	/*
216 | 	 * Checkpoint: run a G/C cycle attempting to reclaim *some*
217 | 	 * objects previously added to the limbo list.  This should be
218 | 	 * called periodically for incremental object reclamation.
219 | 	 *
220 | 	 * WARNING: All gc_cycle() calls must be serialised (using a
221 | 	 * mutex or by running in a single-threaded manner).
222 | 	 */
223 | 	gc_cycle(gc);
224 | 	...
225 | 
226 | 	/*
227 | 	 * Eventually, a full G/C might have to be performed to ensure
228 | 	 * that all objects have been reclaimed.  This call can block.
229 | 	 */
230 | 	gc_full(gc, 1); // sleep for 1 msec before re-checking
231 | ```
232 | 
233 | ## Packages
234 | 
235 | Just build the package, install it and link the library using the
236 | `-lqsbr` flag.
237 | * RPM (tested on RHEL/CentOS 7): `cd pkg && make rpm`
238 | * DEB (tested on Debian 9): `cd pkg && make deb`
239 | 


--------------------------------------------------------------------------------
/pkg/Makefile:
--------------------------------------------------------------------------------
 1 | PROJ=	libqsbr
 2 | 
 3 | all:
 4 | 	@ echo "targets"
 5 | 	@ echo "	make rpm"
 6 | 	@ echo "	make deb"
 7 | 
 8 | rpm:
 9 | 	mkdir -p SOURCES && tar czpvf SOURCES/$(PROJ).tar.gz ../src
10 | 	rpmbuild -ba -v --define "_topdir ${PWD}" SPECS/$(PROJ).spec
11 | 	@ echo && printf "\x1B[32mRPM packages:\033[0m\n" && ls -1 RPMS/*
12 | 
13 | deb:
14 | 	cp -R ../src ./SOURCES
15 | 	dpkg-buildpackage -rfakeroot -us -uc -b
16 | 	@ echo && printf "\x1B[32mDEB packages:\033[0m\n" && ls -1 ../*.deb
17 | 
18 | clean:
19 | 	rm -rf BUILD BUILDROOT RPMS SOURCES SRPMS
20 | 
21 | .PHONY: all rpm deb clean
22 | 


--------------------------------------------------------------------------------
/pkg/SPECS/libqsbr.spec:
--------------------------------------------------------------------------------
 1 | %define version	%(cat %{_topdir}/version.txt)
 2 | 
 3 | Name:		libqsbr
 4 | Version:	%{version}
 5 | Release:	1%{?dist}
 6 | Summary:	EBR and QSBR based reclamation library
 7 | Group:		System Environment/Libraries
 8 | License:	BSD
 9 | URL:		https://github.com/rmind/libqsbr
10 | Source0:	libqsbr.tar.gz
11 | 
12 | BuildRequires:	make
13 | BuildRequires:	libtool
14 | 
15 | %description
16 | Epoch-Based Reclamation (EBR) and Quiescent-State-Based Reclamation (QSBR)
17 | are synchronisation mechanisms which can be used for efficient memory/object
18 | reclamation (garbage collection) in concurrent environment.  Conceptually
19 | they are very similar to the read-copy-update (RCU) mechanism.
20 | 
21 | %prep
22 | %setup -q -n src
23 | 
24 | %build
25 | make %{?_smp_mflags} LIBDIR=%{_libdir}
26 | 
27 | %install
28 | make install \
29 |     DESTDIR=%{buildroot} \
30 |     LIBDIR=%{_libdir} \
31 |     INCDIR=%{_includedir} \
32 |     MANDIR=%{_mandir}
33 | 
34 | %files
35 | %{_libdir}/*
36 | %{_includedir}/*
37 | #%{_mandir}/*
38 | 
39 | %changelog
40 | 


--------------------------------------------------------------------------------
/pkg/debian/changelog:
--------------------------------------------------------------------------------
1 | qsbr (0.0.1) unstable; urgency=medium
2 | 
3 |   * Initial release.
4 | 
5 |  -- rmind <rmind@noxt.eu>  Fri, 18 May 2018 23:34:33 +0100
6 | 


--------------------------------------------------------------------------------
/pkg/debian/compat:
--------------------------------------------------------------------------------
1 | 10
2 | 


--------------------------------------------------------------------------------
/pkg/debian/control:
--------------------------------------------------------------------------------
 1 | Source: qsbr
 2 | Priority: extra
 3 | Maintainer: https://github.com/rmind
 4 | Build-Depends:
 5 |  debhelper (>= 9),
 6 |  make,
 7 |  libtool
 8 | Standards-Version: 3.9.1
 9 | Homepage: https://github.com/rmind/qsbr
10 | License: BSD-2-clause
11 | 
12 | Package: libqsbr1
13 | Section: lib
14 | Architecture: any
15 | Depends: ${shlibs:Depends}, ${misc:Depends}
16 | Description: EBR and QSBR based reclamation library
17 |  Epoch-Based Reclamation (EBR) and Quiescent-State-Based Reclamation (QSBR)
18 |  are synchronisation mechanisms which can be used for efficient memory/object
19 |  reclamation (garbage collection) in concurrent environment.  Conceptually
20 |  they are very similar to the read-copy-update (RCU) mechanism.
21 | 
22 | Package: libqsbr1-dbg
23 | Section: debug
24 | Architecture: any
25 | Depends: ${misc:Depends}, libqsbr1 (= ${binary:Version})
26 | Description: Debug symbols for libqsbr1
27 |  Debug symbols for libqsbr1.
28 | 
29 | Package: libqsbr-dev
30 | Section: libdevel
31 | Architecture: any
32 | Depends: ${shlibs:Depends}, ${misc:Depends}, libqsbr1 (= ${binary:Version})
33 | Description: Development files for libqsbr1
34 |  Development files for libqsbr1.
35 | 


--------------------------------------------------------------------------------
/pkg/debian/libqsbr-dev.install:
--------------------------------------------------------------------------------
1 | usr/include/*
2 | usr/lib/*/lib*.a
3 | usr/lib/*/lib*.so
4 | 


--------------------------------------------------------------------------------
/pkg/debian/libqsbr1.install:
--------------------------------------------------------------------------------
1 | usr/lib/*/lib*.so.*
2 | 


--------------------------------------------------------------------------------
/pkg/debian/rules:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/make -f
 2 | 
 3 | #export DH_VERBOSE = 1
 4 | 
 5 | PKGVERSION:=$(shell cat version.txt)
 6 | DEB_HOST_MULTIARCH?=$(shell dpkg-architecture -qDEB_HOST_MULTIARCH)
 7 | LIBDIR:=/usr/lib/$(DEB_HOST_MULTIARCH)
 8 | INCDIR:=/usr/include
 9 | 
10 | %:
11 | 	dh $@ --sourcedirectory=SOURCES --parallel
12 | 
13 | override_dh_auto_test:
14 | 	dh_auto_test tests
15 | 
16 | override_dh_auto_install:
17 | 	dh_auto_install -- LIBDIR=$(LIBDIR) INCDIR=$(INCDIR)
18 | 
19 | override_dh_strip:
20 | 	dh_strip -p libqsbr1 --dbg-package=libqsbr1-dbg
21 | 	dh_strip -a --remaining-packages
22 | 
23 | override_dh_gencontrol:
24 | 	dh_gencontrol -- -v$(PKGVERSION)
25 | 


--------------------------------------------------------------------------------
/pkg/debian/source/format:
--------------------------------------------------------------------------------
1 | 3.0 (quilt)
2 | 


--------------------------------------------------------------------------------
/pkg/version.txt:
--------------------------------------------------------------------------------
1 | 1.0.0
2 | 


--------------------------------------------------------------------------------
/src/Makefile:
--------------------------------------------------------------------------------
 1 | #
 2 | # This file is in the Public Domain.
 3 | #
 4 | 
 5 | PROJ=		qsbr
 6 | 
 7 | SYSNAME:=	$(shell uname -s)
 8 | SYSARCH:=	$(shell uname -m)
 9 | 
10 | CFLAGS+=	-std=c11 -O2 -g -Wall -Wextra -Werror
11 | ifneq ($(SYSNAME), FreeBSD)
12 | CFLAGS+=	-D_POSIX_C_SOURCE=200809L
13 | endif
14 | CFLAGS+=	-D_GNU_SOURCE -D_DEFAULT_SOURCE
15 | 
16 | #
17 | # Extended warning flags.
18 | #
19 | CFLAGS+=	-Wno-unknown-warning-option # gcc vs clang
20 | 
21 | CFLAGS+=	-Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith
22 | CFLAGS+=	-Wmissing-declarations -Wredundant-decls -Wnested-externs
23 | CFLAGS+=	-Wshadow -Wcast-qual -Wcast-align -Wwrite-strings
24 | CFLAGS+=	-Wold-style-definition
25 | CFLAGS+=	-Wsuggest-attribute=noreturn -Wjump-misses-init
26 | CFLAGS+=	-Wduplicated-cond -Wmisleading-indentation -Wnull-dereference
27 | CFLAGS+=	-Wduplicated-branches -Wrestrict
28 | 
29 | ifeq ($(MAKECMDGOALS),tests)
30 | DEBUG=		1
31 | endif
32 | 
33 | ifeq ($(DEBUG),1)
34 | CFLAGS+=	-Og -DDEBUG -fno-omit-frame-pointer
35 | ifeq ($(SYSARCH),x86_64)
36 | CFLAGS+=	-fsanitize=address -fsanitize=undefined
37 | LDFLAGS+=	-fsanitize=address -fsanitize=undefined
38 | endif
39 | else
40 | CFLAGS+=	-DNDEBUG
41 | endif
42 | 
43 | LIB=		lib$(PROJ)
44 | INCS=		ebr.h qsbr.h gc.h
45 | 
46 | OBJS=		ebr.o qsbr.o gc.o
47 | 
48 | $(LIB).la:	LDFLAGS+=	-rpath $(LIBDIR) -version-info 1:0:0
49 | install/%.la:	ILIBDIR=	$(DESTDIR)/$(LIBDIR)
50 | install:	IINCDIR=	$(DESTDIR)/$(INCDIR)/qsbr
51 | #install:	IMANDIR=	$(DESTDIR)/$(MANDIR)/man3/
52 | 
53 | obj: $(OBJS)
54 | 
55 | lib: $(LIB).la
56 | 
57 | %.lo: %.c
58 | 	libtool --mode=compile --tag CC $(CC) $(CFLAGS) -c $<
59 | 
60 | $(LIB).la: $(shell echo $(OBJS) | sed 's/\.o/\.lo/g')
61 | 	libtool --mode=link --tag CC $(CC) $(LDFLAGS) -o $@ $(notdir $^)
62 | 
63 | install/%.la: %.la
64 | 	mkdir -p $(ILIBDIR)
65 | 	libtool --mode=install install -c $(notdir $@) $(ILIBDIR)/$(notdir $@)
66 | 
67 | install: $(addprefix install/,$(LIB).la)
68 | 	libtool --mode=finish $(LIBDIR)
69 | 	mkdir -p $(IINCDIR) && install -c $(INCS) $(IINCDIR)
70 | 	#mkdir -p $(IMANDIR) && install -c $(MANS) $(IMANDIR)
71 | 
72 | tests: $(OBJS) t_gc.o
73 | 	$(CC) $(CFLAGS) $^ -o t_gc -lpthread
74 | 	./t_gc
75 | 
76 | stress: $(OBJS) t_stress.o
77 | 	$(CC) $(CFLAGS) $^ -o t_stress $(LDFLAGS) -lpthread
78 | 	./t_stress
79 | 
80 | clean:
81 | 	libtool --mode=clean rm
82 | 	@ rm -rf .libs *.o *.lo *.la t_gc t_stress
83 | 
84 | .PHONY: all obj lib install tests stress clean
85 | 


--------------------------------------------------------------------------------
/src/ebr.c:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2016-2018 Mindaugas Rasiukevicius <rmind at noxt eu>
  3 |  * All rights reserved.
  4 |  *
  5 |  * Use is subject to license terms, as specified in the LICENSE file.
  6 |  */
  7 | 
  8 | /*
  9 |  * Epoch-based reclamation (EBR).  Reference:
 10 |  *
 11 |  *	K. Fraser, Practical lock-freedom,
 12 |  *	Technical Report UCAM-CL-TR-579, February 2004
 13 |  *	https://www.cl.cam.ac.uk/techreports/UCAM-CL-TR-579.pdf
 14 |  *
 15 |  * Summary:
 16 |  *
 17 |  * Any workers (threads or processes) actively referencing (accessing)
 18 |  * the globally visible objects must do that in the critical path covered
 19 |  * using the dedicated enter/exit functions.  The grace period is
 20 |  * determined using "epochs" -- implemented as a global counter (and,
 21 |  * for example, a dedicated G/C list for each epoch).  Objects in the
 22 |  * current global epoch can be staged for reclamation (garbage collection).
 23 |  * Then, the objects in the target epoch can be reclaimed after two
 24 |  * successful increments of the global epoch.  Only three epochs are
 25 |  * needed (e, e-1 and e-2), therefore we use clock arithmetics.
 26 |  *
 27 |  * See the comments in the ebr_sync() function for detailed explanation.
 28 |  */
 29 | 
 30 | #include <sys/queue.h>
 31 | #include <stdlib.h>
 32 | #include <stdbool.h>
 33 | #include <string.h>
 34 | #include <time.h>
 35 | #include <errno.h>
 36 | #include <pthread.h>
 37 | #include <sched.h>
 38 | 
 39 | #include "ebr.h"
 40 | #include "utils.h"
 41 | 
 42 | #define	ACTIVE_FLAG		(0x80000000U)
 43 | 
 44 | typedef struct ebr_tls {
 45 | 	/*
 46 | 	 * - A local epoch counter for each thread.
 47 | 	 * - The epoch counter may have the "active" flag set.
 48 | 	 * - Thread list entry (pointer).
 49 | 	 */
 50 | 	unsigned		local_epoch;
 51 | 	LIST_ENTRY(ebr_tls)	entry;
 52 | } ebr_tls_t;
 53 | 
 54 | struct ebr {
 55 | 	/*
 56 | 	 * - There is a global epoch counter which can be 0, 1 or 2.
 57 | 	 * - TLS with a list of the registered threads.
 58 | 	 */
 59 | 	unsigned		global_epoch;
 60 | 	pthread_key_t		tls_key;
 61 | 	pthread_mutex_t		lock;
 62 | 	LIST_HEAD(, ebr_tls)	list;
 63 | };
 64 | 
 65 | ebr_t *
 66 | ebr_create(void)
 67 | {
 68 | 	ebr_t *ebr;
 69 | 	int ret;
 70 | 
 71 | 	ret = posix_memalign((void **)&ebr, CACHE_LINE_SIZE, sizeof(ebr_t));
 72 | 	if (ret != 0) {
 73 | 		errno = ret;
 74 | 		return NULL;
 75 | 	}
 76 | 	memset(ebr, 0, sizeof(ebr_t));
 77 | 	if (pthread_key_create(&ebr->tls_key, free) != 0) {
 78 | 		free(ebr);
 79 | 		return NULL;
 80 | 	}
 81 | 	pthread_mutex_init(&ebr->lock, NULL);
 82 | 	return ebr;
 83 | }
 84 | 
 85 | void
 86 | ebr_destroy(ebr_t *ebr)
 87 | {
 88 | 	pthread_key_delete(ebr->tls_key);
 89 | 	pthread_mutex_destroy(&ebr->lock);
 90 | 	free(ebr);
 91 | }
 92 | 
 93 | /*
 94 |  * ebr_register: register the current worker (thread/process) for EBR.
 95 |  *
 96 |  * => Returns 0 on success and errno on failure.
 97 |  */
 98 | int
 99 | ebr_register(ebr_t *ebr)
100 | {
101 | 	ebr_tls_t *t;
102 | 
103 | 	t = pthread_getspecific(ebr->tls_key);
104 | 	if (__predict_false(t == NULL)) {
105 | 		int ret;
106 | 
107 | 		ret = posix_memalign((void **)&t,
108 | 		    CACHE_LINE_SIZE, sizeof(ebr_tls_t));
109 | 		if (ret != 0) {
110 | 			errno = ret;
111 | 			return -1;
112 | 		}
113 | 		pthread_setspecific(ebr->tls_key, t);
114 | 	}
115 | 	memset(t, 0, sizeof(ebr_tls_t));
116 | 
117 | 	pthread_mutex_lock(&ebr->lock);
118 | 	LIST_INSERT_HEAD(&ebr->list, t, entry);
119 | 	pthread_mutex_unlock(&ebr->lock);
120 | 	return 0;
121 | }
122 | 
123 | void
124 | ebr_unregister(ebr_t *ebr)
125 | {
126 | 	ebr_tls_t *t;
127 | 
128 | 	t = pthread_getspecific(ebr->tls_key);
129 | 	if (t == NULL) {
130 | 		return;
131 | 	}
132 | 	pthread_setspecific(ebr->tls_key, NULL);
133 | 
134 | 	pthread_mutex_lock(&ebr->lock);
135 | 	LIST_REMOVE(t, entry);
136 | 	pthread_mutex_unlock(&ebr->lock);
137 | 	free(t);
138 | }
139 | 
140 | /*
141 |  * ebr_enter: mark the entrance to the critical path.
142 |  */
143 | void
144 | ebr_enter(ebr_t *ebr)
145 | {
146 | 	ebr_tls_t *t;
147 | 	unsigned epoch;
148 | 
149 | 	t = pthread_getspecific(ebr->tls_key);
150 | 	ASSERT(t != NULL);
151 | 
152 | 	/*
153 | 	 * Set the "active" flag and set the local epoch to global
154 | 	 * epoch (i.e. observe the global epoch).  Ensure that the
155 | 	 * epoch is observed before any loads in the critical path.
156 | 	 */
157 | 	epoch = ebr->global_epoch | ACTIVE_FLAG;
158 | 	atomic_store_explicit(&t->local_epoch, epoch, memory_order_relaxed);
159 | 	atomic_thread_fence(memory_order_seq_cst);
160 | }
161 | 
162 | /*
163 |  * ebr_exit: mark the exit of the critical path.
164 |  */
165 | void
166 | ebr_exit(ebr_t *ebr)
167 | {
168 | 	ebr_tls_t *t;
169 | 
170 | 	t = pthread_getspecific(ebr->tls_key);
171 | 	ASSERT(t != NULL);
172 | 
173 | 	/*
174 | 	 * Clear the "active" flag.  Must ensure that any stores in
175 | 	 * the critical path reach global visibility before that.
176 | 	 */
177 | 	ASSERT(t->local_epoch & ACTIVE_FLAG);
178 | 	atomic_thread_fence(memory_order_seq_cst);
179 | 	atomic_store_explicit(&t->local_epoch, 0, memory_order_relaxed);
180 | }
181 | 
182 | /*
183 |  * ebr_sync: attempt to synchronise and announce a new epoch.
184 |  *
185 |  * => Synchronisation points must be serialised.
186 |  * => Return true if a new epoch was announced.
187 |  * => Return the epoch ready for reclamation.
188 |  */
189 | bool
190 | ebr_sync(ebr_t *ebr, unsigned *gc_epoch)
191 | {
192 | 	unsigned epoch;
193 | 	ebr_tls_t *t;
194 | 
195 | 	/*
196 | 	 * Ensure that any loads or stores on the writer side reach
197 | 	 * the global visibility.  We want to allow the callers to
198 | 	 * assume that the ebr_sync() call serves as a full barrier.
199 | 	 */
200 | 	epoch = atomic_load_explicit(&ebr->global_epoch, memory_order_relaxed);
201 | 	atomic_thread_fence(memory_order_seq_cst);
202 | 
203 | 	/*
204 | 	 * Check whether all active workers observed the global epoch.
205 | 	 */
206 | 	LIST_FOREACH(t, &ebr->list, entry) {
207 | 		unsigned local_epoch;
208 | 		bool active;
209 | 
210 | 		local_epoch = atomic_load_explicit(&t->local_epoch,
211 | 		    memory_order_relaxed);
212 | 		active = (local_epoch & ACTIVE_FLAG) != 0;
213 | 
214 | 		if (active && (local_epoch != (epoch | ACTIVE_FLAG))) {
215 | 			/* No, not ready. */
216 | 			*gc_epoch = ebr_gc_epoch(ebr);
217 | 			return false;
218 | 		}
219 | 	}
220 | 
221 | 	/* Yes: increment and announce a new global epoch. */
222 | 	atomic_store_explicit(&ebr->global_epoch,
223 | 	    (epoch + 1) % 3, memory_order_relaxed);
224 | 
225 | 	/*
226 | 	 * Let the new global epoch be 'e'.  At this point:
227 | 	 *
228 | 	 * => Active workers: might still be running in the critical path
229 | 	 *    in the e-1 epoch or might be already entering a new critical
230 | 	 *    path and observing the new epoch e.
231 | 	 *
232 | 	 * => Inactive workers: might become active by entering a critical
233 | 	 *    path before or after the global epoch counter was incremented,
234 | 	 *    observing either e-1 or e.
235 | 	 *
236 | 	 * => Note that the active workers cannot have a stale observation
237 | 	 *    of the e-2 epoch at this point (there is no ABA problem using
238 | 	 *    the clock arithmetics).
239 | 	 *
240 | 	 * => Therefore, there can be no workers still running the critical
241 | 	 *    path in the e-2 epoch.  This is the epoch ready for G/C.
242 | 	 */
243 | 	*gc_epoch = ebr_gc_epoch(ebr);
244 | 	return true;
245 | }
246 | 
247 | /*
248 |  * ebr_staging_epoch: return the epoch where objects can be staged
249 |  * for reclamation.
250 |  */
251 | unsigned
252 | ebr_staging_epoch(ebr_t *ebr)
253 | {
254 | 	/* The current epoch. */
255 | 	return ebr->global_epoch;
256 | }
257 | 
258 | /*
259 |  * ebr_gc_epoch: return the epoch where objects are ready to be
260 |  * reclaimed i.e. it is guaranteed to be safe to destroy them.
261 |  */
262 | unsigned
263 | ebr_gc_epoch(ebr_t *ebr)
264 | {
265 | 	/*
266 | 	 * Since we use only 3 epochs, e-2 is just the next global
267 | 	 * epoch with clock arithmetics.
268 | 	 */
269 | 	return (ebr->global_epoch + 1) % 3;
270 | }
271 | 
272 | void
273 | ebr_full_sync(ebr_t *ebr, unsigned msec_retry)
274 | {
275 | 	const struct timespec dtime = { 0, msec_retry * 1000 * 1000 };
276 | 	const unsigned target_epoch = ebr_staging_epoch(ebr);
277 | 	unsigned epoch, count = SPINLOCK_BACKOFF_MIN;
278 | wait:
279 | 	while (!ebr_sync(ebr, &epoch)) {
280 | 		if (count < SPINLOCK_BACKOFF_MAX) {
281 | 			SPINLOCK_BACKOFF(count);
282 | 		} else if (msec_retry) {
283 | 			(void)nanosleep(&dtime, NULL);
284 | 		} else {
285 | 			sched_yield();
286 | 		}
287 | 	}
288 | 	if (target_epoch != epoch) {
289 | 		goto wait;
290 | 	}
291 | }
292 | 
293 | /*
294 |  * ebr_incrit_p: return true if the current worker is in the critical path,
295 |  * i.e. called ebr_enter(); otherwise, return false.
296 |  *
297 |  * Note: this routine should generally only be used for diagnostic asserts.
298 |  */
299 | bool
300 | ebr_incrit_p(ebr_t *ebr)
301 | {
302 | 	ebr_tls_t *t;
303 | 
304 | 	t = pthread_getspecific(ebr->tls_key);
305 | 	ASSERT(t != NULL);
306 | 
307 | 	return (t->local_epoch & ACTIVE_FLAG) != 0;
308 | }
309 | 


--------------------------------------------------------------------------------
/src/ebr.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (c) 2016 Mindaugas Rasiukevicius <rmind at noxt eu>
 3 |  * All rights reserved.
 4 |  *
 5 |  * Use is subject to license terms, as specified in the LICENSE file.
 6 |  */
 7 | 
 8 | #ifndef	_EBR_H_
 9 | #define	_EBR_H_
10 | 
11 | __BEGIN_DECLS
12 | 
13 | struct ebr;
14 | typedef struct ebr ebr_t;
15 | 
16 | #define	EBR_EPOCHS	3
17 | 
18 | ebr_t *		ebr_create(void);
19 | void		ebr_destroy(ebr_t *);
20 | int		ebr_register(ebr_t *);
21 | void		ebr_unregister(ebr_t *);
22 | 
23 | void		ebr_enter(ebr_t *);
24 | void		ebr_exit(ebr_t *);
25 | bool		ebr_sync(ebr_t *, unsigned *);
26 | unsigned	ebr_staging_epoch(ebr_t *);
27 | unsigned	ebr_gc_epoch(ebr_t *);
28 | void		ebr_full_sync(ebr_t *, unsigned);
29 | bool		ebr_incrit_p(ebr_t *);
30 | 
31 | __END_DECLS
32 | 
33 | #endif
34 | 


--------------------------------------------------------------------------------
/src/gc.c:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2015-2018 Mindaugas Rasiukevicius <rmind at noxt eu>
  3 |  * All rights reserved.
  4 |  *
  5 |  * Use is subject to license terms, as specified in the LICENSE file.
  6 |  */
  7 | 
  8 | /*
  9 |  * Garbage Collection (G/C) interface for multi-threaded environment,
 10 |  * using the Epoch-based reclamation (EBR) mechanism.
 11 |  */
 12 | 
 13 | #include <stdio.h>
 14 | #include <stdlib.h>
 15 | #include <stdbool.h>
 16 | #include <inttypes.h>
 17 | #include <time.h>
 18 | 
 19 | #include "gc.h"
 20 | #include "ebr.h"
 21 | #include "utils.h"
 22 | 
 23 | struct gc {
 24 | 	/*
 25 | 	 * Objects are first inserted into the limbo list.  They move
 26 | 	 * to a current epoch list on a G/C cycle.
 27 | 	 */
 28 | 	gc_entry_t *	limbo;
 29 | 
 30 | 	/*
 31 | 	 * A separate list for each epoch.  Objects in each list
 32 | 	 * are reclaimed incrementally, as ebr_sync() announces new
 33 | 	 * epochs ready to be reclaimed.
 34 | 	 */
 35 | 	gc_entry_t *	epoch_list[EBR_EPOCHS];
 36 | 
 37 | 	/*
 38 | 	 * EBR object and the reclamation function.
 39 | 	 */
 40 | 	ebr_t *		ebr;
 41 | 	unsigned	entry_off;
 42 | 	gc_func_t	reclaim;
 43 | 	void *		arg;
 44 | };
 45 | 
 46 | static void
 47 | gc_default_reclaim(gc_entry_t *entry, void *arg)
 48 | {
 49 | 	gc_t *gc = arg;
 50 | 	const unsigned off = gc->entry_off;
 51 | 	void *obj;
 52 | 
 53 | 	while (entry) {
 54 | 		obj = (void *)((uintptr_t)entry - off);
 55 | 		entry = entry->next;
 56 | 		free(obj);
 57 | 	}
 58 | }
 59 | 
 60 | gc_t *
 61 | gc_create(unsigned off, gc_func_t reclaim, void *arg)
 62 | {
 63 | 	gc_t *gc;
 64 | 
 65 | 	if ((gc = calloc(1, sizeof(gc_t))) == NULL) {
 66 | 		return NULL;
 67 | 	}
 68 | 	gc->ebr = ebr_create();
 69 | 	if (!gc->ebr) {
 70 | 		free(gc);
 71 | 		return NULL;
 72 | 	}
 73 | 	gc->entry_off = off;
 74 | 	if (reclaim) {
 75 | 		gc->reclaim = reclaim;
 76 | 		gc->arg = arg;
 77 | 	} else {
 78 | 		gc->reclaim = gc_default_reclaim;
 79 | 		gc->arg = gc;
 80 | 	}
 81 | 	return gc;
 82 | }
 83 | 
 84 | void
 85 | gc_destroy(gc_t *gc)
 86 | {
 87 | 	for (unsigned i = 0; i < EBR_EPOCHS; i++) {
 88 | 		ASSERT(gc->epoch_list[i] == NULL);
 89 | 	}
 90 | 	ASSERT(gc->limbo == NULL);
 91 | 
 92 | 	ebr_destroy(gc->ebr);
 93 | 	free(gc);
 94 | }
 95 | 
 96 | void
 97 | gc_register(gc_t *gc)
 98 | {
 99 | 	ebr_register(gc->ebr);
100 | }
101 | 
102 | void
103 | gc_unregister(gc_t *gc)
104 | {
105 | 	ebr_unregister(gc->ebr);
106 | }
107 | 
108 | void
109 | gc_crit_enter(gc_t *gc)
110 | {
111 | 	ebr_enter(gc->ebr);
112 | }
113 | 
114 | void
115 | gc_crit_exit(gc_t *gc)
116 | {
117 | 	ebr_exit(gc->ebr);
118 | }
119 | 
120 | /*
121 |  * gc_limbo: insert into the limbo list.
122 |  */
123 | void
124 | gc_limbo(gc_t *gc, void *obj)
125 | {
126 | 	gc_entry_t *ent = (void *)((uintptr_t)obj + gc->entry_off);
127 | 	gc_entry_t *head;
128 | 
129 | 	do {
130 | 		head = gc->limbo;
131 | 		ent->next = head;
132 | 	} while (!atomic_compare_exchange_weak(&gc->limbo, head, ent));
133 | }
134 | 
135 | void
136 | gc_cycle(gc_t *gc)
137 | {
138 | 	unsigned count = EBR_EPOCHS, gc_epoch, staging_epoch;
139 | 	ebr_t *ebr = gc->ebr;
140 | 	gc_entry_t *gc_list;
141 | next:
142 | 	/*
143 | 	 * Call the EBR synchronisation and check whether it announces
144 | 	 * a new epoch.
145 | 	 */
146 | 	if (!ebr_sync(ebr, &gc_epoch)) {
147 | 		/* Not announced -- not ready to reclaim. */
148 | 		return;
149 | 	}
150 | 
151 | 	/*
152 | 	 * Move the objects from the limbo list into the staging epoch.
153 | 	 */
154 | 	staging_epoch = ebr_staging_epoch(ebr);
155 | 	ASSERT(gc->epoch_list[staging_epoch] == NULL);
156 | 	gc->epoch_list[staging_epoch] = atomic_exchange(&gc->limbo, NULL);
157 | 
158 | 	/*
159 | 	 * Reclaim the objects in the G/C epoch list.
160 | 	 */
161 | 	gc_list = gc->epoch_list[gc_epoch];
162 | 	if (!gc_list && count--) {
163 | 		/*
164 | 		 * If there is nothing to G/C -- try a next epoch,
165 | 		 * but loop only for one "full" cycle.
166 | 		 */
167 | 		goto next;
168 | 	}
169 | 	gc->reclaim(gc_list, gc->arg);
170 | 	gc->epoch_list[gc_epoch] = NULL;
171 | }
172 | 
173 | void
174 | gc_full(gc_t *gc, unsigned msec_retry)
175 | {
176 | 	const struct timespec dtime = { 0, msec_retry * 1000 * 1000 };
177 | 	unsigned count = SPINLOCK_BACKOFF_MIN;
178 | 	bool done;
179 | again:
180 | 	/*
181 | 	 * Run a G/C cycle.
182 | 	 */
183 | 	gc_cycle(gc);
184 | 
185 | 	/*
186 | 	 * Check all epochs and the limbo list.
187 | 	 */
188 | 	done = true;
189 | 	for (unsigned i = 0; i < EBR_EPOCHS; i++) {
190 | 		if (gc->epoch_list[i]) {
191 | 			done = false;
192 | 			break;
193 | 		}
194 | 	}
195 | 	if (!done || gc->limbo) {
196 | 		/*
197 | 		 * There are objects waiting for reclaim.  Spin-wait or
198 | 		 * sleep for a little bit and try to reclaim them.
199 | 		 */
200 | 		if (count < SPINLOCK_BACKOFF_MAX) {
201 | 			SPINLOCK_BACKOFF(count);
202 | 		} else {
203 | 			(void)nanosleep(&dtime, NULL);
204 | 		}
205 | 		goto again;
206 | 	}
207 | }
208 | 


--------------------------------------------------------------------------------
/src/gc.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (c) 2015 Mindaugas Rasiukevicius <rmind at noxt eu>
 3 |  * All rights reserved.
 4 |  *
 5 |  * Use is subject to license terms, as specified in the LICENSE file.
 6 |  */
 7 | 
 8 | #ifndef _GC_H_
 9 | #define _GC_H_
10 | 
11 | #include <sys/cdefs.h>
12 | 
13 | typedef struct gc gc_t;
14 | 
15 | typedef struct gc_entry {
16 | 	struct gc_entry *next;
17 | } gc_entry_t;
18 | 
19 | typedef void (*gc_func_t)(gc_entry_t *, void *);
20 | 
21 | __BEGIN_DECLS
22 | 
23 | gc_t *	gc_create(unsigned, gc_func_t, void *);
24 | void	gc_destroy(gc_t *);
25 | void	gc_register(gc_t *);
26 | void	gc_unregister(gc_t *);
27 | 
28 | void	gc_crit_enter(gc_t *);
29 | void	gc_crit_exit(gc_t *);
30 | 
31 | void	gc_limbo(gc_t *, void *);
32 | void	gc_cycle(gc_t *);
33 | void	gc_full(gc_t *, unsigned);
34 | 
35 | __END_DECLS
36 | 
37 | #endif
38 | 


--------------------------------------------------------------------------------
/src/qsbr.c:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2015 Mindaugas Rasiukevicius <rmind at noxt eu>
  3 |  * All rights reserved.
  4 |  *
  5 |  * Use is subject to license terms, as specified in the LICENSE file.
  6 |  */
  7 | 
  8 | /*
  9 |  * Quiescent state based reclamation (QSBR).
 10 |  *
 11 |  * Notes on the usage:
 12 |  *
 13 |  * Each registered thread has to periodically indicate that it is in a
 14 |  * quiescent i.e. the state when it does not hold any memory references
 15 |  * to the objects which may be garbage collected.  A typical use of the
 16 |  * qsbr_checkpoint() function would be e.g. after processing a single
 17 |  * request when any shared state is no longer referenced.  The higher
 18 |  * the period, the higher the reclamation granularity.
 19 |  *
 20 |  * Writers i.e. threads which are trying to garbage collect the object
 21 |  * should ensure that the objects are no longer globally visible and
 22 |  * then issue a barrier using qsbr_barrier() function.  This function
 23 |  * returns a generation number.  It is safe to reclaim the said objects
 24 |  * when qsbr_sync() returns true on a given number.
 25 |  *
 26 |  * Note that this interface is asynchronous.
 27 |  */
 28 | 
 29 | #include <sys/queue.h>
 30 | #include <stdlib.h>
 31 | #include <string.h>
 32 | #include <errno.h>
 33 | #include <pthread.h>
 34 | 
 35 | #include "qsbr.h"
 36 | #include "utils.h"
 37 | 
 38 | /*
 39 |  * FIXME: handle the epoch overflow on 32-bit systems; not a problem
 40 |  * on 64-bit systems.
 41 |  */
 42 | static_assert(sizeof(qsbr_epoch_t) == 8, "expected 64-bit counter");
 43 | 
 44 | typedef struct qsbr_tls {
 45 | 	/*
 46 | 	 * The thread (local) epoch, observed at qsbr_checkpoint().
 47 | 	 * Also, a pointer to the TLS structure of a next thread.
 48 | 	 */
 49 | 	qsbr_epoch_t		local_epoch;
 50 | 	LIST_ENTRY(qsbr_tls)	entry;
 51 | } qsbr_tls_t;
 52 | 
 53 | struct qsbr {
 54 | 	/*
 55 | 	 * The global epoch, TLS key with a list of the registered threads.
 56 | 	 */
 57 | 	qsbr_epoch_t		global_epoch;
 58 | 	pthread_key_t		tls_key;
 59 | 	pthread_mutex_t		lock;
 60 | 	LIST_HEAD(, qsbr_tls)	list;
 61 | };
 62 | 
 63 | qsbr_t *
 64 | qsbr_create(void)
 65 | {
 66 | 	qsbr_t *qs;
 67 | 	int ret;
 68 | 
 69 | 	ret = posix_memalign((void **)&qs, CACHE_LINE_SIZE, sizeof(qsbr_t));
 70 | 	if (ret != 0) {
 71 | 		errno = ret;
 72 | 		return NULL;
 73 | 	}
 74 | 	memset(qs, 0, sizeof(qsbr_t));
 75 | 
 76 | 	if (pthread_key_create(&qs->tls_key, free) != 0) {
 77 | 		free(qs);
 78 | 		return NULL;
 79 | 	}
 80 | 	pthread_mutex_init(&qs->lock, NULL);
 81 | 	qs->global_epoch = 1;
 82 | 	return qs;
 83 | }
 84 | 
 85 | void
 86 | qsbr_destroy(qsbr_t *qs)
 87 | {
 88 | 	pthread_key_delete(qs->tls_key);
 89 | 	pthread_mutex_destroy(&qs->lock);
 90 | 	free(qs);
 91 | }
 92 | 
 93 | /*
 94 |  * qsbr_register: register the current thread for QSBR.
 95 |  */
 96 | int
 97 | qsbr_register(qsbr_t *qs)
 98 | {
 99 | 	qsbr_tls_t *t;
100 | 
101 | 	t = pthread_getspecific(qs->tls_key);
102 | 	if (__predict_false(t == NULL)) {
103 | 		int ret;
104 | 
105 | 		ret = posix_memalign((void **)&t,
106 | 		    CACHE_LINE_SIZE, sizeof(qsbr_tls_t));
107 | 		if (ret != 0) {
108 | 			errno = ret;
109 | 			return -1;
110 | 		}
111 | 		pthread_setspecific(qs->tls_key, t);
112 | 	}
113 | 	memset(t, 0, sizeof(qsbr_tls_t));
114 | 
115 | 	pthread_mutex_lock(&qs->lock);
116 | 	LIST_INSERT_HEAD(&qs->list, t, entry);
117 | 	pthread_mutex_unlock(&qs->lock);
118 | 	return 0;
119 | }
120 | 
121 | void
122 | qsbr_unregister(qsbr_t *qsbr)
123 | {
124 | 	qsbr_tls_t *t;
125 | 
126 | 	t = pthread_getspecific(qsbr->tls_key);
127 | 	if (t == NULL) {
128 | 		return;
129 | 	}
130 | 	pthread_setspecific(qsbr->tls_key, NULL);
131 | 
132 | 	pthread_mutex_lock(&qsbr->lock);
133 | 	LIST_REMOVE(t, entry);
134 | 	pthread_mutex_unlock(&qsbr->lock);
135 | 	free(t);
136 | }
137 | 
138 | /*
139 |  * qsbr_checkpoint: indicate a quiescent state of the current thread.
140 |  */
141 | void
142 | qsbr_checkpoint(qsbr_t *qs)
143 | {
144 | 	qsbr_tls_t *t;
145 | 
146 | 	t = pthread_getspecific(qs->tls_key);
147 | 	ASSERT(t != NULL);
148 | 
149 | 	/*
150 | 	 * Observe the current epoch and issue a load barrier.
151 | 	 *
152 | 	 * Additionally, issue a store barrier before observation,
153 | 	 * so the callers could assume qsbr_checkpoint() being a
154 | 	 * full barrier.
155 | 	 */
156 | 	atomic_thread_fence(memory_order_seq_cst);
157 | 	t->local_epoch = qs->global_epoch;
158 | }
159 | 
160 | qsbr_epoch_t
161 | qsbr_barrier(qsbr_t *qs)
162 | {
163 | 	/* Note: atomic operation will issue a store barrier. */
164 | 	return atomic_fetch_add(&qs->global_epoch, 1) + 1;
165 | }
166 | 
167 | bool
168 | qsbr_sync(qsbr_t *qs, qsbr_epoch_t target)
169 | {
170 | 	qsbr_tls_t *t;
171 | 
172 | 	/*
173 | 	 * First, our thread should observe the epoch itself.
174 | 	 */
175 | 	qsbr_checkpoint(qs);
176 | 
177 | 	/*
178 | 	 * Have all threads observed the target epoch?
179 | 	 */
180 | 	LIST_FOREACH(t, &qs->list, entry) {
181 | 		if (t->local_epoch < target) {
182 | 			/* Not ready to G/C. */
183 | 			return false;
184 | 		}
185 | 	}
186 | 
187 | 	/* Detected the grace period. */
188 | 	return true;
189 | }
190 | 


--------------------------------------------------------------------------------
/src/qsbr.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (c) 2015 Mindaugas Rasiukevicius <rmind at noxt eu>
 3 |  * All rights reserved.
 4 |  *
 5 |  * Use is subject to license terms, as specified in the LICENSE file.
 6 |  */
 7 | 
 8 | #ifndef	_QSBR_H_
 9 | #define	_QSBR_H_
10 | 
11 | #include <sys/cdefs.h>
12 | #include <stdbool.h>
13 | 
14 | struct qsbr;
15 | typedef struct qsbr qsbr_t;
16 | typedef unsigned long qsbr_epoch_t;
17 | 
18 | __BEGIN_DECLS
19 | 
20 | qsbr_t *	qsbr_create(void);
21 | void		qsbr_destroy(qsbr_t *);
22 | 
23 | void		qsbr_unregister(qsbr_t *);
24 | int		qsbr_register(qsbr_t *);
25 | void		qsbr_checkpoint(qsbr_t *);
26 | qsbr_epoch_t	qsbr_barrier(qsbr_t *);
27 | bool		qsbr_sync(qsbr_t *, qsbr_epoch_t);
28 | 
29 | __END_DECLS
30 | 
31 | #endif
32 | 


--------------------------------------------------------------------------------
/src/t_gc.c:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2018 Mindaugas Rasiukevicius <rmind at noxt eu>
  3 |  * All rights reserved.
  4 |  *
  5 |  * Use is subject to license terms, as specified in the LICENSE file.
  6 |  */
  7 | 
  8 | #include <stdio.h>
  9 | #include <stddef.h>
 10 | #include <stdlib.h>
 11 | #include <stdbool.h>
 12 | #include <string.h>
 13 | #include <inttypes.h>
 14 | #include <assert.h>
 15 | 
 16 | #include "gc.h"
 17 | 
 18 | typedef struct {
 19 | 	bool		destroyed;
 20 | 	gc_entry_t	entry;
 21 | } obj_t;
 22 | 
 23 | static void
 24 | free_objs(gc_entry_t *entry, void *arg)
 25 | {
 26 | 	while (entry) {
 27 | 		obj_t *obj;
 28 | 
 29 | 		obj = (void *)((uintptr_t)entry - offsetof(obj_t, entry));
 30 | 		entry = entry->next;
 31 | 
 32 | 		obj->destroyed = true;
 33 | 	}
 34 | 	(void)arg;
 35 | }
 36 | 
 37 | static void
 38 | test_basic(void)
 39 | {
 40 | 	gc_t *gc;
 41 | 	obj_t obj;
 42 | 
 43 | 	gc = gc_create(offsetof(obj_t, entry), free_objs, NULL);
 44 | 	assert(gc != NULL);
 45 | 
 46 | 	/*
 47 | 	 * Basic critical path.
 48 | 	 */
 49 | 	gc_register(gc);
 50 | 	gc_crit_enter(gc);
 51 | 	gc_crit_exit(gc);
 52 | 
 53 | 	/*
 54 | 	 * Basic reclaim.
 55 | 	 */
 56 | 	memset(&obj, 0, sizeof(obj));
 57 | 	assert(!obj.destroyed);
 58 | 
 59 | 	gc_limbo(gc, &obj);
 60 | 	gc_cycle(gc);
 61 | 	assert(obj.destroyed);
 62 | 
 63 | 	/*
 64 | 	 * Basic reclaim.
 65 | 	 */
 66 | 	memset(&obj, 0, sizeof(obj));
 67 | 	assert(!obj.destroyed);
 68 | 
 69 | 	gc_limbo(gc, &obj);
 70 | 	gc_cycle(gc);
 71 | 	assert(obj.destroyed);
 72 | 
 73 | 	/*
 74 | 	 * Active reference.
 75 | 	 */
 76 | 	memset(&obj, 0, sizeof(obj));
 77 | 	assert(!obj.destroyed);
 78 | 
 79 | 	gc_limbo(gc, &obj);
 80 | 	assert(!obj.destroyed);
 81 | 
 82 | 	gc_crit_enter(gc);
 83 | 	gc_cycle(gc);
 84 | 	assert(!obj.destroyed);
 85 | 
 86 | 	gc_crit_exit(gc);
 87 | 	gc_cycle(gc);
 88 | 	assert(obj.destroyed);
 89 | 
 90 | 	/*
 91 | 	 * Full call.
 92 | 	 */
 93 | 	memset(&obj, 0, sizeof(obj));
 94 | 	assert(!obj.destroyed);
 95 | 
 96 | 	gc_limbo(gc, &obj);
 97 | 	gc_full(gc, 1);
 98 | 	assert(obj.destroyed);
 99 | 
100 | 	gc_destroy(gc);
101 | }
102 | 
103 | int
104 | main(void)
105 | {
106 | 	test_basic();
107 | 	puts("ok");
108 | 	return 0;
109 | }
110 | 


--------------------------------------------------------------------------------
/src/t_stress.c:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2016-2018 Mindaugas Rasiukevicius <rmind at noxt eu>
  3 |  * All rights reserved.
  4 |  *
  5 |  * Use is subject to license terms, as specified in the LICENSE file.
  6 |  */
  7 | 
  8 | #include <stdio.h>
  9 | #include <stddef.h>
 10 | #include <stdlib.h>
 11 | #include <stdbool.h>
 12 | #include <inttypes.h>
 13 | #include <signal.h>
 14 | #include <unistd.h>
 15 | #include <string.h>
 16 | #include <pthread.h>
 17 | #include <errno.h>
 18 | #include <err.h>
 19 | 
 20 | #include "ebr.h"
 21 | #include "qsbr.h"
 22 | #include "gc.h"
 23 | #include "utils.h"
 24 | 
 25 | static unsigned			nsec = 10; /* seconds */
 26 | 
 27 | static pthread_barrier_t	barrier;
 28 | static unsigned			nworkers;
 29 | static volatile bool		stop;
 30 | 
 31 | #define	CACHE_LINE_SIZE		64
 32 | 
 33 | typedef struct {
 34 | 	unsigned int *		ptr;
 35 | 	unsigned int		visible;
 36 | 	unsigned int		gc_epoch;
 37 | 	gc_entry_t		gc_entry;
 38 | 	char			_pad[CACHE_LINE_SIZE - 8 - 4 - 4 - 8];
 39 | } data_struct_t;
 40 | 
 41 | #define	DS_COUNT		4
 42 | #define	MAGIC_VAL		0x5a5a5a5a
 43 | #define	EPOCH_OFF		EBR_EPOCHS
 44 | 
 45 | static unsigned			magic_val = MAGIC_VAL;
 46 | 
 47 | static ebr_t *			ebr;
 48 | static qsbr_t *			qsbr;
 49 | static gc_t *			gc;
 50 | 
 51 | static data_struct_t		ds[DS_COUNT]
 52 |     __attribute__((__aligned__(CACHE_LINE_SIZE)));
 53 | static uint64_t			destructions;
 54 | 
 55 | static void
 56 | access_obj(data_struct_t *obj)
 57 | {
 58 | 	if (atomic_load_explicit(&obj->visible, memory_order_relaxed)) {
 59 | 		atomic_thread_fence(memory_order_acquire);
 60 | 		if (*obj->ptr != MAGIC_VAL) {
 61 | 			abort();
 62 | 		}
 63 | 	}
 64 | }
 65 | 
 66 | static void
 67 | mock_insert_obj(data_struct_t *obj)
 68 | {
 69 | 	obj->ptr = &magic_val;
 70 | 	assert(!obj->visible);
 71 | 	atomic_thread_fence(memory_order_release);
 72 | 	atomic_store_explicit(&obj->visible, true, memory_order_relaxed);
 73 | }
 74 | 
 75 | static void
 76 | mock_remove_obj(data_struct_t *obj)
 77 | {
 78 | 	assert(obj->visible);
 79 | 	obj->visible = false;
 80 | }
 81 | 
 82 | static void
 83 | mock_destroy_obj(data_struct_t *obj)
 84 | {
 85 | 	obj->ptr = NULL;
 86 | 	destructions++;
 87 | }
 88 | 
 89 | /*
 90 |  * EBR stress test.
 91 |  */
 92 | 
 93 | static void
 94 | ebr_writer(unsigned target)
 95 | {
 96 | 	data_struct_t *obj = &ds[target];
 97 | 	unsigned gc_epoch;
 98 | 
 99 | 	if (obj->visible) {
100 | 		/*
101 | 		 * The data structure is visible.  First, ensure it is no
102 | 		 * longer visible (think of "remove" semantics).
103 | 		 */
104 | 		mock_remove_obj(obj);
105 | 		obj->gc_epoch = EBR_EPOCHS + ebr_staging_epoch(ebr);
106 | 
107 | 	} else if (!obj->gc_epoch) {
108 | 		/*
109 | 		 * Data structure is not globally visible.  Set the value
110 | 		 * and make it visible (think of the "insert" semantics).
111 | 		 */
112 | 		mock_insert_obj(obj);
113 | 	} else {
114 | 		/* Invisible, but not yet reclaimed. */
115 | 		assert(obj->gc_epoch != 0);
116 | 	}
117 | 
118 | 	ebr_sync(ebr, &gc_epoch);
119 | 
120 | 	for (unsigned i = 0; i < DS_COUNT; i++) {
121 | 		if (obj->gc_epoch == EPOCH_OFF + gc_epoch) {
122 | 			mock_destroy_obj(obj);
123 | 			obj->gc_epoch = 0;
124 | 		}
125 | 	}
126 | }
127 | 
128 | static void *
129 | ebr_stress(void *arg)
130 | {
131 | 	const unsigned id = (uintptr_t)arg;
132 | 	unsigned n = 0;
133 | 
134 | 	ebr_register(ebr);
135 | 
136 | 	/*
137 | 	 * There are NCPU threads concurrently reading data and a single
138 | 	 * writer thread (ID 0) modifying data.  The writer will modify
139 | 	 * the pointer used by the readers to NULL as soon as it considers
140 | 	 * the object ready for reclaim.
141 | 	 */
142 | 
143 | 	pthread_barrier_wait(&barrier);
144 | 	while (!stop) {
145 | 		n = (n + 1) & (DS_COUNT - 1);
146 | 
147 | 		if (id == 0) {
148 | 			ebr_writer(n);
149 | 			continue;
150 | 		}
151 | 
152 | 		/*
153 | 		 * Reader: iterate through the data structures and,
154 | 		 * if the object is visible (think of "lookup" semantics),
155 | 		 * read its value through a pointer.  The writer will set
156 | 		 * the pointer to NULL when it thinks the object is ready
157 | 		 * to be reclaimed.
158 | 		 *
159 | 		 * Incorrect reclamation mechanism would lead to the crash
160 | 		 * in the following pointer dereference.
161 | 		 */
162 | 		ebr_enter(ebr);
163 | 		access_obj(&ds[n]);
164 | 		ebr_exit(ebr);
165 | 	}
166 | 	pthread_barrier_wait(&barrier);
167 | 	ebr_unregister(ebr);
168 | 	pthread_exit(NULL);
169 | 	return NULL;
170 | }
171 | 
172 | /*
173 |  * QSBR stress test.
174 |  */
175 | 
176 | static void
177 | qsbr_writer(unsigned target)
178 | {
179 | 	data_struct_t *obj = &ds[target];
180 | 
181 | 	/*
182 | 	 * See the ebr_writer() function for more details.
183 | 	 */
184 | 	if (obj->visible) {
185 | 		unsigned count = SPINLOCK_BACKOFF_MIN;
186 | 		qsbr_epoch_t target_epoch;
187 | 
188 | 		mock_remove_obj(obj);
189 | 
190 | 		/* QSBR synchronisation barrier. */
191 | 		target_epoch = qsbr_barrier(qsbr);
192 | 		while (!qsbr_sync(qsbr, target_epoch)) {
193 | 			SPINLOCK_BACKOFF(count);
194 | 			if (stop) {
195 | 				/*
196 | 				 * Other threads might have exited and
197 | 				 * the checkpoint would never be passed.
198 | 				 */
199 | 				return;
200 | 			}
201 | 		}
202 | 
203 | 		/* It is safe to "destroy" the object now. */
204 | 		mock_destroy_obj(obj);
205 | 	} else {
206 | 		mock_insert_obj(obj);
207 | 	}
208 | }
209 | 
210 | static void *
211 | qsbr_stress(void *arg)
212 | {
213 | 	const unsigned id = (uintptr_t)arg;
214 | 	unsigned n = 0;
215 | 
216 | 	/*
217 | 	 * See the ebr_stress() function for explanation.
218 | 	 */
219 | 
220 | 	qsbr_register(qsbr);
221 | 	pthread_barrier_wait(&barrier);
222 | 	while (!stop) {
223 | 		n = (n + 1) & (DS_COUNT - 1);
224 | 		if (id == 0) {
225 | 			qsbr_writer(n);
226 | 			continue;
227 | 		}
228 | 		access_obj(&ds[n]);
229 | 		qsbr_checkpoint(qsbr);
230 | 	}
231 | 	pthread_barrier_wait(&barrier);
232 | 	qsbr_unregister(qsbr);
233 | 	pthread_exit(NULL);
234 | 	return NULL;
235 | }
236 | 
237 | /*
238 |  * G/C stress test.
239 |  */
240 | 
241 | static void
242 | gc_func(gc_entry_t *entry, void *arg)
243 | {
244 | 	const unsigned off = offsetof(data_struct_t, gc_entry);
245 | 
246 | 	while (entry) {
247 | 		data_struct_t *obj;
248 | 
249 | 		obj = (void *)((uintptr_t)entry - off);
250 | 		entry = entry->next;
251 | 		mock_destroy_obj(obj);
252 | 	}
253 | 	(void)arg;
254 | }
255 | 
256 | static void
257 | gc_writer(unsigned target)
258 | {
259 | 	data_struct_t *obj = &ds[target];
260 | 
261 | 	if (obj->visible) {
262 | 		mock_remove_obj(obj);
263 | 		gc_limbo(gc, obj);
264 | 	} else if (!obj->ptr) {
265 | 		mock_insert_obj(obj);
266 | 	}
267 | 	gc_cycle(gc);
268 | }
269 | 
270 | static void *
271 | gc_stress(void *arg)
272 | {
273 | 	const unsigned id = (uintptr_t)arg;
274 | 	unsigned n = 0;
275 | 
276 | 	/*
277 | 	 * See the ebr_stress() function for explanation.
278 | 	 */
279 | 
280 | 	gc_register(gc);
281 | 	pthread_barrier_wait(&barrier);
282 | 	while (!stop) {
283 | 		n = (n + 1) & (DS_COUNT - 1);
284 | 		if (id == 0) {
285 | 			gc_writer(n);
286 | 			continue;
287 | 		}
288 | 		gc_crit_enter(gc);
289 | 		access_obj(&ds[n]);
290 | 		gc_crit_exit(gc);
291 | 	}
292 | 	pthread_barrier_wait(&barrier);
293 | 	gc_unregister(gc);
294 | 	pthread_exit(NULL);
295 | 	return NULL;
296 | }
297 | 
298 | /*
299 |  * Helper routines
300 |  */
301 | 
302 | static void
303 | ding(int sig)
304 | {
305 | 	(void)sig;
306 | 	stop = true;
307 | }
308 | 
309 | static void
310 | run_test(void *func(void *))
311 | {
312 | 	struct sigaction sigalarm;
313 | 	pthread_t *thr;
314 | 	int ret;
315 | 
316 | 	/*
317 | 	 * Setup the threads.
318 | 	 */
319 | 	nworkers = sysconf(_SC_NPROCESSORS_CONF);
320 | 	thr = calloc(nworkers, sizeof(pthread_t));
321 | 	pthread_barrier_init(&barrier, NULL, nworkers);
322 | 	stop = false;
323 | 
324 | 	memset(&sigalarm, 0, sizeof(struct sigaction));
325 | 	sigalarm.sa_handler = ding;
326 | 	ret = sigaction(SIGALRM, &sigalarm, NULL);
327 | 	assert(ret == 0); (void)ret;
328 | 
329 | 	/*
330 | 	 * Create some data structures and the EBR object.
331 | 	 */
332 | 	memset(&ds, 0, sizeof(ds));
333 | 	ebr = ebr_create();
334 | 	qsbr = qsbr_create();
335 | 	gc = gc_create(offsetof(data_struct_t, gc_entry), gc_func, NULL);
336 | 	destructions = 0;
337 | 
338 | 	/*
339 | 	 * Spin the test.
340 | 	 */
341 | 	alarm(nsec);
342 | 
343 | 	for (unsigned i = 0; i < nworkers; i++) {
344 | 		if ((errno = pthread_create(&thr[i], NULL,
345 | 		    func, (void *)(uintptr_t)i)) != 0) {
346 | 			err(EXIT_FAILURE, "pthread_create");
347 | 		}
348 | 	}
349 | 	for (unsigned i = 0; i < nworkers; i++) {
350 | 		pthread_join(thr[i], NULL);
351 | 	}
352 | 	pthread_barrier_destroy(&barrier);
353 | 	printf("# %"PRIu64"\n", destructions);
354 | 
355 | 	ebr_destroy(ebr);
356 | 	qsbr_destroy(qsbr);
357 | 
358 | 	gc_full(gc, 1);
359 | 	gc_destroy(gc);
360 | }
361 | 
362 | int
363 | main(int argc, char **argv)
364 | {
365 | 	if (argc >= 2) {
366 | 		nsec = (unsigned)atoi(argv[1]);
367 | 	}
368 | 	puts("stress test");
369 | 	run_test(ebr_stress);
370 | 	run_test(qsbr_stress);
371 | 	run_test(gc_stress);
372 | 	puts("ok");
373 | 	return 0;
374 | }
375 | 


--------------------------------------------------------------------------------
/src/utils.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (c) 2015-2018 Mindaugas Rasiukevicius <rmind at noxt eu>
 3 |  * All rights reserved.
 4 |  *
 5 |  * Use is subject to license terms, as specified in the LICENSE file.
 6 |  */
 7 | 
 8 | #ifndef	_UTILS_H_
 9 | #define	_UTILS_H_
10 | 
11 | #include <assert.h>
12 | 
13 | /*
14 |  * A regular assert (debug/diagnostic only).
15 |  */
16 | #if defined(DEBUG)
17 | #define	ASSERT		assert
18 | #else
19 | #define	ASSERT(x)
20 | #endif
21 | 
22 | /*
23 |  * Branch prediction macros.
24 |  */
25 | #ifndef __predict_true
26 | #define	__predict_true(x)	__builtin_expect((x) != 0, 1)
27 | #define	__predict_false(x)	__builtin_expect((x) != 0, 0)
28 | #endif
29 | 
30 | /*
31 |  * Atomic operations and memory barriers.  If C11 API is not available,
32 |  * then wrap the GCC builtin routines.
33 |  */
34 | #ifndef atomic_compare_exchange_weak
35 | #define	atomic_compare_exchange_weak(ptr, expected, desired) \
36 |     __sync_bool_compare_and_swap(ptr, expected, desired)
37 | #endif
38 | 
39 | #ifndef atomic_exchange
40 | static inline void *
41 | atomic_exchange(volatile void *ptr, void *newval)
42 | {
43 | 	void * volatile *ptrp = (void * volatile *)ptr;
44 | 	void *oldval;
45 | again:
46 | 	oldval = *ptrp;
47 | 	if (!__sync_bool_compare_and_swap(ptrp, oldval, newval)) {
48 | 		goto again;
49 | 	}
50 | 	return oldval;
51 | }
52 | #endif
53 | 
54 | #ifndef atomic_fetch_add
55 | #define	atomic_fetch_add(x,a)	__sync_fetch_and_add(x, a)
56 | #endif
57 | 
58 | #ifndef atomic_thread_fence
59 | #define	memory_order_relaxed	__ATOMIC_RELAXED
60 | #define	memory_order_acquire	__ATOMIC_ACQUIRE
61 | #define	memory_order_release	__ATOMIC_RELEASE
62 | #define	memory_order_seq_cst	__ATOMIC_SEQ_CST
63 | #define	atomic_thread_fence(m)	__atomic_thread_fence(m)
64 | #endif
65 | #ifndef atomic_store_explicit
66 | #define	atomic_store_explicit	__atomic_store_n
67 | #endif
68 | #ifndef atomic_load_explicit
69 | #define	atomic_load_explicit	__atomic_load_n
70 | #endif
71 | 
72 | /*
73 |  * Exponential back-off for the spinning paths.
74 |  */
75 | #define	SPINLOCK_BACKOFF_MIN	4
76 | #define	SPINLOCK_BACKOFF_MAX	128
77 | #if defined(__x86_64__) || defined(__i386__)
78 | #define SPINLOCK_BACKOFF_HOOK	__asm volatile("pause" ::: "memory")
79 | #else
80 | #define SPINLOCK_BACKOFF_HOOK
81 | #endif
82 | #define	SPINLOCK_BACKOFF(count)					\
83 | do {								\
84 | 	for (int __i = (count); __i != 0; __i--) {		\
85 | 		SPINLOCK_BACKOFF_HOOK;				\
86 | 	}							\
87 | 	if ((count) < SPINLOCK_BACKOFF_MAX)			\
88 | 		(count) += (count);				\
89 | } while (/* CONSTCOND */ 0);
90 | 
91 | /*
92 |  * Cache line size - a reasonable upper bound.
93 |  */
94 | #define	CACHE_LINE_SIZE		64
95 | 
96 | #endif
97 | 


--------------------------------------------------------------------------------