├── .travis.yml
├── LICENSE
├── README.md
└── src
    ├── Makefile
    ├── ringbuf.c
    ├── ringbuf.h
    ├── t_ringbuf.c
    ├── t_stress.c
    └── utils.h


/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: c
 2 | 
 3 | compiler:
 4 |   - gcc
 5 |   - clang
 6 | 
 7 | dist: bionic
 8 | 
 9 | matrix:
10 |   include:
11 |     - os: linux
12 |       arch: amd64
13 |     - os: linux
14 |       arch: arm64
15 |     - os: linux
16 |       arch: ppc64le
17 | 
18 | addons:
19 |   apt:
20 |     update: true
21 |     packages:
22 |     - build-essential
23 |     - libtool
24 |     - libtool-bin
25 | 
26 | script:
27 |   - (cd src && make clean && make tests && make stress)
28 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | /*-
 2 |  * Copyright (c) 2016-2017 Mindaugas Rasiukevicius <rmind at noxt eu>
 3 |  * All rights reserved.
 4 |  *
 5 |  * Redistribution and use in source and binary forms, with or without
 6 |  * modification, are permitted provided that the following conditions
 7 |  * are met:
 8 |  * 1. Redistributions of source code must retain the above copyright
 9 |  *    notice, this list of conditions and the following disclaimer.
10 |  * 2. Redistributions in binary form must reproduce the above copyright
11 |  *    notice, this list of conditions and the following disclaimer in the
12 |  *    documentation and/or other materials provided with the distribution.
13 |  *
14 |  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 |  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 |  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 |  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 |  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 |  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 |  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 |  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 |  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 |  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 |  * SUCH DAMAGE.
25 |  */
26 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Lock-free ring buffer
  2 | 
  3 | [![Build Status](https://travis-ci.org/rmind/ringbuf.svg?branch=master)](https://travis-ci.org/rmind/ringbuf)
  4 | 
  5 | Lock-free multi-producer single-consumer (MPSC) ring buffer which supports
  6 | contiguous range operations and which can be conveniently used for message
  7 | passing.  The implementation is written in C11 and distributed under the
  8 | 2-clause BSD license.
  9 | 
 10 | ## API
 11 | 
 12 | * `int ringbuf_setup(ringbuf_t *rbuf, unsigned nworkers, size_t length)`
 13 |   * Setup a new ring buffer of a given _length_.  The `rbuf` is a pointer
 14 |   to the opaque ring buffer object; the caller is responsible to allocate
 15 |   the space for this object.  Typically, the object would be allocated
 16 |   dynamically if using threads or reserved in a shared memory blocked if
 17 |   using processes.  The allocation size for the object shall be obtained
 18 |   using the `ringbuf_get_sizes` function.  Returns 0 on success and -1
 19 |   on failure.
 20 | 
 21 | * `void ringbuf_get_sizes(unsigned nworkers, size_t *ringbuf_obj_size, size_t *ringbuf_worker_size)`
 22 |   * Returns the size of the opaque `ringbuf_t` and, optionally, `ringbuf_worker_t` structures.
 23 |   The size of the `ringbuf_t` structure depends on the number of workers,
 24 |   specified by the `nworkers` parameter.
 25 | 
 26 | * `ringbuf_worker_t *ringbuf_register(ringbuf_t *rbuf, unsigned i)`
 27 |   * Register the current worker (thread or process) as a producer.  Each
 28 |   producer MUST register itself.  The `i` is a worker number, starting
 29 |   from zero (i.e. shall be than `nworkers` used in the setup).  On success,
 30 |   returns a pointer to an opaque `ringbuf_worker_t` structured, which is
 31 |   a part of the `ringbuf_t` memory block.  On failure, returns `NULL`.
 32 | 
 33 | * `void ringbuf_unregister(ringbuf_t *rbuf, ringbuf_worker_t *worker)`
 34 |   * Unregister the specified worker from the list of producers.
 35 | 
 36 | * `ssize_t ringbuf_acquire(ringbuf_t *rbuf, ringbuf_worker_t *worker, size_t len)`
 37 |   * Request a space of a given length in the ring buffer.  Returns the
 38 |   offset at which the space is available or -1 on failure.  Once the data
 39 |   is ready (typically, when writing to the ring buffer is complete), the
 40 |   `ringbuf_produce` function must be called to indicate that.  Nested
 41 |   acquire calls are not allowed.
 42 | 
 43 | * `void ringbuf_produce(ringbuf_t *rbuf, ringbuf_worker_t *worker)`
 44 |   * Indicate that the acquired range in the buffer is produced and is ready
 45 |   to be consumed.
 46 | 
 47 | * `size_t ringbuf_consume(ringbuf_t *rbuf, size_t *offset)`
 48 |   * Get a contiguous range which is ready to be consumed.  Returns zero
 49 |   if there is no data available for consumption.  Once the data is
 50 |   consumed (typically, when reading from the ring buffer is complete),
 51 |   the `ringbuf_release` function must be called to indicate that.
 52 | 
 53 | * `void ringbuf_release(ringbuf_t *rbuf, size_t nbytes)`
 54 |   * Indicate that the consumed range can now be released and may now be
 55 |   reused by the producers.
 56 | 
 57 | ## Notes
 58 | 
 59 | The consumer will return a contiguous block of ranges produced i.e. the
 60 | `ringbuf_consume` call will not return partial ranges.  If you think of
 61 | produced range as a message, then consumer will return a block of messages,
 62 | always ending at the message boundary.  Such behaviour allows us to use
 63 | this ring buffer implementation as a message queue.
 64 | 
 65 | The implementation was extensively tested on a 24-core x86 machine,
 66 | see [the stress test](src/t_stress.c) for the details on the technique.
 67 | It also provides an example how the mechanism can be used for message
 68 | passing.
 69 | 
 70 | ## Caveats
 71 | 
 72 | This ring buffer implementation always provides a contiguous range of
 73 | space for the producer.  It is achieved by an early wrap-around if the
 74 | requested range cannot fit in the end.  The implication of this is that
 75 | the `ringbuf_acquire` call may fail if the requested range is greater
 76 | than half of the buffer size.  Hence, it may be necessary to ensure that
 77 | the ring buffer size is at least twice as large as the maximum production
 78 | unit size.
 79 | 
 80 | It should also be noted that one of the trade-offs of such design is that
 81 | the consumer currently performs an O(n) scan on the list of producers.
 82 | 
 83 | ## Example
 84 | 
 85 | Producers:
 86 | ```c
 87 | if ((w = ringbuf_register(r, worker_id)) == NULL)
 88 | 	err(EXIT_FAILURE, "ringbuf_register")
 89 | 
 90 | ...
 91 | 
 92 | if ((off = ringbuf_acquire(r, w, len)) != -1) {
 93 | 	memcpy(&buf[off], payload, len);
 94 | 	ringbuf_produce(r, tls);
 95 | }
 96 | ```
 97 | 
 98 | Consumer:
 99 | ```c
100 | if ((len = ringbuf_consume(r, &off)) != 0) {
101 | 	process(&buf[off], len);
102 | 	ringbuf_release(r, len);
103 | }
104 | ```
105 | 


--------------------------------------------------------------------------------
/src/Makefile:
--------------------------------------------------------------------------------
 1 | #
 2 | # This file is in the Public Domain.
 3 | #
 4 | 
 5 | SYSNAME:=	$(shell uname -s)
 6 | SYSARCH:=	$(shell uname -m)
 7 | 
 8 | CFLAGS=		-std=c11 -O2 -g -W -Wextra -Werror
 9 | CFLAGS+=	-D_POSIX_C_SOURCE=200809L
10 | CFLAGS+=	-D_GNU_SOURCE -D_DEFAULT_SOURCE
11 | 
12 | #
13 | # Extended warning flags.
14 | #
15 | CFLAGS+=	-Wno-unknown-warning-option # gcc vs clang
16 | 
17 | CFLAGS+=	-Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith
18 | CFLAGS+=	-Wmissing-declarations -Wredundant-decls -Wnested-externs
19 | CFLAGS+=	-Wshadow -Wcast-qual -Wcast-align -Wwrite-strings
20 | CFLAGS+=	-Wold-style-definition
21 | CFLAGS+=	-Wsuggest-attribute=noreturn -Wjump-misses-init
22 | CFLAGS+=	-Wduplicated-cond -Wmisleading-indentation -Wnull-dereference
23 | CFLAGS+=	-Wduplicated-branches -Wrestrict
24 | 
25 | ifeq ($(MAKECMDGOALS),tests)
26 | DEBUG=		1
27 | endif
28 | 
29 | ifeq ($(MAKECMDGOALS),stress)
30 | DEBUG=		1
31 | endif
32 | 
33 | ifeq ($(DEBUG),1)
34 | CFLAGS+=	-Og -DDEBUG -fno-omit-frame-pointer
35 | ifeq ($(SYSARCH),x86_64)
36 | CFLAGS+=	-fsanitize=address -fsanitize=undefined
37 | LDFLAGS+=	-fsanitize=address -fsanitize=undefined
38 | endif
39 | else
40 | CFLAGS+=	-DNDEBUG
41 | endif
42 | 
43 | LIB=		libringbuf
44 | INCS=		ringbuf.h
45 | 
46 | OBJS=		ringbuf.o
47 | 
48 | $(LIB).la:	LDFLAGS+=	-rpath $(LIBDIR)
49 | install/%.la:	ILIBDIR=	$(DESTDIR)/$(LIBDIR)
50 | install:	IINCDIR=	$(DESTDIR)/$(INCDIR)/
51 | #install:	IMANDIR=	$(DESTDIR)/$(MANDIR)/man3/
52 | 
53 | obj: $(OBJS)
54 | 
55 | lib: $(LIB).la
56 | 
57 | %.lo: %.c
58 | 	libtool --mode=compile --tag CC $(CC) $(CFLAGS) -c $<
59 | 
60 | $(LIB).la: $(shell echo $(OBJS) | sed 's/\.o/\.lo/g')
61 | 	libtool --mode=link --tag CC $(CC) $(LDFLAGS) -o $@ $(notdir $^)
62 | 
63 | install/%.la: %.la
64 | 	mkdir -p $(ILIBDIR)
65 | 	libtool --mode=install install -c $(notdir $@) $(ILIBDIR)/$(notdir $@)
66 | 
67 | install: $(addprefix install/,$(LIB).la)
68 | 	libtool --mode=finish $(LIBDIR)
69 | 	mkdir -p $(IINCDIR) && install -c $(INCS) $(IINCDIR)
70 | 	#mkdir -p $(IMANDIR) && install -c $(MANS) $(IMANDIR)
71 | 
72 | tests: $(OBJS) t_ringbuf.o
73 | 	$(CC) $(CFLAGS) $^ -o t_ringbuf -lpthread
74 | 	./t_ringbuf
75 | 
76 | stress: $(OBJS) t_stress.o
77 | 	$(CC) $(CFLAGS) $^ -o t_stress $(LDFLAGS) -lpthread
78 | 	./t_stress
79 | 
80 | clean:
81 | 	libtool --mode=clean rm
82 | 	rm -rf .libs *.o *.lo *.la t_ringbuf t_stress
83 | 
84 | .PHONY: all obj lib install tests stress clean
85 | 


--------------------------------------------------------------------------------
/src/ringbuf.c:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2016-2017 Mindaugas Rasiukevicius <rmind at noxt eu>
  3 |  * All rights reserved.
  4 |  *
  5 |  * Use is subject to license terms, as specified in the LICENSE file.
  6 |  */
  7 | 
  8 | /*
  9 |  * Atomic multi-producer single-consumer ring buffer, which supports
 10 |  * contiguous range operations and which can be conveniently used for
 11 |  * message passing.
 12 |  *
 13 |  * There are three offsets -- think of clock hands:
 14 |  * - NEXT: marks the beginning of the available space,
 15 |  * - WRITTEN: the point up to which the data is actually written.
 16 |  * - Observed READY: point up to which data is ready to be written.
 17 |  *
 18 |  * Producers
 19 |  *
 20 |  *	Observe and save the 'next' offset, then request N bytes from
 21 |  *	the ring buffer by atomically advancing the 'next' offset.  Once
 22 |  *	the data is written into the "reserved" buffer space, the thread
 23 |  *	clears the saved value; these observed values are used to compute
 24 |  *	the 'ready' offset.
 25 |  *
 26 |  * Consumer
 27 |  *
 28 |  *	Writes the data between 'written' and 'ready' offsets and updates
 29 |  *	the 'written' value.  The consumer thread scans for the lowest
 30 |  *	seen value by the producers.
 31 |  *
 32 |  * Key invariant
 33 |  *
 34 |  *	Producers cannot go beyond the 'written' offset; producers are
 35 |  *	also not allowed to catch up with the consumer.  Only the consumer
 36 |  *	is allowed to catch up with the producer i.e. set the 'written'
 37 |  *	offset to be equal to the 'next' offset.
 38 |  *
 39 |  * Wrap-around
 40 |  *
 41 |  *	If the producer cannot acquire the requested length due to little
 42 |  *	available space at the end of the buffer, then it will wraparound.
 43 |  *	WRAP_LOCK_BIT in 'next' offset is used to lock the 'end' offset.
 44 |  *
 45 |  *	There is an ABA problem if one producer stalls while a pair of
 46 |  *	producer and consumer would both successfully wrap-around and set
 47 |  *	the 'next' offset to the stale value of the first producer, thus
 48 |  *	letting it to perform a successful CAS violating the invariant.
 49 |  *	A counter in the 'next' offset (masked by WRAP_COUNTER) is used
 50 |  *	to prevent from this problem.  It is incremented on wraparounds.
 51 |  *
 52 |  *	The same ABA problem could also cause a stale 'ready' offset,
 53 |  *	which could be observed by the consumer.  We set WRAP_LOCK_BIT in
 54 |  *	the 'seen' value before advancing the 'next' and clear this bit
 55 |  *	after the successful advancing; this ensures that only the stable
 56 |  *	'ready' is observed by the consumer.
 57 |  */
 58 | 
 59 | #include <stdio.h>
 60 | #include <stdlib.h>
 61 | #include <stddef.h>
 62 | #include <stdbool.h>
 63 | #include <inttypes.h>
 64 | #include <string.h>
 65 | #include <limits.h>
 66 | #include <errno.h>
 67 | 
 68 | #include "ringbuf.h"
 69 | #include "utils.h"
 70 | 
 71 | #define	RBUF_OFF_MASK	(0x00000000ffffffffUL)
 72 | #define	WRAP_LOCK_BIT	(0x8000000000000000UL)
 73 | #define	RBUF_OFF_MAX	(UINT64_MAX & ~WRAP_LOCK_BIT)
 74 | 
 75 | #define	WRAP_COUNTER	(0x7fffffff00000000UL)
 76 | #define	WRAP_INCR(x)	(((x) + 0x100000000UL) & WRAP_COUNTER)
 77 | 
 78 | typedef uint64_t	ringbuf_off_t;
 79 | 
 80 | struct ringbuf_worker {
 81 | 	volatile ringbuf_off_t	seen_off;
 82 | 	int			registered;
 83 | };
 84 | 
 85 | struct ringbuf {
 86 | 	/* Ring buffer space. */
 87 | 	size_t			space;
 88 | 
 89 | 	/*
 90 | 	 * The NEXT hand is atomically updated by the producer.
 91 | 	 * WRAP_LOCK_BIT is set in case of wrap-around; in such case,
 92 | 	 * the producer can update the 'end' offset.
 93 | 	 */
 94 | 	volatile ringbuf_off_t	next;
 95 | 	ringbuf_off_t		end;
 96 | 
 97 | 	/* The following are updated by the consumer. */
 98 | 	ringbuf_off_t		written;
 99 | 	unsigned		nworkers;
100 | 	ringbuf_worker_t	workers[];
101 | };
102 | 
103 | /*
104 |  * ringbuf_setup: initialise a new ring buffer of a given length.
105 |  */
106 | int
107 | ringbuf_setup(ringbuf_t *rbuf, unsigned nworkers, size_t length)
108 | {
109 | 	if (length >= RBUF_OFF_MASK) {
110 | 		errno = EINVAL;
111 | 		return -1;
112 | 	}
113 | 	memset(rbuf, 0, offsetof(ringbuf_t, workers[nworkers]));
114 | 	rbuf->space = length;
115 | 	rbuf->end = RBUF_OFF_MAX;
116 | 	rbuf->nworkers = nworkers;
117 | 	return 0;
118 | }
119 | 
120 | /*
121 |  * ringbuf_get_sizes: return the sizes of the ringbuf_t and ringbuf_worker_t.
122 |  */
123 | void
124 | ringbuf_get_sizes(unsigned nworkers,
125 |     size_t *ringbuf_size, size_t *ringbuf_worker_size)
126 | {
127 | 	if (ringbuf_size)
128 | 		*ringbuf_size = offsetof(ringbuf_t, workers[nworkers]);
129 | 	if (ringbuf_worker_size)
130 | 		*ringbuf_worker_size = sizeof(ringbuf_worker_t);
131 | }
132 | 
133 | /*
134 |  * ringbuf_register: register the worker (thread/process) as a producer
135 |  * and pass the pointer to its local store.
136 |  */
137 | ringbuf_worker_t *
138 | ringbuf_register(ringbuf_t *rbuf, unsigned i)
139 | {
140 | 	ringbuf_worker_t *w = &rbuf->workers[i];
141 | 
142 | 	w->seen_off = RBUF_OFF_MAX;
143 | 	atomic_store_explicit(&w->registered, true, memory_order_release);
144 | 	return w;
145 | }
146 | 
147 | void
148 | ringbuf_unregister(ringbuf_t *rbuf, ringbuf_worker_t *w)
149 | {
150 | 	w->registered = false;
151 | 	(void)rbuf;
152 | }
153 | 
154 | /*
155 |  * stable_nextoff: capture and return a stable value of the 'next' offset.
156 |  */
157 | static inline ringbuf_off_t
158 | stable_nextoff(ringbuf_t *rbuf)
159 | {
160 | 	unsigned count = SPINLOCK_BACKOFF_MIN;
161 | 	ringbuf_off_t next;
162 | retry:
163 | 	next = atomic_load_explicit(&rbuf->next, memory_order_acquire);
164 | 	if (next & WRAP_LOCK_BIT) {
165 | 		SPINLOCK_BACKOFF(count);
166 | 		goto retry;
167 | 	}
168 | 	ASSERT((next & RBUF_OFF_MASK) < rbuf->space);
169 | 	return next;
170 | }
171 | 
172 | /*
173 |  * stable_seenoff: capture and return a stable value of the 'seen' offset.
174 |  */
175 | static inline ringbuf_off_t
176 | stable_seenoff(ringbuf_worker_t *w)
177 | {
178 | 	unsigned count = SPINLOCK_BACKOFF_MIN;
179 | 	ringbuf_off_t seen_off;
180 | retry:
181 | 	seen_off = atomic_load_explicit(&w->seen_off, memory_order_acquire);
182 | 	if (seen_off & WRAP_LOCK_BIT) {
183 | 		SPINLOCK_BACKOFF(count);
184 | 		goto retry;
185 | 	}
186 | 	return seen_off;
187 | }
188 | 
189 | /*
190 |  * ringbuf_acquire: request a space of a given length in the ring buffer.
191 |  *
192 |  * => On success: returns the offset at which the space is available.
193 |  * => On failure: returns -1.
194 |  */
195 | ssize_t
196 | ringbuf_acquire(ringbuf_t *rbuf, ringbuf_worker_t *w, size_t len)
197 | {
198 | 	ringbuf_off_t seen, next, target;
199 | 
200 | 	ASSERT(len > 0 && len <= rbuf->space);
201 | 	ASSERT(w->seen_off == RBUF_OFF_MAX);
202 | 
203 | 	do {
204 | 		ringbuf_off_t written;
205 | 
206 | 		/*
207 | 		 * Get the stable 'next' offset.  Save the observed 'next'
208 | 		 * value (i.e. the 'seen' offset), but mark the value as
209 | 		 * unstable (set WRAP_LOCK_BIT).
210 | 		 *
211 | 		 * Note: CAS will issue a memory_order_release for us and
212 | 		 * thus ensures that it reaches global visibility together
213 | 		 * with new 'next'.
214 | 		 */
215 | 		seen = stable_nextoff(rbuf);
216 | 		next = seen & RBUF_OFF_MASK;
217 | 		ASSERT(next < rbuf->space);
218 | 		atomic_store_explicit(&w->seen_off, next | WRAP_LOCK_BIT,
219 | 		    memory_order_relaxed);
220 | 
221 | 		/*
222 | 		 * Compute the target offset.  Key invariant: we cannot
223 | 		 * go beyond the WRITTEN offset or catch up with it.
224 | 		 */
225 | 		target = next + len;
226 | 		written = rbuf->written;
227 | 		if (__predict_false(next < written && target >= written)) {
228 | 			/* The producer must wait. */
229 | 			atomic_store_explicit(&w->seen_off,
230 | 			    RBUF_OFF_MAX, memory_order_release);
231 | 			return -1;
232 | 		}
233 | 
234 | 		if (__predict_false(target >= rbuf->space)) {
235 | 			const bool exceed = target > rbuf->space;
236 | 
237 | 			/*
238 | 			 * Wrap-around and start from the beginning.
239 | 			 *
240 | 			 * If we would exceed the buffer, then attempt to
241 | 			 * acquire the WRAP_LOCK_BIT and use the space in
242 | 			 * the beginning.  If we used all space exactly to
243 | 			 * the end, then reset to 0.
244 | 			 *
245 | 			 * Check the invariant again.
246 | 			 */
247 | 			target = exceed ? (WRAP_LOCK_BIT | len) : 0;
248 | 			if ((target & RBUF_OFF_MASK) >= written) {
249 | 				atomic_store_explicit(&w->seen_off,
250 | 				    RBUF_OFF_MAX, memory_order_release);
251 | 				return -1;
252 | 			}
253 | 			/* Increment the wrap-around counter. */
254 | 			target |= WRAP_INCR(seen & WRAP_COUNTER);
255 | 		} else {
256 | 			/* Preserve the wrap-around counter. */
257 | 			target |= seen & WRAP_COUNTER;
258 | 		}
259 | 	} while (!atomic_compare_exchange_weak(&rbuf->next, &seen, target));
260 | 
261 | 	/*
262 | 	 * Acquired the range.  Clear WRAP_LOCK_BIT in the 'seen' value
263 | 	 * thus indicating that it is stable now.
264 | 	 *
265 | 	 * No need for memory_order_release, since CAS issued a fence.
266 | 	 */
267 | 	atomic_store_explicit(&w->seen_off, w->seen_off & ~WRAP_LOCK_BIT,
268 | 	    memory_order_relaxed);
269 | 
270 | 	/*
271 | 	 * If we set the WRAP_LOCK_BIT in the 'next' (because we exceed
272 | 	 * the remaining space and need to wrap-around), then save the
273 | 	 * 'end' offset and release the lock.
274 | 	 */
275 | 	if (__predict_false(target & WRAP_LOCK_BIT)) {
276 | 		/* Cannot wrap-around again if consumer did not catch-up. */
277 | 		ASSERT(rbuf->written <= next);
278 | 		ASSERT(rbuf->end == RBUF_OFF_MAX);
279 | 		rbuf->end = next;
280 | 		next = 0;
281 | 
282 | 		/*
283 | 		 * Unlock: ensure the 'end' offset reaches global
284 | 		 * visibility before the lock is released.
285 | 		 */
286 | 		atomic_store_explicit(&rbuf->next,
287 | 		    (target & ~WRAP_LOCK_BIT), memory_order_release);
288 | 	}
289 | 	ASSERT((target & RBUF_OFF_MASK) <= rbuf->space);
290 | 	return (ssize_t)next;
291 | }
292 | 
293 | /*
294 |  * ringbuf_produce: indicate the acquired range in the buffer is produced
295 |  * and is ready to be consumed.
296 |  */
297 | void
298 | ringbuf_produce(ringbuf_t *rbuf, ringbuf_worker_t *w)
299 | {
300 | 	(void)rbuf;
301 | 	ASSERT(w->registered);
302 | 	ASSERT(w->seen_off != RBUF_OFF_MAX);
303 | 	atomic_store_explicit(&w->seen_off, RBUF_OFF_MAX, memory_order_release);
304 | }
305 | 
306 | /*
307 |  * ringbuf_consume: get a contiguous range which is ready to be consumed.
308 |  */
309 | size_t
310 | ringbuf_consume(ringbuf_t *rbuf, size_t *offset)
311 | {
312 | 	ringbuf_off_t written = rbuf->written, next, ready;
313 | 	size_t towrite;
314 | retry:
315 | 	/*
316 | 	 * Get the stable 'next' offset.  Note: stable_nextoff() issued
317 | 	 * a load memory barrier.  The area between the 'written' offset
318 | 	 * and the 'next' offset will be the *preliminary* target buffer
319 | 	 * area to be consumed.
320 | 	 */
321 | 	next = stable_nextoff(rbuf) & RBUF_OFF_MASK;
322 | 	if (written == next) {
323 | 		/* If producers did not advance, then nothing to do. */
324 | 		return 0;
325 | 	}
326 | 
327 | 	/*
328 | 	 * Observe the 'ready' offset of each producer.
329 | 	 *
330 | 	 * At this point, some producer might have already triggered the
331 | 	 * wrap-around and some (or all) seen 'ready' values might be in
332 | 	 * the range between 0 and 'written'.  We have to skip them.
333 | 	 */
334 | 	ready = RBUF_OFF_MAX;
335 | 
336 | 	for (unsigned i = 0; i < rbuf->nworkers; i++) {
337 | 		ringbuf_worker_t *w = &rbuf->workers[i];
338 | 		ringbuf_off_t seen_off;
339 | 
340 | 		/*
341 | 		 * Skip if the worker has not registered.
342 | 		 *
343 | 		 * Get a stable 'seen' value.  This is necessary since we
344 | 		 * want to discard the stale 'seen' values.
345 | 		 */
346 | 		if (!atomic_load_explicit(&w->registered, memory_order_relaxed))
347 | 			continue;
348 | 		seen_off = stable_seenoff(w);
349 | 
350 | 		/*
351 | 		 * Ignore the offsets after the possible wrap-around.
352 | 		 * We are interested in the smallest seen offset that is
353 | 		 * not behind the 'written' offset.
354 | 		 */
355 | 		if (seen_off >= written) {
356 | 			ready = MIN(seen_off, ready);
357 | 		}
358 | 		ASSERT(ready >= written);
359 | 	}
360 | 
361 | 	/*
362 | 	 * Finally, we need to determine whether wrap-around occurred
363 | 	 * and deduct the safe 'ready' offset.
364 | 	 */
365 | 	if (next < written) {
366 | 		const ringbuf_off_t end = MIN(rbuf->space, rbuf->end);
367 | 
368 | 		/*
369 | 		 * Wrap-around case.  Check for the cut off first.
370 | 		 *
371 | 		 * Reset the 'written' offset if it reached the end of
372 | 		 * the buffer or the 'end' offset (if set by a producer).
373 | 		 * However, we must check that the producer is actually
374 | 		 * done (the observed 'ready' offsets are clear).
375 | 		 */
376 | 		if (ready == RBUF_OFF_MAX && written == end) {
377 | 			/*
378 | 			 * Clear the 'end' offset if was set.
379 | 			 */
380 | 			if (rbuf->end != RBUF_OFF_MAX) {
381 | 				rbuf->end = RBUF_OFF_MAX;
382 | 			}
383 | 
384 | 			/*
385 | 			 * Wrap-around the consumer and start from zero.
386 | 			 */
387 | 			written = 0;
388 | 			atomic_store_explicit(&rbuf->written,
389 | 			    written, memory_order_release);
390 | 			goto retry;
391 | 		}
392 | 
393 | 		/*
394 | 		 * We cannot wrap-around yet; there is data to consume at
395 | 		 * the end.  The ready range is smallest of the observed
396 | 		 * 'ready' or the 'end' offset.  If neither is set, then
397 | 		 * the actual end of the buffer.
398 | 		 */
399 | 		ASSERT(ready > next);
400 | 		ready = MIN(ready, end);
401 | 		ASSERT(ready >= written);
402 | 	} else {
403 | 		/*
404 | 		 * Regular case.  Up to the observed 'ready' (if set)
405 | 		 * or the 'next' offset.
406 | 		 */
407 | 		ready = MIN(ready, next);
408 | 	}
409 | 	towrite = ready - written;
410 | 	*offset = written;
411 | 
412 | 	ASSERT(ready >= written);
413 | 	ASSERT(towrite <= rbuf->space);
414 | 	return towrite;
415 | }
416 | 
417 | /*
418 |  * ringbuf_release: indicate that the consumed range can now be released.
419 |  */
420 | void
421 | ringbuf_release(ringbuf_t *rbuf, size_t nbytes)
422 | {
423 | 	const size_t nwritten = rbuf->written + nbytes;
424 | 
425 | 	ASSERT(rbuf->written <= rbuf->space);
426 | 	ASSERT(rbuf->written <= rbuf->end);
427 | 	ASSERT(nwritten <= rbuf->space);
428 | 
429 | 	rbuf->written = (nwritten == rbuf->space) ? 0 : nwritten;
430 | }
431 | 


--------------------------------------------------------------------------------
/src/ringbuf.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (c) 2016 Mindaugas Rasiukevicius <rmind at noxt eu>
 3 |  * All rights reserved.
 4 |  *
 5 |  * Use is subject to license terms, as specified in the LICENSE file.
 6 |  */
 7 | 
 8 | #ifndef _RINGBUF_H_
 9 | #define _RINGBUF_H_
10 | 
11 | __BEGIN_DECLS
12 | 
13 | typedef struct ringbuf ringbuf_t;
14 | typedef struct ringbuf_worker ringbuf_worker_t;
15 | 
16 | int		ringbuf_setup(ringbuf_t *, unsigned, size_t);
17 | void		ringbuf_get_sizes(unsigned, size_t *, size_t *);
18 | 
19 | ringbuf_worker_t *ringbuf_register(ringbuf_t *, unsigned);
20 | void		ringbuf_unregister(ringbuf_t *, ringbuf_worker_t *);
21 | 
22 | ssize_t		ringbuf_acquire(ringbuf_t *, ringbuf_worker_t *, size_t);
23 | void		ringbuf_produce(ringbuf_t *, ringbuf_worker_t *);
24 | size_t		ringbuf_consume(ringbuf_t *, size_t *);
25 | void		ringbuf_release(ringbuf_t *, size_t);
26 | 
27 | __END_DECLS
28 | 
29 | #endif
30 | 


--------------------------------------------------------------------------------
/src/t_ringbuf.c:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2017 Mindaugas Rasiukevicius <rmind at netbsd org>
  3 |  * All rights reserved.
  4 |  *
  5 |  * Use is subject to license terms, as specified in the LICENSE file.
  6 |  */
  7 | 
  8 | #include <stdio.h>
  9 | #include <stdlib.h>
 10 | #include <assert.h>
 11 | 
 12 | #include "ringbuf.h"
 13 | 
 14 | #define	MAX_WORKERS	2
 15 | 
 16 | static size_t		ringbuf_obj_size;
 17 | 
 18 | static void
 19 | test_wraparound(void)
 20 | {
 21 | 	const size_t n = 1000;
 22 | 	ringbuf_t *r = malloc(ringbuf_obj_size);
 23 | 	ringbuf_worker_t *w;
 24 | 	size_t len, woff;
 25 | 	ssize_t off;
 26 | 
 27 | 	/* Size n, but only (n - 1) can be produced at a time. */
 28 | 	ringbuf_setup(r, MAX_WORKERS, n);
 29 | 	w = ringbuf_register(r, 0);
 30 | 
 31 | 	/* Produce (n / 2 + 1) and then attempt another (n / 2 - 1). */
 32 | 	off = ringbuf_acquire(r, w, n / 2 + 1);
 33 | 	assert(off == 0);
 34 | 	ringbuf_produce(r, w);
 35 | 
 36 | 	off = ringbuf_acquire(r, w, n / 2 - 1);
 37 | 	assert(off == -1);
 38 | 
 39 | 	/* Consume (n / 2 + 1) bytes. */
 40 | 	len = ringbuf_consume(r, &woff);
 41 | 	assert(len == (n / 2 + 1) && woff == 0);
 42 | 	ringbuf_release(r, len);
 43 | 
 44 | 	/* All consumed, attempt (n / 2 + 1) now. */
 45 | 	off = ringbuf_acquire(r, w, n / 2 + 1);
 46 | 	assert(off == -1);
 47 | 
 48 | 	/* However, wraparound can be successful with (n / 2). */
 49 | 	off = ringbuf_acquire(r, w, n / 2);
 50 | 	assert(off == 0);
 51 | 	ringbuf_produce(r, w);
 52 | 
 53 | 	/* Consume (n / 2) bytes. */
 54 | 	len = ringbuf_consume(r, &woff);
 55 | 	assert(len == (n / 2) && woff == 0);
 56 | 	ringbuf_release(r, len);
 57 | 
 58 | 	ringbuf_unregister(r, w);
 59 | 	free(r);
 60 | }
 61 | 
 62 | static void
 63 | test_multi(void)
 64 | {
 65 | 	ringbuf_t *r = malloc(ringbuf_obj_size);
 66 | 	ringbuf_worker_t *w;
 67 | 	size_t len, woff;
 68 | 	ssize_t off;
 69 | 
 70 | 	ringbuf_setup(r, MAX_WORKERS, 3);
 71 | 	w = ringbuf_register(r, 0);
 72 | 
 73 | 	/*
 74 | 	 * Produce 2 bytes.
 75 | 	 */
 76 | 
 77 | 	off = ringbuf_acquire(r, w, 1);
 78 | 	assert(off == 0);
 79 | 	ringbuf_produce(r, w);
 80 | 
 81 | 	off = ringbuf_acquire(r, w, 1);
 82 | 	assert(off == 1);
 83 | 	ringbuf_produce(r, w);
 84 | 
 85 | 	off = ringbuf_acquire(r, w, 1);
 86 | 	assert(off == -1);
 87 | 
 88 | 	/*
 89 | 	 * Consume 2 bytes.
 90 | 	 */
 91 | 	len = ringbuf_consume(r, &woff);
 92 | 	assert(len == 2 && woff == 0);
 93 | 	ringbuf_release(r, len);
 94 | 
 95 | 	len = ringbuf_consume(r, &woff);
 96 | 	assert(len == 0);
 97 | 
 98 | 	/*
 99 | 	 * Produce another 2 with wrap-around.
100 | 	 */
101 | 
102 | 	off = ringbuf_acquire(r, w, 2);
103 | 	assert(off == -1);
104 | 
105 | 	off = ringbuf_acquire(r, w, 1);
106 | 	assert(off == 2);
107 | 	ringbuf_produce(r, w);
108 | 
109 | 	off = ringbuf_acquire(r, w, 1);
110 | 	assert(off == 0);
111 | 	ringbuf_produce(r, w);
112 | 
113 | 	off = ringbuf_acquire(r, w, 1);
114 | 	assert(off == -1);
115 | 
116 | 	/*
117 | 	 * Consume 1 byte at the end and 1 byte at the beginning.
118 | 	 */
119 | 
120 | 	len = ringbuf_consume(r, &woff);
121 | 	assert(len == 1 && woff == 2);
122 | 	ringbuf_release(r, len);
123 | 
124 | 	len = ringbuf_consume(r, &woff);
125 | 	assert(len == 1 && woff == 0);
126 | 	ringbuf_release(r, len);
127 | 
128 | 	ringbuf_unregister(r, w);
129 | 	free(r);
130 | }
131 | 
132 | static void
133 | test_overlap(void)
134 | {
135 | 	ringbuf_t *r = malloc(ringbuf_obj_size);
136 | 	ringbuf_worker_t *w1, *w2;
137 | 	size_t len, woff;
138 | 	ssize_t off;
139 | 
140 | 	ringbuf_setup(r, MAX_WORKERS, 10);
141 | 	w1 = ringbuf_register(r, 0);
142 | 	w2 = ringbuf_register(r, 1);
143 | 
144 | 	/*
145 | 	 * Producer 1: acquire 5 bytes.  Consumer should fail.
146 | 	 */
147 | 	off = ringbuf_acquire(r, w1, 5);
148 | 	assert(off == 0);
149 | 
150 | 	len = ringbuf_consume(r, &woff);
151 | 	assert(len == 0);
152 | 
153 | 	/*
154 | 	 * Producer 2: acquire 3 bytes.  Consumer should still fail.
155 | 	 */
156 | 	off = ringbuf_acquire(r, w2, 3);
157 | 	assert(off == 5);
158 | 
159 | 	len = ringbuf_consume(r, &woff);
160 | 	assert(len == 0);
161 | 
162 | 	/*
163 | 	 * Producer 1: commit.  Consumer can get the first range.
164 | 	 */
165 | 	ringbuf_produce(r, w1);
166 | 	len = ringbuf_consume(r, &woff);
167 | 	assert(len == 5 && woff == 0);
168 | 	ringbuf_release(r, len);
169 | 
170 | 	len = ringbuf_consume(r, &woff);
171 | 	assert(len == 0);
172 | 
173 | 	/*
174 | 	 * Producer 1: acquire-produce 4 bytes, triggering wrap-around.
175 | 	 * Consumer should still fail.
176 | 	 */
177 | 	off = ringbuf_acquire(r, w1, 4);
178 | 	assert(off == 0);
179 | 
180 | 	len = ringbuf_consume(r, &woff);
181 | 	assert(len == 0);
182 | 
183 | 	ringbuf_produce(r, w1);
184 | 	len = ringbuf_consume(r, &woff);
185 | 	assert(len == 0);
186 | 
187 | 	/*
188 | 	 * Finally, producer 2 commits its 3 bytes.
189 | 	 * Consumer can proceed for both ranges.
190 | 	 */
191 | 	ringbuf_produce(r, w2);
192 | 	len = ringbuf_consume(r, &woff);
193 | 	assert(len == 3 && woff == 5);
194 | 	ringbuf_release(r, len);
195 | 
196 | 	len = ringbuf_consume(r, &woff);
197 | 	assert(len == 4 && woff == 0);
198 | 	ringbuf_release(r, len);
199 | 
200 | 	ringbuf_unregister(r, w1);
201 | 	ringbuf_unregister(r, w2);
202 | 	free(r);
203 | }
204 | 
205 | static void
206 | test_random(void)
207 | {
208 | 	ringbuf_t *r = malloc(ringbuf_obj_size);
209 | 	ringbuf_worker_t *w1, *w2;
210 | 	ssize_t off1 = -1, off2 = -1;
211 | 	unsigned n = 1000 * 1000 * 50;
212 | 	unsigned char buf[500];
213 | 
214 | 	ringbuf_setup(r, MAX_WORKERS, sizeof(buf));
215 | 	w1 = ringbuf_register(r, 0);
216 | 	w2 = ringbuf_register(r, 1);
217 | 
218 | 	while (n--) {
219 | 		size_t len, woff;
220 | 
221 | 		len = random() % (sizeof(buf) / 2) + 1;
222 | 		switch (random() % 3) {
223 | 		case 0:	// consumer
224 | 			len = ringbuf_consume(r, &woff);
225 | 			if (len > 0) {
226 | 				size_t vlen = 0;
227 | 				assert(woff < sizeof(buf));
228 | 				while (vlen < len) {
229 | 					size_t mlen = (unsigned)buf[woff];
230 | 					assert(mlen > 0);
231 | 					vlen += mlen;
232 | 					woff += mlen;
233 | 				}
234 | 				assert(vlen == len);
235 | 				ringbuf_release(r, len);
236 | 			}
237 | 			break;
238 | 		case 1:	// producer 1
239 | 			if (off1 == -1) {
240 | 				if ((off1 = ringbuf_acquire(r, w1, len)) >= 0) {
241 | 					assert((size_t)off1 < sizeof(buf));
242 | 					buf[off1] = len - 1;
243 | 				}
244 | 			} else {
245 | 				buf[off1]++;
246 | 				ringbuf_produce(r, w1);
247 | 				off1 = -1;
248 | 			}
249 | 			break;
250 | 		case 2:	// producer 2
251 | 			if (off2 == -1) {
252 | 				if ((off2 = ringbuf_acquire(r, w2, len)) >= 0) {
253 | 					assert((size_t)off2 < sizeof(buf));
254 | 					buf[off2] = len - 1;
255 | 				}
256 | 			} else {
257 | 				buf[off2]++;
258 | 				ringbuf_produce(r, w2);
259 | 				off2 = -1;
260 | 			}
261 | 			break;
262 | 		}
263 | 	}
264 | 	ringbuf_unregister(r, w1);
265 | 	ringbuf_unregister(r, w2);
266 | 	free(r);
267 | }
268 | 
269 | int
270 | main(void)
271 | {
272 | 	ringbuf_get_sizes(MAX_WORKERS, &ringbuf_obj_size, NULL);
273 | 	test_wraparound();
274 | 	test_multi();
275 | 	test_overlap();
276 | 	test_random();
277 | 	puts("ok");
278 | 	return 0;
279 | }
280 | 


--------------------------------------------------------------------------------
/src/t_stress.c:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2017 Mindaugas Rasiukevicius <rmind at netbsd org>
  3 |  * All rights reserved.
  4 |  *
  5 |  * Use is subject to license terms, as specified in the LICENSE file.
  6 |  */
  7 | 
  8 | #include <stdio.h>
  9 | #include <stdlib.h>
 10 | #include <stdbool.h>
 11 | #include <inttypes.h>
 12 | #include <string.h>
 13 | #include <unistd.h>
 14 | #include <limits.h>
 15 | #include <signal.h>
 16 | #include <pthread.h>
 17 | #include <assert.h>
 18 | #include <errno.h>
 19 | #include <err.h>
 20 | 
 21 | #include "ringbuf.h"
 22 | #include "utils.h"
 23 | 
 24 | static unsigned			nsec = 10; /* seconds */
 25 | 
 26 | static pthread_barrier_t	barrier;
 27 | static unsigned			nworkers;
 28 | static volatile bool		stop;
 29 | 
 30 | static ringbuf_t *		ringbuf;
 31 | static size_t			ringbuf_obj_size;
 32 | __thread uint32_t		fast_random_seed = 5381;
 33 | 
 34 | #define	RBUF_SIZE		(512)
 35 | #define	MAGIC_BYTE		(0x5a)
 36 | 
 37 | /* Note: leave one byte for the magic byte. */
 38 | static uint8_t			rbuf[RBUF_SIZE + 1];
 39 | 
 40 | /*
 41 |  * Simple xorshift; random() causes huge lock contention on Linux/glibc,
 42 |  * which would "hide" the possible race conditions.
 43 |  */
 44 | static unsigned long
 45 | fast_random(void)
 46 | {
 47 | 	uint32_t x = fast_random_seed;
 48 | 	x ^= x << 13;
 49 | 	x ^= x >> 17;
 50 | 	x ^= x << 5;
 51 | 	fast_random_seed = x;
 52 | 	return x;
 53 | }
 54 | 
 55 | /*
 56 |  * Generate a random message of a random length (up to the given size)
 57 |  * and simple XOR based checksum.  The first byte is reserved for the
 58 |  * message length and the last byte is reserved for a checksum.
 59 |  */
 60 | static size_t
 61 | generate_message(unsigned char *buf, size_t buflen)
 62 | {
 63 | 	const unsigned len = fast_random() % (buflen - 2);
 64 | 	unsigned i = 1, n = len;
 65 | 	unsigned char cksum = 0;
 66 | 
 67 | 	while (n--) {
 68 | 		buf[i] = '!' + (fast_random() % ('~' - '!'));
 69 | 		cksum ^= buf[i];
 70 | 		i++;
 71 | 	}
 72 | 	/*
 73 | 	 * Write the length and checksum last, trying to exploit a
 74 | 	 * possibility of a race condition.  NOTE: depending on an
 75 | 	 * architecture, might want to try a memory barrier here.
 76 | 	 */
 77 | 	buf[i++] = cksum;
 78 | 	buf[0] = len;
 79 | 	return i;
 80 | }
 81 | 
 82 | /*
 83 |  * Take an arbitrary message of a variable length and verify its checksum.
 84 |  */
 85 | static ssize_t
 86 | verify_message(const unsigned char *buf)
 87 | {
 88 | 	unsigned i = 1, len = (unsigned char)buf[0];
 89 | 	unsigned char cksum = 0;
 90 | 
 91 | 	while (len--) {
 92 | 		cksum ^= buf[i++];
 93 | 	}
 94 | 	if (buf[i] != cksum) {
 95 | 		return -1;
 96 | 	}
 97 | 	return (unsigned)buf[0] + 2;
 98 | }
 99 | 
100 | static void *
101 | ringbuf_stress(void *arg)
102 | {
103 | 	const unsigned id = (uintptr_t)arg;
104 | 	ringbuf_worker_t *w;
105 | 
106 | 	w = ringbuf_register(ringbuf, id);
107 | 	assert(w != NULL);
108 | 
109 | 	/*
110 | 	 * There are NCPU threads concurrently generating and producing
111 | 	 * random messages and a single consumer thread (ID 0) verifying
112 | 	 * and releasing the messages.
113 | 	 */
114 | 
115 | 	pthread_barrier_wait(&barrier);
116 | 	while (!stop) {
117 | 		unsigned char buf[MIN((1 << CHAR_BIT), RBUF_SIZE)];
118 | 		size_t len, off;
119 | 		ssize_t ret;
120 | 
121 | 		/* Check that the buffer is never overrun. */
122 | 		assert(rbuf[RBUF_SIZE] == MAGIC_BYTE);
123 | 
124 | 		if (id == 0) {
125 | 			if ((len = ringbuf_consume(ringbuf, &off)) != 0) {
126 | 				size_t rem = len;
127 | 				assert(off < RBUF_SIZE);
128 | 				while (rem) {
129 | 					ret = verify_message(&rbuf[off]);
130 | 					assert(ret > 0);
131 | 					assert(ret <= (ssize_t)rem);
132 | 					off += ret, rem -= ret;
133 | 				}
134 | 				ringbuf_release(ringbuf, len);
135 | 			}
136 | 			continue;
137 | 		}
138 | 		len = generate_message(buf, sizeof(buf) - 1);
139 | 		if ((ret = ringbuf_acquire(ringbuf, w, len)) != -1) {
140 | 			off = (size_t)ret;
141 | 			assert(off < RBUF_SIZE);
142 | 			memcpy(&rbuf[off], buf, len);
143 | 			ringbuf_produce(ringbuf, w);
144 | 		}
145 | 	}
146 | 	pthread_barrier_wait(&barrier);
147 | 	pthread_exit(NULL);
148 | 	return NULL;
149 | }
150 | 
151 | static void
152 | ding(int sig)
153 | {
154 | 	(void)sig;
155 | 	stop = true;
156 | }
157 | 
158 | static void
159 | run_test(void *func(void *))
160 | {
161 | 	struct sigaction sigalarm;
162 | 	pthread_t *thr;
163 | 	int ret;
164 | 
165 | 	/*
166 | 	 * Setup the threads.
167 | 	 */
168 | 	nworkers = sysconf(_SC_NPROCESSORS_CONF) + 1;
169 | 	thr = calloc(nworkers, sizeof(pthread_t));
170 | 	pthread_barrier_init(&barrier, NULL, nworkers);
171 | 	stop = false;
172 | 
173 | 	memset(&sigalarm, 0, sizeof(struct sigaction));
174 | 	sigalarm.sa_handler = ding;
175 | 	ret = sigaction(SIGALRM, &sigalarm, NULL);
176 | 	assert(ret == 0); (void)ret;
177 | 
178 | 	/*
179 | 	 * Create a ring buffer.
180 | 	 */
181 | 	ringbuf_get_sizes(nworkers, &ringbuf_obj_size, NULL);
182 | 	ringbuf = malloc(ringbuf_obj_size);
183 | 	assert(ringbuf != NULL);
184 | 
185 | 	ringbuf_setup(ringbuf, nworkers, RBUF_SIZE);
186 | 	memset(rbuf, MAGIC_BYTE, sizeof(rbuf));
187 | 
188 | 	/*
189 | 	 * Spin the test.
190 | 	 */
191 | 	alarm(nsec);
192 | 
193 | 	for (unsigned i = 0; i < nworkers; i++) {
194 | 		if ((errno = pthread_create(&thr[i], NULL,
195 | 		    func, (void *)(uintptr_t)i)) != 0) {
196 | 			err(EXIT_FAILURE, "pthread_create");
197 | 		}
198 | 	}
199 | 	for (unsigned i = 0; i < nworkers; i++) {
200 | 		pthread_join(thr[i], NULL);
201 | 	}
202 | 	pthread_barrier_destroy(&barrier);
203 | 	free(ringbuf);
204 | 	free(thr);
205 | }
206 | 
207 | int
208 | main(int argc, char **argv)
209 | {
210 | 	if (argc >= 2) {
211 | 		nsec = (unsigned)atoi(argv[1]);
212 | 	}
213 | 	puts("stress test");
214 | 	run_test(ringbuf_stress);
215 | 	puts("ok");
216 | 	return 0;
217 | }
218 | 


--------------------------------------------------------------------------------
/src/utils.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 1991, 1993
  3 |  *	The Regents of the University of California.  All rights reserved.
  4 |  *
  5 |  * This code is derived from software contributed to Berkeley by
  6 |  * Berkeley Software Design, Inc.
  7 |  *
  8 |  * Redistribution and use in source and binary forms, with or without
  9 |  * modification, are permitted provided that the following conditions
 10 |  * are met:
 11 |  * 1. Redistributions of source code must retain the above copyright
 12 |  *    notice, this list of conditions and the following disclaimer.
 13 |  * 2. Redistributions in binary form must reproduce the above copyright
 14 |  *    notice, this list of conditions and the following disclaimer in the
 15 |  *    documentation and/or other materials provided with the distribution.
 16 |  * 3. Neither the name of the University nor the names of its contributors
 17 |  *    may be used to endorse or promote products derived from this software
 18 |  *    without specific prior written permission.
 19 |  *
 20 |  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 21 |  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 22 |  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 23 |  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 24 |  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 25 |  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 26 |  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 27 |  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 28 |  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 29 |  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 30 |  * SUCH DAMAGE.
 31 |  *
 32 |  *	@(#)cdefs.h	8.8 (Berkeley) 1/9/95
 33 |  */
 34 | 
 35 | #ifndef _UTILS_H_
 36 | #define _UTILS_H_
 37 | 
 38 | #include <assert.h>
 39 | 
 40 | /*
 41 |  * A regular assert (debug/diagnostic only).
 42 |  */
 43 | #if defined(DEBUG)
 44 | #define	ASSERT		assert
 45 | #else
 46 | #define	ASSERT(x)
 47 | #endif
 48 | 
 49 | /*
 50 |  * Minimum, maximum and rounding macros.
 51 |  */
 52 | 
 53 | #ifndef MIN
 54 | #define	MIN(x, y)	((x) < (y) ? (x) : (y))
 55 | #endif
 56 | 
 57 | #ifndef MAX
 58 | #define	MAX(x, y)	((x) > (y) ? (x) : (y))
 59 | #endif
 60 | 
 61 | /*
 62 |  * Branch prediction macros.
 63 |  */
 64 | #ifndef __predict_true
 65 | #define	__predict_true(x)	__builtin_expect((x) != 0, 1)
 66 | #define	__predict_false(x)	__builtin_expect((x) != 0, 0)
 67 | #endif
 68 | 
 69 | /*
 70 |  * Atomic operations and memory barriers.  If C11 API is not available,
 71 |  * then wrap the GCC builtin routines.
 72 |  *
 73 |  * Note: This atomic_compare_exchange_weak does not do the C11 thing of
 74 |  * filling *(expected) with the actual value, because we don't need
 75 |  * that here.
 76 |  */
 77 | #ifndef atomic_compare_exchange_weak
 78 | #define	atomic_compare_exchange_weak(ptr, expected, desired) \
 79 |     __sync_bool_compare_and_swap(ptr, *(expected), desired)
 80 | #endif
 81 | 
 82 | #ifndef atomic_thread_fence
 83 | #define	memory_order_relaxed	__ATOMIC_RELAXED
 84 | #define	memory_order_acquire	__ATOMIC_ACQUIRE
 85 | #define	memory_order_release	__ATOMIC_RELEASE
 86 | #define	memory_order_seq_cst	__ATOMIC_SEQ_CST
 87 | #define	atomic_thread_fence(m)	__atomic_thread_fence(m)
 88 | #endif
 89 | #ifndef atomic_store_explicit
 90 | #define	atomic_store_explicit	__atomic_store_n
 91 | #endif
 92 | #ifndef atomic_load_explicit
 93 | #define	atomic_load_explicit	__atomic_load_n
 94 | #endif
 95 | 
 96 | /*
 97 |  * Exponential back-off for the spinning paths.
 98 |  */
 99 | #define	SPINLOCK_BACKOFF_MIN	4
100 | #define	SPINLOCK_BACKOFF_MAX	128
101 | #if defined(__x86_64__) || defined(__i386__)
102 | #define SPINLOCK_BACKOFF_HOOK	__asm volatile("pause" ::: "memory")
103 | #else
104 | #define SPINLOCK_BACKOFF_HOOK
105 | #endif
106 | #define	SPINLOCK_BACKOFF(count)					\
107 | do {								\
108 | 	for (int __i = (count); __i != 0; __i--) {		\
109 | 		SPINLOCK_BACKOFF_HOOK;				\
110 | 	}							\
111 | 	if ((count) < SPINLOCK_BACKOFF_MAX)			\
112 | 		(count) += (count);				\
113 | } while (/* CONSTCOND */ 0);
114 | 
115 | #endif
116 | 


--------------------------------------------------------------------------------