├── CustomIndexType
    └── LockFreeMPMCQueue.h
├── LICENSE
├── MPMCQueue.h
├── README.md
└── Tests
    ├── Main.cpp
    └── MutexQueue.h


/CustomIndexType/LockFreeMPMCQueue.h:
--------------------------------------------------------------------------------
  1 | // THIS FILE IS NOT MAINTAINED AND MAY NOT CONTAIN ALL FIXES IN ../LockFreeMPMPCQueue.h
  2 | 
  3 | #pragma once
  4 | 
  5 | #include <atomic>
  6 | 
  7 | template <typename T, typename index_t = size_t> class LockFreeMPMCQueue
  8 | {
  9 |     public:
 10 | 	explicit LockFreeMPMCQueue( size_t size )
 11 | 	    : m_data( new T[size] ), m_size( size ), m_head_1( 0 ), m_head_2( 0 ), m_tail_1( 0 ), m_tail_2( 0 )
 12 | 	{
 13 | 	}
 14 | 
 15 | 	virtual ~LockFreeMPMCQueue() { delete[] m_data; }
 16 | 
 17 | 	bool try_enqueue( const T& value )
 18 | 	{
 19 | 		index_t tail = m_tail_1.load( std::memory_order_relaxed );
 20 | 		const index_t head = m_head_2.load( std::memory_order_relaxed );
 21 | 
 22 | 		const index_t count =
 23 | 		    tail > head ? tail - head : ( std::numeric_limits<index_t>::max() - head ) + tail + 1;
 24 | 
 25 | 		// count could be greater than size if between the reading of head, and the reading of tail, both head
 26 | 		// and tail have been advanced
 27 | 		if( count >= m_size )
 28 | 		{
 29 | 			return false;
 30 | 		}
 31 | 
 32 | 		const index_t next_tail = tail < std::numeric_limits<index_t>::max() ? tail + 1 : 0;
 33 | 
 34 | 		if( !std::atomic_compare_exchange_strong_explicit(
 35 | 			&m_tail_1, &tail, next_tail, std::memory_order_relaxed, std::memory_order_relaxed ) )
 36 | 		{
 37 | 			return false;
 38 | 		}
 39 | 
 40 | 		m_data[tail % m_size] = value;
 41 | 
 42 | 		while( m_tail_2.load( std::memory_order_relaxed ) != tail )
 43 | 		{
 44 | 			std::this_thread::yield();
 45 | 		}
 46 | 
 47 | 		// Release - read/write before can't be reordered with writes after
 48 | 		// Make sure the write of the value to m_data is
 49 | 		// not reordered past the write to m_tail_2
 50 | 		std::atomic_thread_fence( std::memory_order_release );
 51 | 		m_tail_2.store( tail + 1, std::memory_order_relaxed );
 52 | 
 53 | 		return true;
 54 | 	}
 55 | 
 56 | 	bool try_dequeue( T& out )
 57 | 	{
 58 | 		// Order matters here, because we're testing for emptiness with head==tail
 59 | 		// Could test with head >= tail, but if index_t is < 64bits then this will have
 60 | 		// issues when tail has wrapped around to 0
 61 | 		index_t head = m_head_1.load( std::memory_order_relaxed );
 62 | 		// Acquire - read before can't be reordered with read/write after
 63 | 		std::atomic_thread_fence( std::memory_order_acquire );
 64 | 		const index_t tail = m_tail_2.load( std::memory_order_relaxed );
 65 | 
 66 | 		if( head == tail )
 67 | 		{
 68 | 			return false;
 69 | 		}
 70 | 
 71 | 		const index_t next_head = head < std::numeric_limits<index_t>::max() ? head + 1 : 0;
 72 | 
 73 | 		if( !std::atomic_compare_exchange_strong_explicit(
 74 | 			&m_head_1, &head, next_head, std::memory_order_relaxed, std::memory_order_relaxed ) )
 75 | 		{
 76 | 			return false;
 77 | 		}
 78 | 
 79 | 		std::atomic_thread_fence( std::memory_order_acquire );
 80 | 		out = m_data[head % m_size];
 81 | 
 82 | 		while( m_head_2.load( std::memory_order_relaxed ) != head )
 83 | 		{
 84 | 			std::this_thread::yield();
 85 | 		}
 86 | 
 87 | 		// Release - read/write before can't be reordered with writes after
 88 | 		// Make sure the read of value from m_data is
 89 | 		// not reordered past the write to m_head_2
 90 | 		std::atomic_thread_fence( std::memory_order_release );
 91 | 		m_head_2.store( head + 1, std::memory_order_relaxed );
 92 | 
 93 | 		return true;
 94 | 	}
 95 | 
 96 | 	size_t capacity() const { return m_size; }
 97 | 
 98 |     private:
 99 | 	T* m_data;
100 | 	size_t m_size;
101 | 	char pad1[64];
102 | 	std::atomic<index_t> m_head_1;
103 | 	char pad2[64];
104 | 	std::atomic<index_t> m_head_2;
105 | 	char pad3[64];
106 | 	std::atomic<index_t> m_tail_1;
107 | 	char pad4[64];
108 | 	std::atomic<index_t> m_tail_2;
109 | };
110 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2018 Joe Best-Rotheray
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/MPMCQueue.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <atomic>
 4 | #include <cstdint>
 5 | 
 6 | template <typename T, size_t cache_line_size = 64> class MPMCQueue
 7 | {
 8 | public:
 9 | 	explicit MPMCQueue( size_t capacity )
10 | 		: m_items( static_cast<Item*>( _aligned_malloc( sizeof( Item ) * capacity, cache_line_size ) ) )
11 | 		, m_capacity( capacity )
12 | 		, m_head( 0 )
13 | 		, m_tail( 0 )
14 | 	{
15 | 		for( size_t i = 0; i < capacity; ++i )
16 | 		{
17 | 			m_items[i].version = i;
18 | 		}
19 | 	}
20 | 
21 | 	virtual ~MPMCQueue() { _aligned_free( m_items ); }
22 | 
23 | 	// non-copyable
24 | 	MPMCQueue( const MPMCQueue<T>& ) = delete;
25 | 	MPMCQueue( const MPMCQueue<T>&& ) = delete;
26 | 	MPMCQueue<T>& operator=( const MPMCQueue<T>& ) = delete;
27 | 	MPMCQueue<T>& operator=( const MPMCQueue<T>&& ) = delete;
28 | 
29 | 	bool try_enqueue( const T& value )
30 | 	{
31 | 		uint64_t tail = m_tail.load( std::memory_order_relaxed );
32 | 
33 | 		if( m_items[tail % m_capacity].version.load( std::memory_order_acquire ) != tail )
34 | 		{
35 | 			return false;
36 | 		}
37 | 
38 | 		if( !m_tail.compare_exchange_strong( tail, tail + 1, std::memory_order_relaxed ) )
39 | 		{
40 | 			return false;
41 | 		}
42 | 
43 | 		m_items[tail % m_capacity].value = value;
44 | 
45 | 		// Release operation, all reads/writes before this store cannot be reordered past it
46 | 		// Writing version to tail + 1 signals reader threads when to read payload
47 | 		m_items[tail % m_capacity].version.store( tail + 1, std::memory_order_release );
48 | 
49 | 		return true;
50 | 	}
51 | 
52 | 	bool try_dequeue( T& out )
53 | 	{
54 | 		uint64_t head = m_head.load( std::memory_order_relaxed );
55 | 
56 | 		// Acquire here makes sure read of m_data[head].value is not reordered before this
57 | 		// Also makes sure side effects in try_enqueue are visible here
58 | 		if( m_items[head % m_capacity].version.load( std::memory_order_acquire ) != (head + 1) )
59 | 		{
60 | 			return false;
61 | 		}
62 | 
63 | 		if( !m_head.compare_exchange_strong( head, head + 1, std::memory_order_relaxed ) )
64 | 		{
65 | 			return false;
66 | 		}
67 | 
68 | 		out = m_items[head % m_capacity].value;
69 | 
70 | 		// This signals to writer threads that they can now write something to this index
71 | 		m_items[head % m_capacity].version.store( head + m_capacity, std::memory_order_release );
72 | 
73 | 		return true;
74 | 	}
75 | 
76 | 	size_t capacity() const { return m_capacity; }
77 | 
78 | private:
79 | 	struct alignas(cache_line_size)Item
80 | 	{
81 | 		std::atomic<uint64_t> version;
82 | 		T value;
83 | 	};
84 | 
85 | 	struct alignas(cache_line_size)AlignedAtomicU64 : public std::atomic<uint64_t>
86 | 	{
87 | 		using std::atomic<uint64_t>::atomic;
88 | 	};
89 | 	
90 | 	Item* m_items;
91 | 	size_t m_capacity;
92 | 
93 | 	// Make sure each index is on a different cache line
94 | 	AlignedAtomicU64 m_head;
95 | 	AlignedAtomicU64 m_tail;
96 | };
97 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # A fast (not quite lock-free) multi-producer multi-consumer queue
2 | 
3 | ## The only file needed is MPMCQueue.h
4 | 
5 | **CustomIndexType/** - some old code no longer maintained, but shows a strategy for dealing with index overflow if 64-bit CAS operations are not lock-free on a given platform
6 | 
7 | **Tests/** - some slightly unsightly code to compare this queue against a naive mutex implementation, and spits out a google chart to show results
8 | 


--------------------------------------------------------------------------------
/Tests/Main.cpp:
--------------------------------------------------------------------------------
  1 | #include <chrono>
  2 | #include <cstdint>
  3 | #include <thread>
  4 | #include <vector>
  5 | 
  6 | #include "../MPMCQueue.h"
  7 | #include "MutexQueue.h"
  8 | 
  9 | // this is a horrible mess :(
 10 | 
 11 | template <template <typename, size_t> class Queue, typename Value, size_t cache_line_size>
 12 | std::chrono::milliseconds::rep test( const size_t num_threads, char* memory, const size_t num_values,
 13 | 				     Queue<Value, cache_line_size>& queue )
 14 | {
 15 | 	memset( memory, 0, sizeof( char ) * num_values );
 16 | 
 17 | 	const size_t num_values_per_thread = num_values / num_threads;
 18 | 
 19 | 	std::thread* reader_threads = new std::thread[num_threads];
 20 | 	std::thread* writer_threads = new std::thread[num_threads];
 21 | 
 22 | 	auto start = std::chrono::high_resolution_clock::now();
 23 | 
 24 | 	for( size_t i = 0; i < num_threads; ++i )
 25 | 	{
 26 | 		reader_threads[i] = std::thread( [i, &queue, memory, num_values_per_thread]() {
 27 | 			for( size_t x = 0; x < num_values_per_thread; ++x )
 28 | 			{
 29 | 				Value value;
 30 | 				while( !queue.try_dequeue( value ) )
 31 | 				{
 32 | 					std::this_thread::yield();
 33 | 				}
 34 | 				memory[value] = 1;
 35 | 			}
 36 | 		} );
 37 | 	}
 38 | 
 39 | 	for( size_t i = 0; i < num_threads; ++i )
 40 | 	{
 41 | 		writer_threads[i] = std::thread( [i, &queue, num_values_per_thread]() {
 42 | 			const size_t offset = i * num_values_per_thread;
 43 | 			for( size_t x = 0; x < num_values_per_thread; ++x )
 44 | 			{
 45 | 				const size_t value = offset + x;
 46 | 				while( !queue.try_enqueue( value ) )
 47 | 				{
 48 | 					std::this_thread::yield();
 49 | 				}
 50 | 			}
 51 | 		} );
 52 | 	}
 53 | 
 54 | 	for( size_t i = 0; i < num_threads; ++i )
 55 | 	{
 56 | 		reader_threads[i].join();
 57 | 		writer_threads[i].join();
 58 | 	}
 59 | 
 60 | 	auto time_taken = std::chrono::high_resolution_clock::now() - start;
 61 | 
 62 | 	delete[] reader_threads;
 63 | 	delete[] writer_threads;
 64 | 
 65 | 	bool fail = false;
 66 | 	for( size_t i = 0; i < num_values; ++i )
 67 | 	{
 68 | 		if( memory[i] == 0 )
 69 | 		{
 70 | 			printf( "%zu = 0\n", i );
 71 | 			fail = true;
 72 | 		}
 73 | 	}
 74 | 
 75 | 	if( fail )
 76 | 	{
 77 | 		printf( "FAIL!\n" );
 78 | 	}
 79 | 
 80 | 	return std::chrono::duration_cast<std::chrono::milliseconds>( time_taken ).count();
 81 | }
 82 | 
 83 | template <class Queue>
 84 | std::vector<double> test_batch( const size_t num_threads_max, const size_t num_values, const size_t queue_size,
 85 | 				const size_t num_samples, char* memory )
 86 | {
 87 | 	std::vector<double> results;
 88 | 	Queue queue( queue_size );
 89 | 	double inv_num_samples = 1.0 / double( num_samples );
 90 | 
 91 | 	for( size_t num_threads = 1; num_threads <= num_threads_max; num_threads *= 2 )
 92 | 	{
 93 | 		double avg_time_taken = 0.0;
 94 | 
 95 | 		for( size_t i = 0; i < num_samples; ++i )
 96 | 		{
 97 | 			avg_time_taken += test( num_threads, memory, num_values, queue ) * inv_num_samples;
 98 | 		}
 99 | 
100 | 		results.push_back( avg_time_taken );
101 | 	}
102 | 
103 | 	return results;
104 | }
105 | 
106 | void benchmark_comparison( void )
107 | {
108 | 	const size_t num_threads_max = 32;
109 | 	const size_t num_values = 1 << 13;
110 | 	const size_t queue_size = 128;
111 | 	const size_t num_samples = 64;
112 | 
113 | 	std::vector<double> results[2];
114 | 
115 | 	char* memory = new char[num_values];
116 | 
117 | 	results[0] =
118 | 	    test_batch<MPMCQueue<size_t>>( num_threads_max, num_values, queue_size, num_samples, memory );
119 | 	results[1] = test_batch<MutexQueue<size_t>>( num_threads_max, num_values, queue_size, num_samples, memory );
120 | 
121 | 	printf( "<html>\n" );
122 | 	printf( "<head>\n" );
123 | 	printf( "<!--Load the AJAX API-->\n" );
124 | 	printf( "<script type=\"text/javascript\" src=\"https://www.gstatic.com/charts/loader.js\"></script>\n" );
125 | 	printf( "<script type=\"text/javascript\">\n\n" );
126 | 	printf( "google.charts.load('current', {'packages':['corechart', 'line']});\n\n" );
127 | 	printf( "google.charts.setOnLoadCallback(drawChart);\n\n" );
128 | 	printf( "function drawChart() {\n" );
129 | 	printf( "var data = new google.visualization.DataTable();\n" );
130 | 	printf( "data.addColumn('number', 'X');\n" );
131 | 	printf( "data.addColumn('number', 'MPMCQueue');\n" );
132 | 	printf( "data.addColumn('number', 'MutexQueue');\n" );
133 | 	printf( "data.addRows([\n" );
134 | 
135 | 	for( size_t i = 0, num_threads = 1; i < results[0].size(); ++i, num_threads *= 2 )
136 | 	{
137 | 		printf( "[%zu,%f,%f]", num_threads, results[0][i], results[1][i] );
138 | 		if( i < ( results[0].size() - 1 ) )
139 | 		{
140 | 			printf( "," );
141 | 		}
142 | 		printf( "\n" );
143 | 	}
144 | 
145 | 	printf( "]);\n" );
146 | 	printf( "var options = {\n" );
147 | 	printf( "hAxis: {title: 'Threads'},\n" );
148 | 	printf( "vAxis: {title: 'Avg Time Taken (milliseconds)'},\n" );
149 | 	printf( "series: {1: {curveType: 'function'}},\n" );
150 | 	printf( "height: 800,\n" );
151 | 	printf( "pointsVisible: true\n" );
152 | 	printf( "};\n" );
153 | 	printf( "var chart = new google.visualization.LineChart(document.getElementById('chart_div'));\n" );
154 | 	printf( "chart.draw(data, options);\n" );
155 | 	printf( "}\n" );
156 | 	printf( "</script>\n" );
157 | 	printf( "</head>\n" );
158 | 	printf( "<body>\n" );
159 | 	printf( "<div id=\"chart_div\"></div>\n" );
160 | 	printf( "</body>\n" );
161 | 	printf( "</html>\n" );
162 | 
163 | 	delete[] memory;
164 | }
165 | 
166 | void stress_test( void )
167 | {
168 | 	const size_t queue_size = 128;
169 | 	const size_t num_threads = 2;
170 | 	const size_t num_values = 1024 * 16;
171 | 	const size_t num_tests = 1024 * 16;
172 | 
173 | 	char* memory = new char[num_values];
174 | 
175 | 	MPMCQueue<size_t> queue( queue_size );
176 | 
177 | 	size_t stage = 0; // give progress update every 10%
178 | 	for( int i = 0; i < num_tests; ++i )
179 | 	{
180 | 		test( num_threads, memory, num_values, queue );
181 | 
182 | 		size_t new_stage = ( ( i + 1 ) * 10 ) / num_tests;
183 | 		if( stage != new_stage )
184 | 		{
185 | 			stage = new_stage;
186 | 			printf( "%zu%%\n", ( stage * 10 ) );
187 | 		}
188 | 	}
189 | 	printf( "done\n" );
190 | 
191 | 	delete[] memory;
192 | }
193 | 
194 | int main( int argc, char* argv[] )
195 | {
196 | 	//stress_test();
197 | 	benchmark_comparison();
198 | 
199 | 	return 0;
200 | }
201 | 


--------------------------------------------------------------------------------
/Tests/MutexQueue.h:
--------------------------------------------------------------------------------
 1 | #ifndef __MUTEXQUEUE_H__
 2 | #define __MUTEXQUEUE_H__
 3 | 
 4 | #include <cstdint>
 5 | #include <mutex>
 6 | 
 7 | template <typename T, size_t cache_line_size = 64> class MutexQueue
 8 | {
 9 |     public:
10 | 	explicit MutexQueue( size_t capacity ) 
11 | 		: m_items( static_cast<Item*>( _aligned_malloc( sizeof( Item ) * capacity, cache_line_size ) ) )
12 | 		, m_capacity( capacity )
13 | 		, m_head( 0 )
14 | 		, m_tail( 0 ) 
15 | 	{}
16 | 
17 | 	virtual ~MutexQueue() { _aligned_free( m_items ); }
18 | 
19 | 	bool try_enqueue( const T& value )
20 | 	{
21 | 		m_mutex.lock();
22 | 
23 | 		const uint64_t count = m_tail - m_head;
24 | 
25 | 		if( count == m_capacity )
26 | 		{
27 | 			m_mutex.unlock();
28 | 			return false;
29 | 		}
30 | 
31 | 		m_items[m_tail % m_capacity].value = value;
32 | 		++m_tail;
33 | 
34 | 		m_mutex.unlock();
35 | 		return true;
36 | 	}
37 | 
38 | 	bool try_dequeue( T& out )
39 | 	{
40 | 		m_mutex.lock();
41 | 
42 | 		if( m_head == m_tail )
43 | 		{
44 | 			m_mutex.unlock();
45 | 			return false;
46 | 		}
47 | 
48 | 		out = m_items[m_head % m_capacity].value;
49 | 		++m_head;
50 | 
51 | 		m_mutex.unlock();
52 | 		return true;
53 | 	}
54 | 
55 | 	size_t capacity() const { return m_capacity; }
56 | 
57 |     private:
58 | 	struct alignas(cache_line_size)Item
59 | 	{
60 | 		std::atomic<uint64_t> version;
61 | 		T value;
62 | 	};
63 | 
64 | 	struct alignas(cache_line_size)AlignedAtomicU64 : public std::atomic<uint64_t>
65 | 	{
66 | 		using std::atomic<uint64_t>::atomic;
67 | 	};
68 | 
69 | 	Item* m_items;
70 | 	size_t m_capacity;
71 | 	std::mutex m_mutex;
72 | 
73 | 	AlignedAtomicU64 m_head;
74 | 	AlignedAtomicU64 m_tail;
75 | };
76 | 
77 | #endif


--------------------------------------------------------------------------------