├── .gitignore
├── locks
    ├── mcs_lock.cpp
    ├── mcs_futex_lock.cpp
    ├── tatas_lock.hpp
    ├── mutex_lock.hpp
    ├── pthreads_lock.hpp
    ├── waitable_lock.hpp
    ├── mcs_lock.hpp
    ├── futex_lock.hpp
    ├── ticket_futex_lock.hpp
    └── mcs_futex_lock.hpp
├── examples
    └── counter
    │   ├── cs.hpp
    │   ├── counter_atomic.cpp
    │   ├── counter_std.cpp
    │   ├── counter_futex.cpp
    │   ├── counter_mcsfutex.cpp
    │   ├── counter_pthreads.cpp
    │   ├── cs.cpp
    │   ├── cs_atomic.cpp
    │   ├── counter_qd.cpp
    │   ├── README.md
    │   └── Makefile
├── .github
    ├── scripts
    │   ├── make_qd.sh
    │   └── get_gtest.sh
    └── workflows
    │   ├── codespell.yml
    │   └── ci.yml
├── threadid.cpp
├── util
    ├── pause.hpp
    └── type_tools.hpp
├── tests
    ├── CMakeLists.txt
    ├── lock.hpp
    └── gentests.py
├── readindicator
    └── reader_groups.hpp
├── waiting_future.hpp
├── threadid.hpp
├── LICENSE
├── qd.hpp
├── CMakeLists.txt
├── queues
    ├── simple_locked_queue.hpp
    ├── entry_queue.hpp
    ├── dual_buffer_queue.hpp
    └── buffer_queue.hpp
├── qd_condition_variable.hpp
├── padded.hpp
├── README.md
├── qdlock.hpp
├── mrqdlock.hpp
├── hqdlock.hpp
└── qdlock_base.hpp


/.gitignore:
--------------------------------------------------------------------------------
1 | build/
2 | tests/googletest/
3 | *~
4 | 


--------------------------------------------------------------------------------
/locks/mcs_lock.cpp:
--------------------------------------------------------------------------------
1 | #include "mcs_lock.hpp"
2 | 
3 | thread_local std::map<mcs_lock*, mcs_node2> mcs_lock::mcs_node_store;
4 | 


--------------------------------------------------------------------------------
/locks/mcs_futex_lock.cpp:
--------------------------------------------------------------------------------
1 | #include "mcs_futex_lock.hpp"
2 | 
3 | thread_local std::map<mcs_futex_lock*, mcs_node> mcs_futex_lock::mcs_node_store;
4 | 


--------------------------------------------------------------------------------
/examples/counter/cs.hpp:
--------------------------------------------------------------------------------
1 | #ifndef cs_hpp
2 | #define cs_hpp cs_hpp
3 | 
4 | void cs_init();
5 | void cs();
6 | void cs_finish();
7 | 
8 | #endif // cs_hpp
9 | 


--------------------------------------------------------------------------------
/.github/scripts/make_qd.sh:
--------------------------------------------------------------------------------
1 | #! /usr/bin/env bash
2 | 
3 | mkdir build && cd build
4 | cmake -DCMAKE_CXX_COMPILER=${QD_CXX} -DQD_DEBUG=${QD_DBG} ../
5 | make -j8
6 | 


--------------------------------------------------------------------------------
/threadid.cpp:
--------------------------------------------------------------------------------
1 | #include "threadid.hpp"
2 | 
3 | unsigned long thread_id_store::max_id = 0;
4 | std::set<unsigned long> thread_id_store::orphans;
5 | std::mutex thread_id_store::mutex;
6 | 
7 | thread_local thread_id_t thread_id;
8 | 


--------------------------------------------------------------------------------
/.github/workflows/codespell.yml:
--------------------------------------------------------------------------------
1 | name: codespell
2 | on: [push, pull_request]
3 | jobs:
4 |   codespell:
5 |     runs-on: ubuntu-latest
6 |     steps:
7 |     - uses: actions/checkout@v3
8 |     - uses: codespell-project/actions-codespell@master
9 | 


--------------------------------------------------------------------------------
/.github/scripts/get_gtest.sh:
--------------------------------------------------------------------------------
 1 | #! /usr/bin/env bash
 2 | 
 3 | VSN=1.11.0
 4 | 
 5 | cd tests
 6 | wget https://github.com/google/googletest/archive/release-${VSN}.zip
 7 | unzip release-${VSN}.zip
 8 | mv googletest-release-${VSN} googletest
 9 | cd ..
10 | 


--------------------------------------------------------------------------------
/util/pause.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef qd_pause_hpp
 2 | #define qd_pause_hpp qd_pause_hpp
 3 | 
 4 | namespace qd {
 5 | 
 6 | static inline void pause() {
 7 | 	asm("pause");
 8 | //	std::this_thread::yield();
 9 | }
10 | 
11 | } /* namespace qd */
12 | 
13 | #endif /* qd_pause_hpp */
14 | 


--------------------------------------------------------------------------------
/examples/counter/counter_atomic.cpp:
--------------------------------------------------------------------------------
 1 | #include<array>
 2 | #include<iostream>
 3 | #include<mutex>
 4 | #include<thread>
 5 | #include "cs.hpp"
 6 | 
 7 | void call_cs() {
 8 | 	for(int i = 0; i < 100000000/THREADS; i++) {
 9 | 		cs();
10 | 	}
11 | }
12 | 
13 | int main() {
14 | 	std::cout << THREADS << " threads / no locking (using atomic instruction)\n";
15 | 	cs_init();
16 | 	std::array<std::thread, THREADS> ts;
17 | 	for(auto& t : ts) {
18 | 		t = std::thread(&call_cs);
19 | 	}
20 | 	for(auto& t : ts) {
21 | 		t.join();
22 | 	}
23 | 	cs_finish();
24 | }
25 | 
26 | 


--------------------------------------------------------------------------------
/examples/counter/counter_std.cpp:
--------------------------------------------------------------------------------
 1 | #include<array>
 2 | #include<iostream>
 3 | #include<mutex>
 4 | #include<thread>
 5 | #include "cs.hpp"
 6 | 
 7 | static std::mutex lock;
 8 | 
 9 | void call_cs() {
10 | 	for(int i = 0; i < 100000000/THREADS; i++) {
11 | 		std::lock_guard<std::mutex> l(lock);
12 | 		cs();
13 | 	}
14 | }
15 | 
16 | int main() {
17 | 	std::cout << THREADS << " threads / std::mutex locking\n";
18 | 	cs_init();
19 | 	std::array<std::thread, THREADS> ts;
20 | 	for(auto& t : ts) {
21 | 		t = std::thread(&call_cs);
22 | 	}
23 | 	for(auto& t : ts) {
24 | 		t.join();
25 | 	}
26 | 	cs_finish();
27 | }
28 | 
29 | 


--------------------------------------------------------------------------------
/examples/counter/counter_futex.cpp:
--------------------------------------------------------------------------------
 1 | #include<array>
 2 | #include<iostream>
 3 | #include<mutex>
 4 | #include<thread>
 5 | #include "cs.hpp"
 6 | #include "locks/futex_lock.hpp"
 7 | 
 8 | static futex_lock lock;
 9 | 
10 | void call_cs() {
11 | 	for(int i = 0; i < 100000000/THREADS; i++) {
12 | 		std::lock_guard<futex_lock> l(lock);
13 | 		cs();
14 | 	}
15 | }
16 | 
17 | int main() {
18 | 	std::cout << THREADS << " threads / futex locking\n";
19 | 	cs_init();
20 | 	std::array<std::thread, THREADS> ts;
21 | 	for(auto& t : ts) {
22 | 		t = std::thread(&call_cs);
23 | 	}
24 | 	for(auto& t : ts) {
25 | 		t.join();
26 | 	}
27 | 	cs_finish();
28 | }
29 | 
30 | 


--------------------------------------------------------------------------------
/examples/counter/counter_mcsfutex.cpp:
--------------------------------------------------------------------------------
 1 | #include<array>
 2 | #include<iostream>
 3 | #include<mutex>
 4 | #include<thread>
 5 | #include "cs.hpp"
 6 | #include "../../locks/mcs_futex_lock.hpp"
 7 | 
 8 | static mcs_futex_lock lock;
 9 | 
10 | void call_cs() {
11 | 	for(int i = 0; i < 100000000/THREADS; i++) {
12 | 		std::lock_guard<mcs_futex_lock> l(lock);
13 | 		cs();
14 | 	}
15 | }
16 | 
17 | int main() {
18 | 	std::cout << THREADS << " threads / mcs_futex locking\n";
19 | 	cs_init();
20 | 	std::array<std::thread, THREADS> ts;
21 | 	for(auto& t : ts) {
22 | 		t = std::thread(&call_cs);
23 | 	}
24 | 	for(auto& t : ts) {
25 | 		t.join();
26 | 	}
27 | 	cs_finish();
28 | }
29 | 
30 | 


--------------------------------------------------------------------------------
/util/type_tools.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef qd_type_tools_hpp
 2 | #define qd_type_tools_hpp qd_type_tools_hpp
 3 | 
 4 | template<typename... Ps>
 5 | struct sumsizes;
 6 | template<typename T, typename... Ps>
 7 | struct sumsizes<T, Ps...> {
 8 | 	static constexpr long size = sizeof(T) + sumsizes<Ps...>::size;
 9 | };
10 | template<>
11 | struct sumsizes<> {
12 | 	static constexpr long size = 0;
13 | };
14 | 
15 | /* structure to create lists of types */
16 | template<typename... Ts>
17 | class types;
18 | template<typename T, typename... Ts>
19 | class types<T, Ts...> {
20 | 	public:
21 | 		typedef T type;
22 | 		typedef types<Ts...> tail;
23 | };
24 | template<>
25 | class types<> {};
26 | 
27 | #endif /* qd_type_tools_hpp */
28 | 


--------------------------------------------------------------------------------
/examples/counter/counter_pthreads.cpp:
--------------------------------------------------------------------------------
 1 | #include<array>
 2 | #include<iostream>
 3 | #include<pthread.h>
 4 | #include "cs.hpp"
 5 | 
 6 | static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
 7 | 
 8 | void* call_cs(void*) {
 9 | 	for(int i = 0; i < 100000000/THREADS; i++) {
10 | 		pthread_mutex_lock(&mutex);
11 | 		cs();
12 | 		pthread_mutex_unlock(&mutex);
13 | 	}
14 | 	return nullptr;
15 | }
16 | 
17 | int main() {
18 | 	std::cout << THREADS << " threads / pthreads locking\n";
19 | 	cs_init();
20 | 	std::array<pthread_t, THREADS> ts;
21 | 	for(auto& t : ts) {
22 | 		pthread_create(&t, NULL, call_cs, NULL);
23 | 	}
24 | 	for(auto& t : ts) {
25 | 		void* ignore;
26 | 		pthread_join(t, &ignore);
27 | 	}
28 | 	cs_finish();
29 | }
30 | 
31 | 


--------------------------------------------------------------------------------
/examples/counter/cs.cpp:
--------------------------------------------------------------------------------
 1 | #include "cs.hpp"
 2 | 
 3 | #include<chrono>
 4 | #include<iostream>
 5 | 
 6 | static int shared_counter;
 7 | static std::chrono::time_point<std::chrono::high_resolution_clock> start_time;
 8 | 
 9 | void cs_init() {
10 | 	shared_counter = 0;
11 | 	start_time = std::chrono::high_resolution_clock::now();
12 | }
13 | 
14 | void cs() {
15 | 	shared_counter += 1;
16 | }
17 | 
18 | void cs_finish() {
19 | 	auto end_time = std::chrono::high_resolution_clock::now();
20 | 	auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time);
21 | 	std::cout << "final counter value: " << shared_counter << std::endl;
22 | 	std::cout << "time needed: " << duration.count() << " ms" << std::endl;
23 | }
24 | 
25 | 


--------------------------------------------------------------------------------
/examples/counter/cs_atomic.cpp:
--------------------------------------------------------------------------------
 1 | #include<atomic>
 2 | #include<chrono>
 3 | #include<iostream>
 4 | 
 5 | static std::atomic<int> shared_counter;
 6 | static std::chrono::time_point<std::chrono::high_resolution_clock> start_time;
 7 | 
 8 | void cs_init() {
 9 | 	shared_counter = 0;
10 | 	start_time = std::chrono::high_resolution_clock::now();
11 | }
12 | 
13 | void cs() {
14 | 	shared_counter += 1;
15 | }
16 | 
17 | void cs_finish() {
18 | 	auto end_time = std::chrono::high_resolution_clock::now();
19 | 	auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time);
20 | 	std::cout << "final counter value: " << shared_counter.load() << std::endl;
21 | 	std::cout << "time needed: " << duration.count() << " ms" << std::endl;
22 | }
23 | 
24 | 


--------------------------------------------------------------------------------
/tests/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | add_subdirectory(googletest)
 2 | include_directories(${gtest_SOURCE_DIR}/include ${gtest_SOURCE_DIR})
 3 | 
 4 | set(genfiles
 5 | 	generated_test0.cpp
 6 | 	generated_test1.cpp
 7 | 	generated_test2.cpp
 8 | 	generated_test3.cpp
 9 | 	generated_test4.cpp
10 | 	generated_test5.cpp
11 | 	generated_test6.cpp
12 | 	generated_test7.cpp)
13 | 
14 | add_custom_command(
15 | 	OUTPUT ${genfiles}
16 | 	COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/gentests.py
17 | 	WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
18 | 	DEPENDS gentests.py)
19 | 
20 | include_directories(${CMAKE_CURRENT_BINARY_DIR})
21 | 
22 | add_executable(lockTests lock.hpp ${genfiles})
23 | target_link_libraries(lockTests qd gtest gtest_main)
24 | add_test(lockTests ${CMAKE_BINARY_DIR}/bin/lockTests)
25 | 


--------------------------------------------------------------------------------
/locks/tatas_lock.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef qd_tatas_lock_hpp
 2 | #define qd_tatas_lock_hpp qd_tatas_lock_hpp
 3 | 
 4 | #include<atomic>
 5 | 
 6 | #include "util/pause.hpp"
 7 | 
 8 | /** @brief a test-and-test-and-set lock */
 9 | class tatas_lock {
10 | 	std::atomic<bool> locked; /* TODO can std::atomic_flag be used? */
11 | 	public:
12 | 		tatas_lock() : locked(false) {};
13 | 		tatas_lock(tatas_lock&) = delete; /* TODO? */
14 | 		bool try_lock() {
15 | 			if(is_locked()) return false;
16 | 			return !locked.exchange(true, std::memory_order_acq_rel);
17 | 		}
18 | 		void unlock() {
19 | 			locked.store(false, std::memory_order_release);
20 | 		}
21 | 		bool is_locked() {
22 | 			return locked.load(std::memory_order_acquire);
23 | 		}
24 | 		void lock() {
25 | 			while(!try_lock()) {
26 | 				qd::pause();
27 | 			}
28 | 		}
29 | 		void wake() {}
30 | };
31 | 
32 | #endif /* qd_tatas_lock_hpp */
33 | 


--------------------------------------------------------------------------------
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
 1 | name: CI
 2 | on: [push, pull_request]
 3 | 
 4 | jobs:
 5 |   Build-and-Test:
 6 |     if: ${{ !contains(github.event.head_commit.message, 'ci skip') }}
 7 |     strategy:
 8 |       fail-fast: false
 9 |       matrix:
10 |         os: [ubuntu-latest]
11 |         cxx: [g++,clang++]
12 |         dbg: [OFF, ON]
13 |     name: On ${{ matrix.os }} CXX=${{ matrix.cxx }} DBG=${{ matrix.dbg }}
14 |     runs-on: ${{ matrix.os }}
15 |     env:
16 |       QD_CXX: ${{ matrix.cxx }}
17 |       QD_DBG: ${{ matrix.dbg }}
18 |     steps:
19 |       - name: Checkout code
20 |         uses: actions/checkout@v3
21 |       - name: Install packages
22 |         run: |
23 |           sudo apt-get update -qq
24 |           sudo apt-get install -y libnuma-dev
25 |       - name: Before install
26 |         run: ./.github/scripts/get_gtest.sh
27 |       - name: Build
28 |         run: ./.github/scripts/make_qd.sh
29 |       - name: Test
30 |         run: cd build && make test
31 | 


--------------------------------------------------------------------------------
/readindicator/reader_groups.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef qd_reader_groups_hpp
 2 | #define qd_reader_groups_hpp qd_reader_groups_hpp
 3 | 
 4 | #include "threadid.hpp"
 5 | 
 6 | template<int GROUPS>
 7 | class reader_groups {
 8 | 	struct alignas(64) counter_t {
 9 | 		char pad1[64];
10 | 		std::atomic<int> cnt;
11 | 		char pad2[64];
12 | 		counter_t() : cnt(0) {}
13 | 	};
14 | 	std::array<counter_t, GROUPS> counters;
15 | 	public:
16 | 		reader_groups() {
17 | 			for(int i = 0; i < GROUPS; i++) {
18 | 				counters[i].cnt.store(0, std::memory_order_release);
19 | 			}
20 | 		}
21 | 		bool query() {
22 | 			for(counter_t& counter : counters)
23 | 				if(counter.cnt.load(std::memory_order_acquire) > 0) return true;
24 | 			return false;
25 | 		}
26 | 		void arrive() {
27 | 			counters[thread_id % GROUPS].cnt.fetch_add(1, std::memory_order_release);
28 | 		}
29 | 		void depart() {
30 | 			counters[thread_id % GROUPS].cnt.fetch_sub(1, std::memory_order_release);
31 | 		}
32 | };
33 | 
34 | #endif /* qd_reader_groups_hpp */
35 | 


--------------------------------------------------------------------------------
/examples/counter/counter_qd.cpp:
--------------------------------------------------------------------------------
 1 | #include<array>
 2 | #include<iostream>
 3 | #include<thread>
 4 | #include "cs.hpp"
 5 | #include "qd.hpp"
 6 | 
 7 | static qdlock lock;
 8 | 
 9 | void call_cs() {
10 | 	for(int i = 0; i < 100000000/THREADS; i++) {
11 | 		/* DELEGATE_N does not wait for a result */
12 | 		lock.DELEGATE_N(cs);
13 | 	}
14 | }
15 | 
16 | /* an empty function can be used to wait for previously delegated sections:
17 |  * when the (empty) function is executed, everything preceding it is also done.
18 |  */
19 | void empty() {
20 | }
21 | 
22 | int main() {
23 | 	std::cout << THREADS << " threads / QD locking\n";
24 | 	cs_init();
25 | 	std::array<std::thread, THREADS> ts;
26 | 	for(auto& t : ts) {
27 | 		t = std::thread(&call_cs);
28 | 	}
29 | 	for(auto& t : ts) {
30 | 		t.join();
31 | 	}
32 | 	/* the empty function is used as a marker:
33 | 	 * all preceding sections are executed before it */
34 | 	auto b = lock.DELEGATE_F(&empty);
35 | 	b.wait(); /* wait for the call to empty() */
36 | 
37 | 	cs_finish();
38 | }
39 | 
40 | 


--------------------------------------------------------------------------------
/waiting_future.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef waiting_future_hpp
 2 | #define waiting_future_hpp waiting_future_hpp
 3 | 
 4 | #include<future>
 5 | 
 6 | template<typename T>
 7 | class waiting_future : public std::future<T> {
 8 | 	public:
 9 | 		waiting_future() {}
10 | 		waiting_future(waiting_future& rhs) : std::future<T>(rhs) {}
11 | 		waiting_future(waiting_future&& rhs) : std::future<T>(std::move(rhs)) {}
12 | 		waiting_future(std::future<T>& rhs) : std::future<T>(rhs) {}
13 | 		waiting_future(std::future<T>&& rhs) : std::future<T>(std::move(rhs)) {}
14 | 		~waiting_future() {
15 | 			if(this->valid()) {
16 | 				this->wait();
17 | 			}
18 | 		}
19 | 		waiting_future& operator=(waiting_future& rhs) {
20 | 			std::future<T>::operator=(rhs);
21 | 			return *this;
22 | 		}
23 | 		waiting_future& operator=(waiting_future&& rhs) {
24 | 			std::future<T>::operator=(std::move(rhs));
25 | 			return *this;
26 | 		}
27 | 		void discard() {
28 | 			std::future<T> tmp;
29 | 			std::swap(tmp, *this);
30 | 		}
31 | };
32 | 
33 | #endif // waiting_future_hpp
34 | 


--------------------------------------------------------------------------------
/threadid.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef threadid_hpp
 2 | #define threadid_hpp threadid_hpp
 3 | 
 4 | #include <mutex>
 5 | #include <set>
 6 | 
 7 | class thread_id_store {
 8 | 	static unsigned long max_id;
 9 | 	static std::set<unsigned long> orphans;
10 | 	static std::mutex mutex;
11 | 	typedef std::lock_guard<std::mutex> scoped_lock;
12 | 	public:
13 | 		static unsigned long get() {
14 | 			scoped_lock lock(mutex);
15 | 			if(orphans.empty()) {
16 | 				max_id++;
17 | 				return max_id;
18 | 			} else {
19 | 				auto first = orphans.begin();
20 | 				auto result = *first;
21 | 				orphans.erase(first);
22 | 				return result;
23 | 			}
24 | 		}
25 | 		static void free(unsigned long idx) {
26 | 			scoped_lock lock(mutex);
27 | 			if(idx == max_id) {
28 | 				max_id--;
29 | 				while(orphans.erase(max_id)) {
30 | 					max_id--;
31 | 				}
32 | 			} else {
33 | 				orphans.insert(idx);
34 | 			}
35 | 		}
36 | };
37 | 
38 | class thread_id_t {
39 | 	unsigned long id;
40 | 	public:
41 | 		operator unsigned long() {
42 | 			return id;
43 | 		}
44 | 		thread_id_t() : id(thread_id_store::get()) {}
45 | 		~thread_id_t() {
46 | 			thread_id_store::free(id);
47 | 		}
48 | };
49 | 
50 | extern thread_local thread_id_t thread_id;
51 | 
52 | #endif // threadid_hpp
53 | 


--------------------------------------------------------------------------------
/locks/mutex_lock.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef qd_mutex_lock_hpp
 2 | #define qd_mutex_lock_hpp qd_mutex_lock_hpp
 3 | 
 4 | #include<atomic>
 5 | #include<mutex>
 6 | 
 7 | /** @brief a std::mutex based lock */
 8 | class mutex_lock {
 9 | 	std::atomic<bool> locked;
10 | 	std::mutex mutex;
11 | 	public:
12 | 		mutex_lock() : locked(false), mutex() {};
13 | 		mutex_lock(mutex_lock&) = delete; /* TODO? */
14 | 		bool try_lock() {
15 | 			if(!is_locked() && mutex.try_lock()) {
16 | 				locked.store(true, std::memory_order_release);
17 | 				return true;
18 | 			} else {
19 | 				return false;
20 | 			}
21 | 		}
22 | 		void unlock() {
23 | 			locked.store(false, std::memory_order_release);
24 | 			mutex.unlock();
25 | 		}
26 | 		bool is_locked() {
27 | 			/* This may sometimes return false when the lock is already acquired.
28 | 			 * This is safe, because the locking call that acquired the lock in
29 | 			 * that case has not yet returned (it needs to set the locked flag first),
30 | 			 * so this is concurrent with calling is_locked first and then locking the lock.
31 | 			 * 
32 | 			 * This may also sometimes return false when the lock is still locked, but
33 | 			 * about to be unlocked. This is safe, because of a similar argument as above.
34 | 			 */
35 | 			return locked.load(std::memory_order_acquire);
36 | 		}
37 | 		void lock() {
38 | 			mutex.lock();
39 | 			locked.store(true, std::memory_order_release);
40 | 		}
41 | };
42 | 
43 | #endif /* qd_mutex_lock_hpp */
44 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 |   Copyright (c) 2013-2016, David Klaftenegger (<david.klaftenegger@it.uu.se>).
 2 |   All rights reserved.
 3 | 
 4 | Redistribution and use in source and binary forms, with or without modification,
 5 | are permitted provided that the following conditions are met:
 6 | 
 7 |     1. Redistributions of source code must retain the above copyright notice,
 8 |        this list of conditions and the following disclaimer.
 9 | 
10 |     2. Redistributions in binary form must reproduce the above copyright
11 |        notice, this list of conditions and the following disclaimer in the
12 |        documentation and/or other materials provided with the distribution.
13 | 
14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
15 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
16 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
18 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
19 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
20 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
21 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
23 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 | 


--------------------------------------------------------------------------------
/locks/pthreads_lock.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef qd_pthreads_lock_hpp
 2 | #define qd_pthreads_lock_hpp qd_pthreads_lock_hpp
 3 | 
 4 | #include<atomic>
 5 | #include<pthread.h>
 6 | 
 7 | /** @brief a pthreads based lock */
 8 | class pthreads_lock {
 9 | 	std::atomic<bool> locked;
10 | 	pthread_mutex_t mutex;
11 | 	public:
12 | 		pthreads_lock() : locked(false), mutex(PTHREAD_MUTEX_INITIALIZER) {};
13 | 		pthreads_lock(pthreads_lock&) = delete; /* TODO? */
14 | 		bool try_lock() {
15 | 			if(!is_locked() && !pthread_mutex_trylock(&mutex)) {
16 | 				locked.store(true, std::memory_order_release);
17 | 				return true;
18 | 			} else {
19 | 				return false;
20 | 			}
21 | 		}
22 | 		void unlock() {
23 | 			locked.store(false, std::memory_order_release);
24 | 			pthread_mutex_unlock(&mutex);
25 | 		}
26 | 		bool is_locked() {
27 | 			/* This may sometimes return false when the lock is already acquired.
28 | 			 * This is safe, because the locking call that acquired the lock in
29 | 			 * that case has not yet returned (it needs to set the locked flag first),
30 | 			 * so this is concurrent with calling is_locked first and then locking the lock.
31 | 			 * 
32 | 			 * This may also sometimes return false when the lock is still locked, but
33 | 			 * about to be unlocked. This is safe, because of a similar argument as above.
34 | 			 */
35 | 			return locked.load(std::memory_order_acquire);
36 | 		}
37 | 		void lock() {
38 | 			pthread_mutex_lock(&mutex);
39 | 			locked.store(true, std::memory_order_release);
40 | 		}
41 | };
42 | 
43 | #endif /* qd_pthreads_lock_hpp */
44 | 


--------------------------------------------------------------------------------
/qd.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef qd_qd_hpp
 2 | #define qd_qd_hpp qd_qd_hpp
 3 | 
 4 | #include "locks/waitable_lock.hpp"
 5 | #include "locks/pthreads_lock.hpp"
 6 | #include "locks/tatas_lock.hpp"
 7 | #include "locks/mutex_lock.hpp"
 8 | #include "locks/futex_lock.hpp"
 9 | #include "locks/mcs_futex_lock.hpp"
10 | #include "locks/mcs_lock.hpp"
11 | #include "locks/ticket_futex_lock.hpp"
12 | 
13 | #include "queues/buffer_queue.hpp"
14 | #include "queues/dual_buffer_queue.hpp"
15 | #include "queues/entry_queue.hpp"
16 | #include "queues/simple_locked_queue.hpp"
17 | 
18 | #include "qdlock.hpp"
19 | #include "hqdlock.hpp"
20 | #include "mrqdlock.hpp"
21 | 
22 | #include "qd_condition_variable.hpp"
23 | 
24 | template<typename Lock>
25 | class extended_lock : public Lock {
26 | 	public:
27 | 		bool try_lock_or_wait() {
28 | 			return this->try_lock();
29 | 		}
30 | };
31 | 
32 | using internal_lock = mcs_futex_lock;
33 | using qdlock = qdlock_impl<mcs_futex_lock, dual_buffer_queue<6144, 24, atomic_instruction_policy_t::use_fetch_and_add>, starvation_policy_t::starvation_free>;
34 | using mrqdlock = mrqdlock_impl<internal_lock, dual_buffer_queue<6144,24>, reader_groups<64>, 65536>;
35 | //using qd_condition_variable = qd_condition_variable_impl<mutex_lock, simple_locked_queue>;
36 | 
37 | #define DELEGATE_F(function, ...) template delegate_f<decltype(function), function>(__VA_ARGS__)
38 | #define DELEGATE_N(function, ...) template delegate_n<decltype(function), function>(__VA_ARGS__)
39 | #define DELEGATE_P(function, ...) template delegate_p<decltype(function), function>(__VA_ARGS__)
40 | #define DELEGATE_FP(function, ...) template delegate_fp<decltype(function), function>(__VA_ARGS__)
41 | #define WAIT_REDELEGATE_P(function, ...) template wait_redelegate_p<decltype(function), function>(__VA_ARGS__)
42 | 
43 | #endif /* qd_qd_hpp */
44 | 


--------------------------------------------------------------------------------
/locks/waitable_lock.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef qd_waitable_lock_hpp
 2 | #define qd_waitable_lock_hpp qd_waitable_lock_hpp
 3 | 
 4 | #include<condition_variable>
 5 | 
 6 | /**
 7 |  * @brief lock class wrapper to add wait/notify functionality
 8 |  * @tparam Lock a locking class
 9 |  * @details This wrapper adds functionality to wait on each instance of a class
10 |  *          without requiring a lock to be taken/released. This is useful when
11 |  *          implementing another lock, so that spinning can be avoided.
12 |  * @warning  This implementation relies on std::condition_variable_any not actually needing a lock,
13 |  *       which violates its preconditions.
14 |  * @remarks This is likely not the most efficient way of implementing waiting.
15 |  * @todo The private inheritance is used to get a memory layout in which
16 |  *       clang++-3.4 spills less than if these structures appeared in the opposite
17 |  *       order. This "optimization" might not be the best solution.
18 |  */
19 | template<typename Lock>
20 | class waitable_lock : private std::condition_variable_any, public Lock {
21 | 	/**
22 | 	 * @brief a dummy lock class
23 | 	 * @warning This lock does not provide locking.
24 | 	 * @details This class is not intended for use as a lock, but as
25 | 	 *          std::condition_variable_any requires a lock class,
26 | 	 *          this provides it.
27 | 	 */
28 | 	struct null_lock {
29 | 		void lock() {}
30 | 		void unlock() {}
31 | 	};
32 | 
33 | 	/** @brief an associated dummy lock for the std::condition_variable_any */
34 | 	null_lock not_a_lock;
35 | 
36 | 	public:
37 | 
38 | 		/** @brief wait until notified */
39 | 		void wait() {
40 | 			std::condition_variable_any::wait(not_a_lock);
41 | 		}
42 | 
43 | 		/** @brief notify (at least) one waiting thread */
44 | 		void notify_one() {
45 | 			std::condition_variable_any::notify_one();
46 | 		}
47 | 
48 | 		/** @brief notify all waiting threads */
49 | 		void notify_all() {
50 | 			std::condition_variable_any::notify_all();
51 | 		}
52 | };
53 | 
54 | #endif /* qd_waitable_lock_hpp */
55 | 


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required(VERSION 3.3)
 2 | project(QDLibrary VERSION 0.1)
 3 | 
 4 | set(DEFAULT_C_FLAGS "-std=c11 -pthread")
 5 | set(DEFAULT_CXX_FLAGS "-std=c++11 -pthread")
 6 | set(DEFAULT_LINK_FLAGS "")
 7 | 
 8 | option(QD_DEBUG
 9 | 	"Build the QD library without optimization and with debugging symbols" OFF)
10 | if(QD_DEBUG)
11 | 	set(DEFAULT_C_FLAGS "${DEFAULT_C_FLAGS} -O0 -g -Wall -Wextra -Werror")
12 | 	set(DEFAULT_CXX_FLAGS
13 | 		"${DEFAULT_CXX_FLAGS} -O0 -g -Wall -Wextra -Werror -Wno-unused-private-field -fsanitize=undefined -fsanitize=address")
14 | 	set(DEFAULT_LINK_FLAGS "${DEFAULT_LINK_FLAGS} -g -Wall -Wextra -Werror")
15 | else(QD_DEBUG)
16 | 	set(DEFAULT_C_FLAGS "${DEFAULT_C_FLAGS} -O3")
17 | 	set(DEFAULT_CXX_FLAGS "${DEFAULT_CXX_FLAGS} -O3")
18 | 	set(DEFAULT_LINK_FLAGS "${DEFAULT_LINK_FLAGS} -O3")
19 | endif(QD_DEBUG)
20 | 
21 | set(CMAKE_C_FLAGS  "${CMAKE_C_FLAGS} ${DEFAULT_C_FLAGS}")
22 | set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} ${DEFAULT_CXX_FLAGS}")
23 | set(CMAKE_EXE_LINKER_FLAGS  "${CMAKE_EXE_LINKER_FLAGS} ${DEFAULT_LINK_FLAGS}")
24 | 
25 | set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
26 | set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
27 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
28 | 
29 | include_directories("${PROJECT_SOURCE_DIR}")
30 | 
31 | # put together all source files 
32 | set(qd_sources "threadid.cpp")
33 | set(lock_sources locks/mcs_lock.cpp locks/mcs_futex_lock.cpp)
34 | 
35 | add_library(qd SHARED ${qd_sources} ${lock_sources})
36 | 
37 | option(QD_USE_LIBNUMA
38 | 	"Use libnuma to determine NUMA structure" ON)
39 | if(QD_USE_LIBNUMA)
40 | 	target_link_libraries(qd numa)
41 | 	add_definitions(-DQD_USE_LIBNUMA)
42 | endif(QD_USE_LIBNUMA)
43 | 
44 | install(TARGETS qd
45 | 	COMPONENT "Runtime"
46 | 	RUNTIME DESTINATION bin
47 | 	LIBRARY DESTINATION lib
48 | 	ARCHIVE DESTINATION lib)
49 | 
50 | install(DIRECTORY .
51 | 	DESTINATION include/qd/
52 | 	COMPONENT "Development"
53 | 	FILES_MATCHING
54 | 		PATTERN "*.h"
55 | 		PATTERN "*.hpp")
56 | 
57 | option(QD_TESTS
58 | 	"Build tests for QD locking" ON)
59 | if(QD_TESTS)
60 | 	enable_testing()
61 | 	include_directories("${PROJECT_SOURCE_DIR}/tests")
62 | 	add_subdirectory(tests)
63 | endif(QD_TESTS)
64 | 


--------------------------------------------------------------------------------
/examples/counter/README.md:
--------------------------------------------------------------------------------
 1 | Shared Counter Example
 2 | ======================
 3 | Introduction
 4 | ------------
 5 | In this example we evaluate various ways of implementing a program that
 6 | launches N threads, which each execute a critical section X times,
 7 | such that the total number of critical section executions equals:
 8 | X/N - 100,000,000
 9 | 
10 | Building
11 | --------
12 | `make`
13 | builds a version for 5, 50 and 500 threads for each of the locking schemes:
14 |  * `pthreads` uses pthreads directly: `counter_pthreads{5,50,500}`
15 |  * `std` uses `std::mutex` and `std::lock_guard` for locking: `counter_std{5,50,500}`
16 |  * `qd` uses QD locking as provided by this library: `counter_qd{5,50,500}`
17 | 
18 | Comparison with atomic instructions
19 | -----------------------------------
20 | For comparison, this example includes code that implements the critical section
21 | using atomic instructions. While this is not executing the same code that the
22 | other locking methods use, it shows the "optimal" implementation for this
23 | particular example program, using an already thread-safe "critical" section.
24 | 
25 | `make atomics`
26 | additionally builds versions using atomic instructions instead of locks.
27 | 
28 | Running
29 | -------
30 | You can use the following command to view output including runtimes:
31 | ```
32 | for counter in *5 *50 *500; do ./$counter; echo; done
33 | ```
34 | 
35 | Results
36 | -------
37 | **This is not a benchmark:**
38 | These example programs do not try to minimize variation of results. E.g., the
39 | worker threads start working immediately, not when all threads are spawned.
40 | The workload in this example also is not representative of any real world load.
41 | However, the numbers below show how much time is spent on synchronization in
42 | different implementations.
43 | 
44 | ### Intel(R) Core(TM) i7-3770 CPU @ 3.40GHz ###
45 | 4 cores with 2 threads each (times in ms)
46 | ```
47 |             |total number of threads|
48 |             |   5   |  50   |  500  |
49 |             | ----- | ----- | ----- |
50 |  pthreads   | 17229 | 21879 | 18162 |
51 |  std::mutex | 16644 | 24020 | 21734 |
52 |  QD locking |  3422 |  4037 |  4878 |
53 |  atomic cs  |  1689 |  1830 |  1833 | 
54 | ```
55 | 
56 | Files
57 | -----
58 | * `counter_pthreads.cpp` - pthreads-only implementation
59 | * `counter_std.cpp` - std::mutex based implementation
60 | * `counter_qd.cpp` - using the qd locks provided by this library
61 | * `cs.hpp` - interface definition for the critical section
62 | * `cs.cpp` - the critical section implementation (not thread-safe)
63 | 
64 | * `counter_pthreads.cpp` - atomic instruction implementation
65 | * `cs_atomic.cpp` - thread-safe variant of cs.cpp using atomic instructions
66 | 


--------------------------------------------------------------------------------
/queues/simple_locked_queue.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef qd_simple_locked_queue_hpp
 2 | #define qd_simple_locked_queue_hpp qd_simple_locked_queue_hpp
 3 | 
 4 | #include<array>
 5 | #include<cassert>
 6 | #include<mutex>
 7 | #include<queue>
 8 | 
 9 | /**
10 |  * @brief a simple lock-based queue
11 |  * @note this is not a tantrum queue, and thus cannot provide starvation freedom guarantees
12 |  */
13 | class simple_locked_queue {
14 | 	/** @brief internal lock to protect the queue */
15 | 	std::mutex lock;
16 | 
17 | 	/** @brief internal queue */
18 | 	std::queue<std::array<char, 128>> queue;
19 | 
20 | 	/** @brief convenience type alias for managing the lock */
21 | 	typedef std::lock_guard<std::mutex> scoped_guard;
22 | 
23 | 	/** @brief type alias for stored function type */
24 | 	typedef void(*ftype)(char*);
25 | 	
26 | 	/* some constants */
27 | 	static const bool CLOSED = false;
28 | 	static const bool SUCCESS = true;
29 | 
30 | 	void forwardall(char*, long i) {
31 | 		assert(i <= 120);
32 | 		if(i > 120) throw "up";
33 | 	};
34 | 	template<typename P, typename... Ts>
35 | 	void forwardall(char* buffer, long offset, P&& p, Ts&&... ts) {
36 | 		assert(offset <= 120);
37 | 		auto ptr = reinterpret_cast<P*>(&buffer[offset]);
38 | 		new (ptr) P(std::forward<P>(p));
39 | 		forwardall(buffer, offset+sizeof(p), std::forward<Ts>(ts)...);
40 | 	}
41 | 	public:
42 | 		void open() {
43 | 			/* TODO this function should not even be here */
44 | 			/* no-op as this is an "infinite" queue that always accepts more data */
45 | 		}
46 | 		/**
47 | 		 * @brief enqueues an entry
48 | 		 * @tparam P return type of associated function
49 | 		 * @param op wrapper function for associated function
50 | 		 * @return SUCCESS on successful storing in queue, CLOSED otherwise
51 | 		 */
52 | 		template<typename... Ps>
53 | 		bool enqueue(ftype op, Ps*... ps) {
54 | 			std::array<char, 128> val;
55 | 			scoped_guard l(lock);
56 | 			queue.push(val);
57 | 			forwardall(queue.back().data(), 0, std::move(op), std::move(*ps)...);
58 | 			return SUCCESS;
59 | 		}
60 | 
61 | 		/** execute all stored operations */
62 | 		void flush() {
63 | 			scoped_guard l(lock);
64 | 			while(!queue.empty()) {
65 | 				auto operation = queue.front();
66 | 				char* ptr = operation.data();
67 | 				ftype* fun = reinterpret_cast<ftype*>(ptr);
68 | 				ptr += sizeof(ftype*);
69 | 				(*fun)(ptr);
70 | 				queue.pop();
71 | 			}
72 | 		}
73 | 		/** execute one stored operation */
74 | 		void flush_one() {
75 | 			scoped_guard l(lock);
76 | 			if(!queue.empty()) {
77 | 				char* ptr = queue.front().data();
78 | 				ftype* fun = reinterpret_cast<ftype*>(ptr);
79 | 				ptr += sizeof(ftype);
80 | 				(*fun)(ptr);
81 | 				queue.pop();
82 | 			}
83 | 		}
84 | };
85 | 
86 | #endif /* qd_simple_locked_queue_hpp */
87 | 


--------------------------------------------------------------------------------
/examples/counter/Makefile:
--------------------------------------------------------------------------------
 1 | CXX=clang++
 2 | CXXFLAGS=-O2 -std=c++11 -I../../
 3 | LDFLAGS=-O2 -std=c++11 -pthreads -L/usr/local/lib/ -lqd
 4 | 
 5 | .PHONY: qd_comparison atomic qd_only all clean
 6 | 
 7 | COMPNUMS = 5 50 500
 8 | #COMPNUMS = 1 2 4 8 16 32 64 128
 9 | 
10 | ONLYNUMS = 1 2 3 4 5 6 7 8 9
11 | 
12 | qd_comparison: $(addprefix counter_qd,${COMPNUMS}) $(addprefix counter_pthreads,${COMPNUMS}) $(addprefix counter_std,${COMPNUMS}) #$(addprefix counter_mcsfutex,${COMPNUMS}) $(addprefix counter_futex,${COMPNUMS})
13 | 
14 | atomic: $(addprefix counter_atomic,${COMPNUMS})
15 | 
16 | qd_only: $(addprefix counter_qd,${ONLYNUMS})
17 | 
18 | all: qd_comparison atomic qd_only
19 | 
20 | NUMS = $(shell seq 1 500)
21 | QD_OBJ       := $(addsuffix .o,$(addprefix counter_qd_,${NUMS}))
22 | PTHREADS_OBJ := $(addsuffix .o,$(addprefix counter_pthreads_,${NUMS}))
23 | STD_OBJ      := $(addsuffix .o,$(addprefix counter_std_,${NUMS}))
24 | MCSFUTEX_OBJ := $(addsuffix .o,$(addprefix counter_mcsfutex_,${NUMS}))
25 | FUTEX_OBJ    := $(addsuffix .o,$(addprefix counter_futex_,${NUMS}))
26 | ATOMIC_OBJ   := $(addsuffix .o,$(addprefix counter_atomic_,${NUMS}))
27 | 
28 | 
29 | .PRECIOUS: counter_qd_%.o
30 | $(QD_OBJ) : counter_qd_%.o : counter_qd.cpp
31 | 	$(CXX) -c -o $@ $< $(CXXFLAGS) -DTHREADS=$*
32 | 
33 | .PRECIOUS: counter_pthreads_%.o
34 | $(PTHREADS_OBJ) : counter_pthreads_%.o : counter_pthreads.cpp
35 | 	$(CXX) -c -o $@ $< $(CXXFLAGS) -DTHREADS=$*
36 | 
37 | .PRECIOUS: counter_std_%.o
38 | $(STD_OBJ) : counter_std_%.o : counter_std.cpp
39 | 	$(CXX) -c -o $@ $< $(CXXFLAGS) -DTHREADS=$*
40 | 
41 | .PRECIOUS: counter_mcsfutex_%.o
42 | $(MCSFUTEX_OBJ) : counter_mcsfutex_%.o : counter_mcsfutex.cpp
43 | 	$(CXX) -c -o $@ $< $(CXXFLAGS) -DTHREADS=$*
44 | 
45 | .PRECIOUS: counter_futex_%.o
46 | $(FUTEX_OBJ) : counter_futex_%.o : counter_futex.cpp
47 | 	$(CXX) -c -o $@ $< $(CXXFLAGS) -DTHREADS=$*
48 | 
49 | .PRECIOUS: counter_atomic_%.o
50 | $(ATOMIC_OBJ) : counter_atomic_%.o : counter_atomic.cpp
51 | 	$(CXX) -c -o $@ $< $(CXXFLAGS) -DTHREADS=$*
52 | 
53 | cs.o cs_atomic.o: %.o : %.cpp
54 | 	$(CXX) -c -o $@ $< $(CXXFLAGS)
55 | 
56 | counter_qd%: counter_qd_%.o cs.o
57 | 	$(CXX) -o $@ $^ $(LDFLAGS)
58 | counter_pthreads%: counter_pthreads_%.o cs.o
59 | 	$(CXX) -o $@ $^ $(LDFLAGS)
60 | counter_std%: counter_std_%.o cs.o
61 | 	$(CXX) -o $@ $^ $(LDFLAGS)
62 | counter_mcsfutex%: counter_mcsfutex_%.o cs.o
63 | 	$(CXX) -o $@ $^ $(LDFLAGS)
64 | counter_futex%: counter_futex_%.o cs.o
65 | 	$(CXX) -o $@ $^ $(LDFLAGS)
66 | counter_atomic%: counter_atomic_%.o cs_atomic.o
67 | 	$(CXX) -o $@ $^ $(LDFLAGS)
68 | 
69 | 
70 | clean:
71 | 	-@rm counter_pthreads[0-9]* \
72 | 		counter_std[0-9]* \
73 | 		counter_qd[0-9]* \
74 | 		counter_futex[0-9]* \
75 | 		counter_mcsfutex[0-9]* \
76 | 		counter_atomic[0-9]* \
77 | 		*.o 2>/dev/null || true
78 | 


--------------------------------------------------------------------------------
/tests/lock.hpp:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * @file
  3 |  * @brief This file provides simple unit tests for the QD locks
  4 |  */
  5 | 
  6 | #include <thread>
  7 | 
  8 | #include "qd.hpp"
  9 | #include "gtest/gtest.h"
 10 | 
 11 | /**
 12 |  * @brief Class to allow instantiating Tests for multiple implementation types.
 13 |  */
 14 | template <typename Lock>
 15 | class LockTestBase : public testing::Test {
 16 | 
 17 | protected:
 18 | 	LockTestBase() {
 19 | 	}
 20 | 
 21 | 	~LockTestBase() {
 22 | 	}
 23 | 
 24 | 	Lock lock;
 25 | };
 26 | 
 27 | template <typename Lock>
 28 | class LockTest : public LockTestBase<Lock> {};
 29 | 
 30 | template <typename DelegationLock>
 31 | class DelegationTest : public LockTestBase<DelegationLock> {};
 32 | 
 33 | TYPED_TEST_SUITE_P(LockTest);
 34 | TYPED_TEST_SUITE_P(DelegationTest);
 35 | 
 36 | /**
 37 |  * @brief Tests lock functionality
 38 |  */
 39 | TYPED_TEST_P(LockTest, LockSpawnThread) {
 40 | 	int* counter = new int;
 41 | 	*counter = 0;
 42 | 	std::thread threads[1];
 43 | 	threads[0] = std::thread(
 44 | 		[this, counter]() {
 45 | 			ASSERT_NO_THROW(this->lock.lock());
 46 | 			(*counter)++;
 47 | 			this->lock.unlock();
 48 | 		}
 49 | 	);
 50 | 	
 51 | 	threads[0].join();
 52 | 	ASSERT_EQ(*counter, 1);
 53 | 	delete counter;
 54 | }
 55 | 
 56 | /**
 57 |  * @brief Tests lock functionality with multiple threads
 58 |  */
 59 | TYPED_TEST_P(LockTest, LockSpawnManyThread) {
 60 | 	const int threadcount = 128;
 61 | 	int* counter = new int;
 62 | 	*counter = 0;
 63 | 	std::thread threads[threadcount];
 64 | 	for(auto& t : threads) {
 65 | 		t = std::thread(
 66 | 			[this, counter]() {
 67 | 				ASSERT_NO_THROW(this->lock.lock());
 68 | 				(*counter)++;
 69 | 				this->lock.unlock();
 70 | 			}
 71 | 		);
 72 | 	}
 73 | 	for(auto& t : threads) {	
 74 | 		t.join();
 75 | 	}
 76 | 	ASSERT_EQ(*counter, threadcount);
 77 | 	delete counter;
 78 | }
 79 | REGISTER_TYPED_TEST_SUITE_P(LockTest, LockSpawnThread, LockSpawnManyThread);
 80 | 
 81 | /**
 82 |  * @brief Tests delegation lock functionality
 83 |  */
 84 | TYPED_TEST_P(DelegationTest, DelegateSpawnThread) {
 85 | 	int* counter = new int;
 86 | 	*counter = 0;
 87 | 	std::thread threads[1];
 88 | 	threads[0] = std::thread(
 89 | 		[this, counter]() {
 90 | 			ASSERT_NO_THROW(this->lock.delegate_n([counter]() { (*counter)++; }));
 91 | 		}
 92 | 	);
 93 | 	
 94 | 	threads[0].join();
 95 | 	ASSERT_EQ(*counter, 1);
 96 | 	delete counter;
 97 | }
 98 | 
 99 | /**
100 |  * @brief Tests delegation lock functionality with multiple threads
101 |  */
102 | TYPED_TEST_P(DelegationTest, DelegateSpawnManyThread) {
103 | 	const int threadcount = 128;
104 | 	int* counter = new int;
105 | 	*counter = 0;
106 | 	std::thread threads[threadcount];
107 | 	for(auto& t : threads) {
108 | 		t = std::thread(
109 | 			[this, counter]() {
110 | 				ASSERT_NO_THROW(this->lock.delegate_n([counter]() { (*counter)++; }));
111 | 			}
112 | 		);
113 | 	}
114 | 	for(auto& t : threads) {	
115 | 		t.join();
116 | 	}
117 | 	ASSERT_EQ(*counter, threadcount);
118 | 	delete counter;
119 | }
120 | REGISTER_TYPED_TEST_SUITE_P(DelegationTest, DelegateSpawnThread, DelegateSpawnManyThread);
121 | 


--------------------------------------------------------------------------------
/qd_condition_variable.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef qd_condition_variable_hpp
 2 | #define qd_condition_variable_hpp qd_condition_variable_hpp
 3 | 
 4 | #include "util/pause.hpp"
 5 | #include "qdlock_base.hpp"
 6 | 
 7 | template<class MLock, class DQueue>
 8 | class qd_condition_variable_impl : private qdlock_base<MLock, DQueue> {
 9 | 	typedef qdlock_base<MLock, DQueue> base;
10 | 	public:
11 | 		qd_condition_variable_impl() {
12 | 			this->delegation_queue.open();
13 | 		}
14 | 		qd_condition_variable_impl(const qd_condition_variable_impl&) = delete;
15 | 		qd_condition_variable_impl& operator=(const qd_condition_variable_impl&) = delete;
16 | 
17 | 		/* TODO: these notify implementations / flush implementations run at risk of deadlocking
18 | 		 *       when the notifying thread becomes a helper and also needs to perform additional
19 | 		 *       synchronization steps.
20 | 		 */
21 | 		void notify_one() {
22 | 			this->mutex_lock.lock();
23 | 			this->delegation_queue.flush_one();
24 | 			this->mutex_lock.unlock();
25 | 		}
26 | 		void notify_all() {
27 | 			this->mutex_lock.lock();
28 | 			this->delegation_queue.flush();
29 | 			this->mutex_lock.unlock();
30 | 		}
31 | 		
32 | 		/* interface _p functions: User provides a promise, which is used explicitly by the delegated (void) function */
33 | 		template<typename Function, Function f, typename Lock, typename Promise, typename... Ps>
34 | 		auto wait_redelegate_p(Lock* l, Promise&& result, Ps&&... ps)
35 | 		-> void
36 | 		{
37 | 			wait_redelegate<Function, f, Lock, Promise, Ps...>(l, std::forward<Promise>(result), std::forward<Ps>(ps)...);
38 | 		}
39 | 		template<typename Function, typename Lock, typename Promise, typename... Ps>
40 | 		auto wait_redelegate_p(Function&& f, Lock* l, Promise&& result, Ps&&... ps)
41 | 		-> void
42 | 		{
43 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
44 | 			wait_redelegate<std::nullptr_t, nullptr, Lock, Promise, Function, Ps...>(l, std::forward<Promise>(result), std::forward<Function>(f), std::forward<Ps>(ps)...);
45 | 		}
46 | 	private:
47 | 		
48 | 		
49 | 		template<typename Function, Function f, typename Lock, typename Promise, typename... Ps>
50 | 		auto wait_redelegate(Lock* l, Promise&& result, Ps&&... ps)
51 | 		-> void
52 | 		{
53 | 			while(true) {
54 | 				/* TODO enqueue a function that re-delegates the provided function with its parameter to Lock l TODO */
55 | 				std::nullptr_t no_promise;
56 | 				if(this->template enqueue<void (*)(Lock*, typename Promise::promise&&, Ps&&...), redelegate<Lock, Function, f, Promise&&, Ps&&...>>(&no_promise, &l, &result, (&ps)...)) {
57 | 					return;
58 | 				}
59 | 				qd::pause();
60 | 			}
61 | 		}
62 | 		template<typename Lock, typename Function, Function f, typename Promise, typename... Ps>
63 | 		static void redelegate(Lock* l, Promise&& p, Ps&&... ps) {
64 | 			using no_promise = typename base::no_promise::promise;
65 | 			l->template delegate<Function, f, no_promise, typename Lock::reader_indicator_t, Promise, Ps...>(nullptr, std::forward<Promise>(p), std::forward<Ps>(ps)...);
66 | 	}
67 | 
68 | 	/* TODO: wait_for and wait_until for time-based waiting */
69 | 
70 | };
71 | 
72 | #endif
73 | 


--------------------------------------------------------------------------------
/padded.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef padded_hpp
  2 | #define padded_hpp
  3 | 
  4 | template<typename T, unsigned int PADSIZE, bool T_is_class>
  5 | class padded_base;
  6 | 
  7 | template<typename T, unsigned int PADSIZE>
  8 | class padded_base<T, PADSIZE, false> {
  9 | 	T value;
 10 | 	char padding[PADSIZE - (sizeof(T)%PADSIZE)];
 11 | 	typedef padded_base<T, PADSIZE, false> P_type;
 12 | 	typedef typename std::remove_pointer<T>::type T_dereferenced;
 13 | 	friend class padded_base<T*, PADSIZE, false>;
 14 | 	public:
 15 | 		padded_base() {} /* this is a basic type, it is NOT initialized to 0 */
 16 | 		padded_base(const T v) : value(v) {}
 17 | 		padded_base(const P_type& v) : value(v) {}
 18 | 		padded_base(padded_base<T_dereferenced, PADSIZE, true>* const v) : value(v) {}
 19 | 		padded_base(padded_base<T_dereferenced, PADSIZE, false>* const v) : value(&v->value) {}
 20 | 
 21 | 		operator T&() {
 22 | 			return value;
 23 | 		}
 24 | 		operator const T&() const {
 25 | 			return value;
 26 | 		}
 27 | 		P_type& operator=(P_type other) {
 28 | 			swap(*this, other);
 29 | 			return *this;
 30 | 		}
 31 | 		P_type& operator=(T other) {
 32 | 			using std::swap;
 33 | 			swap(value, other);
 34 | 			return *this;
 35 | 		}
 36 | 
 37 | 		bool operator==(const T other) {
 38 | 			return value == other;
 39 | 		}
 40 | 		bool operator!=(const T other) {
 41 | 			return !(*this == other);
 42 | 		}
 43 | 		T_dereferenced& operator*() {
 44 | 			return *value;
 45 | 		}
 46 | 		const T_dereferenced& operator*() const {
 47 | 			return *value;
 48 | 		}
 49 | 		T_dereferenced* operator->() {
 50 | 			return value;
 51 | 		}
 52 | 		const T_dereferenced* operator->() const {
 53 | 			return value;
 54 | 		}
 55 | 
 56 | 		T& get() {
 57 | 			return value;
 58 | 		}
 59 | 
 60 | 		const T& get() const {
 61 | 			return value;
 62 | 		}
 63 | 
 64 | 		friend void swap(P_type& first, P_type& second) {
 65 | 			using std::swap;
 66 | 			swap(first.value, second.value);
 67 | 		}
 68 | };
 69 | 
 70 | template<typename T, unsigned int PADSIZE>
 71 | class padded_base<T, PADSIZE, true> : public T {
 72 | 	char padding[PADSIZE - (sizeof(T)%PADSIZE)];
 73 | 	//typedef padded_base<T, PADSIZE, true> P_type;
 74 | 	//typedef typename std::remove_pointer<T>::type T_dereferenced;
 75 | 	//friend class padded_base<T*, PADSIZE, false>;
 76 | 	public:
 77 | 		using T::T;
 78 | 		using T::operator=;
 79 | //		T_dereferenced& operator*() {
 80 | //			return **this;
 81 | //		}
 82 | //		const T_dereferenced& operator*() const {
 83 | //			return *value;
 84 | //		}
 85 | //		T_dereferenced* operator->() {
 86 | //			return value;
 87 | //		}
 88 | //		const T_dereferenced* operator->() const {
 89 | //			return value;
 90 | //		}
 91 | //
 92 | 		T& get() {
 93 | 			return *this;
 94 | 		}
 95 | 
 96 | 		const T& get() const {
 97 | 			return *this;
 98 | 		}
 99 | };
100 | 
101 | template<typename T, int PADSIZE = 128>
102 | class padded : public padded_base<T, PADSIZE, std::is_class<T>::value> {
103 | 	typedef padded_base<T, PADSIZE, std::is_class<T>::value> base_type;
104 | 	public:
105 | 		using base_type::base_type;
106 | 		using base_type::operator=;
107 | };
108 | 
109 | #endif // padded_hpp
110 | 


--------------------------------------------------------------------------------
/tests/gentests.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | 
 3 | import itertools
 4 | 
 5 | atomic_instr = ["atomic_instruction_policy_t::use_fetch_and_add", "atomic_instruction_policy_t::use_compare_and_swap"]
 6 | 
 7 | starvation = ["starvation_policy_t::starvation_free", "starvation_policy_t::may_starve"]
 8 | locks = [
 9 | 	"extended_lock<pthreads_lock>",
10 | 	"extended_lock<tatas_lock>",
11 | 	"mcs_lock",
12 | 	"extended_lock<mutex_lock>",
13 | 	"futex_lock",
14 | 	"mcs_futex_lock",
15 | 	"ticket_futex_lock"
16 | 	]
17 | 
18 | buffer_q = ["buffer_queue<{0}>".format(x) for x in [262144, 262139]]
19 | entry_q = ["entry_queue<{0}, {1}>".format(x,y) for (x,y) in itertools.product([4096], [32])]
20 | dual_buffer_q = ["dual_buffer_queue<{0}, {1}, {2}>".format(x,y,z) for ((x,y),z) in itertools.product([(16384, 7), (16384, 8), (16000, 7), (16000, 8), (4096, 32), (6144, 24)], atomic_instr)]
21 | 
22 | #queues = ["simple_locked_queue"]
23 | queues = []
24 | queues.extend(buffer_q)
25 | queues.extend(entry_q)
26 | queues.extend(dual_buffer_q)
27 | 
28 | 
29 | #SQT = ["qdlock_impl<{0}, {1}, {2}>".format(L, Q, S) for (L,Q,S) in itertools.product(locks, ["simple_locked_queue"], starvation)]
30 | BQT = ["qdlock_impl<{0}, {1}, {2}>".format(L, Q, S) for (L,Q,S) in itertools.product(locks, buffer_q, starvation)]
31 | EQT = ["qdlock_impl<{0}, {1}, {2}>".format(L, Q, S) for (L,Q,S) in itertools.product(locks, entry_q, starvation)]
32 | DBQT = ["qdlock_impl<{0}, {1}, {2}>".format(L, Q, S) for (L,Q,S) in itertools.product(locks, dual_buffer_q, starvation)]
33 | 
34 | #for q in queues:
35 | #    print("{0}, \\".format(q))
36 | 
37 | #print("#define QDTypes \\\nqdlock, \\")
38 | #print(", \\\n".join(qd_impls))
39 | 
40 | lock_typelists = [
41 | 	("StandardLocks", locks),
42 | #	("SimpleQueue", SQT),
43 | 	("BufferQueue", BQT), 
44 | 	("EntryQueue", EQT), 
45 | 	("DualBufferQueue", DBQT) 
46 | 	]
47 | delegation_typelists = [
48 | #	("SimpleQueue", SQT),
49 | 	("BufferQueue", BQT), 
50 | 	("EntryQueue", EQT), 
51 | 	("DualBufferQueue", DBQT) 
52 | 	]
53 | 
54 | def split_typelists(typelists): 
55 | 	splitpoint = 16
56 | 	split_typelists = []
57 | 	for (name, types) in typelists:
58 | 		i = 0
59 | 		while len(types) > splitpoint:
60 | 			split_typelists.append(("{0}{1}".format(name, str(i)), types[:splitpoint]))
61 | 			types = types[splitpoint:]
62 | 			i = i + 1
63 | 		if i > 0:
64 | 			name = "{0}{1}".format(name, str(i))
65 | 		split_typelists.append((name, types))
66 | 	return split_typelists
67 | 
68 | 
69 | split_locks = split_typelists(lock_typelists)
70 | split_delegate = split_typelists(delegation_typelists)
71 | 
72 | filelimit = 8
73 | cnt = 0
74 | for i in range(filelimit):
75 | 	fname = "generated_test{0}.cpp".format(str(i))
76 | 	f = open(fname, 'w');
77 | 	print("#include<lock.hpp>", file=f)
78 | 	f.close()
79 | 
80 | def roundrobin_tests(tests, split_typelists):
81 | 	global cnt
82 | 	for (name, types) in split_typelists:
83 | 		fname = "generated_test{0}.cpp".format(str(cnt % filelimit))
84 | 		f = open(fname, 'a');
85 | 		if len(types) > 50:
86 | 			raise "Too many types for a single instance!"
87 | 		print("typedef ::testing::Types <", file=f)
88 | 		print(", \n".join(types), file=f)
89 | 		print("> GeneratedTypes{0};".format(str(cnt)), file=f)
90 | 		print("INSTANTIATE_TYPED_TEST_SUITE_P({1}, {0}, GeneratedTypes{2});".format(tests, name, str(cnt)), file=f)
91 | 		cnt = cnt + 1
92 | 		f.close()
93 | 
94 | roundrobin_tests("LockTest", split_locks)
95 | roundrobin_tests("DelegationTest", split_delegate)
96 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | C++ QD Locking Library
  2 | ======================
  3 | [![Build Status](https://github.com/davidklaftenegger/qd_library/workflows/CI/badge.svg)](https://github.com/davidklaftenegger/qd_library/actions)
  4 | 
  5 | This is the C++ QD Locking library, which provides implementations of Queue Delegation
  6 | (QD) locks for C++11 programs as described in the following two publications:
  7 | 
  8 |  1. David Klaftenegger, Konstantinos Sagonas, and Kjell Winblad.
  9 |     [Queue Delegation Locking](https://ieeexplore.ieee.org/document/8093701).
 10 |     In IEEE Transactions on Parallel and Distributed Systems, 29(3):687-704,
 11 |     March 2018. IEEE.
 12 |     DOI: [10.1109/TPDS.2017.2767046](https://doi.org/10.1109/TPDS.2017.2767046)
 13 | 
 14 |  2. David Klaftenegger, Konstantinos Sagonas, and Kjell Winblad.
 15 |     [Delegation Locking Libraries for Improved Performance of Multithreaded
 16 |      Programs](https://link.springer.com/chapter/10.1007/978-3-319-09873-9_48).
 17 |     In Euro-Par 2014, Proceedings of the 20th International Conference,
 18 |     pp. 572-583, Volume 8632 in LNCS, August 2014. Springer.
 19 |     DOI: [10.1007/978-3-319-09873-9_48](https://doi.org/10.1007/978-3-319-09873-9_48)
 20 | 
 21 | The QD Locking library is distributed under the Simplified BSD License.
 22 | Details can be found in the LICENSE file.
 23 | 
 24 | QD locks allow programmers to *delegate* critical sections to a lock. If the
 25 | lock is currently free, this will use an efficient lock implementation.
 26 | However, if the lock is already in use, the critical section will be delegated
 27 | to the thread currently holding the lock, who becomes the *helper*. The helper
 28 | will continue to execute critical sections for other threads until either it
 29 | finds no more work, or some maximum number of executions has been reached.
 30 | 
 31 | This scheme results in drastically higher throughput than with traditional
 32 | locking algorithms that transfer execution and shared data between threads.
 33 | 
 34 | For more details on QD locks see [this website](https://www.it.uu.se/research/group/languages/software/qd_lock_lib).
 35 | 
 36 | Requirements
 37 | ------------
 38 | 
 39 | This library makes use of [libnuma](https://man7.org/linux/man-pages/man3/numa.3.html), available on Linux.
 40 | For other operating systems support can be turned off, at potential performance costs for NUMA machines.
 41 | To install libnuma on Debian/Ubuntu and derivatives, run
 42 | ```
 43 | sudo apt-get install libnuma-dev
 44 | ```
 45 | For gentoo, run
 46 | ```
 47 | sudo emerge sys-process/numactl
 48 | ```
 49 | 
 50 | The following compilers are currently supported:
 51 |  * GCC: g++ versions 4.9 -- 10.2.1
 52 |  * Clang: clang++ versions 3.8 -- 11.0.1
 53 | 
 54 | Installation
 55 | ------------
 56 | 
 57 | ```
 58 | mkdir build
 59 | cd build
 60 | cmake ../
 61 | make -j8
 62 | make test
 63 | make install
 64 | ```
 65 | 
 66 | Usage
 67 | -----
 68 | 
 69 | To use this library, include `qd.hpp` in your programs.
 70 | 
 71 | Here is a minimal example for how to use them:
 72 | 
 73 | ```c++
 74 | #include<iostream>
 75 | #include<qd.hpp>
 76 | int main() {
 77 | 	int counter = 0;
 78 | 	qdlock lock;
 79 | 	std::thread threads[4];
 80 | 	for(auto& t : threads) {
 81 | 		t = std::thread(
 82 | 			[&lock, &counter]() {
 83 | 				lock.delegate_n([&counter]() { counter++; });
 84 | 			}
 85 | 		);
 86 | 	}
 87 | 	for(auto& t : threads) {
 88 | 	  t.join();
 89 | 	}
 90 | 	std::cout << "The counter value is " << counter << std::endl;
 91 | }
 92 | ```
 93 | 
 94 | To compile and run a program it may be required to add `/usr/local/include/` to your include path and `/usr/local/lib` to your library path.
 95 | Compiling and running should then work using
 96 | ```
 97 | g++ -I/usr/local/include/qd/ -L/usr/local/lib/ -Wl,-rpath=/usr/local/lib/ myprogram.cpp -pthread -lqd -o myprogram
 98 | ./myprogram
 99 | ```
100 | 
101 | Advanced Configuration
102 | ----------------------
103 | 
104 | If required, you can adjust some settings in CMake:
105 | ```
106 | cmake -DQD_DEBUG=OFF                     \
107 |       -DQD_TESTS=ON                      \
108 |       -DQD_USE_LIBNUMA=ON                \
109 |       -DCMAKE_INSTALL_PREFIX=/usr/local  \
110 |       ../
111 | ```
112 | 
113 | C library
114 | ---------
115 | 
116 | For a library for C code, please see https://github.com/kjellwinblad/qd_lock_lib
117 | 


--------------------------------------------------------------------------------
/locks/mcs_lock.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef qd_mcs_lock_hpp
  2 | #define qd_mcs_lock_hpp qd_mcs_lock_hpp
  3 | 
  4 | #include<atomic>
  5 | #include<map>
  6 | #include<unistd.h>
  7 | #include<stack>
  8 | #include<sys/param.h>
  9 | #include<thread>
 10 | #include "util/pause.hpp"
 11 | 
 12 | struct mcs_node2 {
 13 | 	enum field_t { free, taken, contended };
 14 | 	std::atomic<field_t> is_locked;
 15 | 	std::atomic<mcs_node2*> next;
 16 | 	mcs_node2() : next(nullptr) {}
 17 | };
 18 | 
 19 | /**
 20 |  * @brief a MCS based lock with futex functionality
 21 |  * @details This lock cooperates with the Linux kernel when there is no chance on taking the lock.
 22 |  *          The sleep-waiting can be triggered by lock() and try_lock_or_wait().
 23 |  *          When the lock cannot be taken these two functions wait until the Linux kernel wakes them up again,
 24 |  *          which is triggered by unlock() operations.
 25 |  * @note lock() and unlock() function largely like an MCS lock,
 26 |  *       while try_lock() and try_lock_or_wait() are more similar to the futex lock.
 27 |  */
 28 | class mcs_lock {
 29 | 	using mcs_node = ::mcs_node2;
 30 | 	using field_t = mcs_node*;
 31 | 	std::atomic<field_t> locked;
 32 | 	thread_local static std::map<mcs_lock*, mcs_node> mcs_node_store;
 33 | 	public:
 34 | 		mcs_lock() : locked(nullptr) {}
 35 | 		mcs_lock(mcs_lock&) = delete; /* TODO? */
 36 | 
 37 | 		/**
 38 | 		 * @brief take the lock
 39 | 		 * @details This function will issue a futex wait syscall
 40 | 		 *          while waiting for the lock.
 41 | 		 */
 42 | 		void lock() {
 43 | 			mcs_node* mynode = &mcs_node_store[this];
 44 | 			mcs_node* found = this->locked.exchange(mynode, std::memory_order_acq_rel);
 45 | 			if(found != nullptr) {
 46 | 				mynode->is_locked.store(mcs_node::taken, std::memory_order_release);
 47 | 				found->next = mynode;
 48 | 				for(int i = 0; i < 512; i++) {
 49 | 					if(mynode->is_locked.load(std::memory_order_acquire) == mcs_node::free) {
 50 | 						break;
 51 | 					}
 52 | 					qd::pause();
 53 | 				}
 54 | 				while(mynode->is_locked.load(std::memory_order_acquire) != mcs_node::free) {
 55 | 				}
 56 | 			}
 57 | 		}
 58 | 
 59 | 		/**
 60 | 		 * @brief release the lock
 61 | 		 * @details This will unlock the lock. If there are sleeping threads,
 62 | 		 *          they will also be woken up.
 63 | 		 */
 64 | 		void unlock() {
 65 | 			mcs_node* mynode = &mcs_node_store[this];
 66 | 			/* thread_local mynode is set while locked */
 67 | 			field_t c = mynode;
 68 | 			if(mynode->next == nullptr) {
 69 | 				if(this->locked.compare_exchange_strong(c, nullptr, std::memory_order_acq_rel)) {
 70 | 					return;
 71 | 				} else if(c == reinterpret_cast<field_t>(mynode)) {
 72 | 					unlock();
 73 | 					return;
 74 | 				}
 75 | 			}
 76 | 			while(mynode->next == nullptr) {
 77 | 				qd::pause();
 78 | 				/* wait for nextpointer */
 79 | 			}
 80 | 			mcs_node* next = mynode->next.load(); /* TODO */
 81 | 			next->is_locked.store(mcs_node::free, std::memory_order_release);
 82 | 
 83 | 			mynode->next.store(nullptr, std::memory_order_relaxed);
 84 | 		}
 85 | 
 86 | 		/**
 87 | 		 * @brief non-blocking trylock.
 88 | 		 * @return true iff the lock has been taken
 89 | 		 */
 90 | 		bool try_lock() {
 91 | 			if(this->is_locked()) {
 92 | 				return false;
 93 | 			}
 94 | 			field_t c = nullptr;
 95 | 			mcs_node* mynode = &mcs_node_store[this];
 96 | 			bool success = this->locked.compare_exchange_strong(c, reinterpret_cast<field_t>(mynode), std::memory_order_acq_rel);
 97 | 			return success;
 98 | 		}
 99 | 
100 | 		/**
101 | 		 * @brief blocking trylock.
102 | 		 * @return true iff the lock has been taken,
103 | 		 *         false after waiting for an unlock() operation on the lock.
104 | 		 */
105 | 		bool try_lock_or_wait() {
106 | 			return try_lock();
107 | 		}
108 | 
109 | 		/**
110 | 		 * @brief check lock state
111 | 		 * @return true iff the lock is taken, false otherwise
112 | 		 */
113 | 		bool is_locked() {
114 | 			return locked.load(std::memory_order_acquire) != nullptr;
115 | 		}
116 | 		void wake() {}
117 | 	private:
118 | 		static_assert(sizeof(locked) == sizeof(field_t), "std::atomic<field_t> size differs from size of field_t. Your architecture does not support the MCS futex_lock");
119 | 		static_assert(__BYTE_ORDER == __LITTLE_ENDIAN, "Your architecture's endianness is currently not supported. Please report this as a bug.");
120 | };
121 | 
122 | #endif /* qd_mcs_lock_hpp */
123 | 


--------------------------------------------------------------------------------
/queues/entry_queue.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef qd_entry_queue_hpp
  2 | #define qd_entry_queue_hpp qd_entry_queue_hpp
  3 | 
  4 | #include<algorithm>
  5 | #include<array>
  6 | #include<atomic>
  7 | #include<iostream>
  8 | #include<typeinfo>
  9 | 
 10 | /**
 11 |  * @brief a buffer-based tantrum queue with fixed-size entries
 12 |  * @tparam ENTRIES the number of entries
 13 |  */
 14 | template<long ENTRIES, int BUFFER_SIZE>
 15 | class entry_queue {
 16 | 	/** type for the size field for queue entries, loads must not be optimized away in flush */
 17 | 	typedef std::atomic<long> sizetype;
 18 | 
 19 | 	/** type for function pointers to be stored in this queue */
 20 | 	typedef void(*ftype)(char*);
 21 | 
 22 | 	struct entry_t {
 23 | 		std::atomic<ftype> fun;
 24 | 		char buf[BUFFER_SIZE];
 25 | 	};
 26 | 	void forwardall(long, long) {};
 27 | 	template<typename P, typename... Ts>
 28 | 	void forwardall(long idx, long offset, P&& p, Ts&&... ts) {
 29 | 		auto ptr = reinterpret_cast<P*>(&entry_array[idx].buf[offset]);
 30 | 		new (ptr) P(std::forward<P>(p));
 31 | 		forwardall(idx, offset+sizeof(p), std::forward<Ts>(ts)...);
 32 | 	}
 33 | 	public:
 34 | 		/** constants for current state of the queue */
 35 | 		enum class status : long { OPEN=0, SUCCESS=0, FULL, CLOSED };
 36 | 
 37 | 		entry_queue() : counter(ENTRIES), closed(status::CLOSED) {}
 38 | 		/** opens the queue */
 39 | 		void open() {
 40 | 			counter.store(0, std::memory_order_relaxed);
 41 | 			closed.store(status::OPEN, std::memory_order_relaxed);
 42 | 		}
 43 | 
 44 | 		/**
 45 | 		 * @brief enqueues an entry
 46 | 		 * @tparam P return type of associated function
 47 | 		 * @param op wrapper function for associated function
 48 | 		 * @return SUCCESS on successful storing in queue, FULL if the queue is full and CLOSED if the queue is closed explicitly
 49 | 		 */
 50 | 		template<typename... Ps>
 51 | 		status enqueue(void (*op)(char*), Ps*... ps) {
 52 | 			auto current_status = closed.load(std::memory_order_relaxed);
 53 | 			if(current_status != status::OPEN) {
 54 | 				return current_status;
 55 | 			}
 56 | 			/* entry size = size of size + size of wrapper functor + size of promise + size of all parameters*/
 57 | 			constexpr long size = sumsizes<Ps...>::size;
 58 | 			/* get memory in buffer */
 59 | 			long index = counter.fetch_add(1, std::memory_order_relaxed);
 60 | 			if(index < ENTRIES) {
 61 | 				static_assert(size <= BUFFER_SIZE, "entry_queue buffer per entry too small.");
 62 | 				/* entry available: move op, p and parameters to buffer, then set size of entry */
 63 | 				forwardall(index, 0, std::move(*ps)...);
 64 | 				entry_array[index].fun.store(op, std::memory_order_release);
 65 | 				return status::SUCCESS;
 66 | 			} else {
 67 | 				return status::FULL;
 68 | 			}
 69 | 		}
 70 | 
 71 | 		/** execute all stored operations, leave queue in closed state */
 72 | 		void flush() {
 73 | 			long todo = 0;
 74 | 			bool open = true;
 75 | 			while(open) {
 76 | 				long done = todo;
 77 | 				todo = counter.load(std::memory_order_relaxed);
 78 | 				if(todo == done) { /* close queue */
 79 | 					todo = counter.exchange(ENTRIES, std::memory_order_relaxed);
 80 | 					closed.store(status::CLOSED, std::memory_order_relaxed);
 81 | 					open = false;
 82 | 				}
 83 | 				if(todo >= static_cast<long>(ENTRIES)) { /* queue closed */
 84 | 					todo = ENTRIES;
 85 | 					closed.store(status::CLOSED, std::memory_order_relaxed);
 86 | 					open = false;
 87 | 				}
 88 | 				for(long index = done; index < todo; index++) {
 89 | 					/* synchronization on entry size field: 0 until entry available */
 90 | 					ftype fun = nullptr;
 91 | 					do {
 92 | 						fun = entry_array[index].fun.load(std::memory_order_acquire);
 93 | 					} while(!fun);
 94 | 
 95 | 					/* call functor with pointer to promise (of unknown type) */
 96 | 					fun(&entry_array[index].buf[0]);
 97 | 
 98 | 					/* cleanup: call destructor of (now empty) functor and clear buffer area */
 99 | //					fun->~ftype();
100 | 					entry_array[index].fun.store(nullptr, std::memory_order_relaxed);
101 | 				}
102 | 			}
103 | 		}
104 | 	private:
105 | 		/** counter for how many entries are already in use */
106 | 		std::atomic<long> counter;
107 | 		char pad[128];
108 | 		/** optimization flag: no writes when queue in known-closed state */
109 | 		std::atomic<status> closed;
110 | 		char pad2[128];
111 | 		/** the buffer for entries to this queue */
112 | 		std::array<entry_t, ENTRIES> entry_array;
113 | };
114 | 
115 | #endif /* qd_buffer_queue_hpp */
116 | 


--------------------------------------------------------------------------------
/locks/futex_lock.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef qd_futex_lock_hpp
  2 | #define qd_futex_lock_hpp qd_futex_lock_hpp
  3 | 
  4 | #include<atomic>
  5 | #include<unistd.h>
  6 | #include<linux/futex.h>
  7 | #include<sys/syscall.h>
  8 | 
  9 | /**
 10 |  * @brief a futex based lock
 11 |  * @details This lock cooperates with the Linux kernel when there is no chance on taking the lock.
 12 |  *          The sleep-waiting can be triggered by lock() and try_lock_or_wait().
 13 |  *          When the lock cannot be taken these two functions wait until the Linux kernel wakes them up again,
 14 |  *          which is triggered by unlock() operations.
 15 |  * @note lock() and unlock() are taken from mutex in http://www.akkadia.org/drepper/futex.pdf with additional documentation used from http://locklessinc.com/articles/mutex_cv_futex/
 16 |  */
 17 | class futex_lock {
 18 | 	enum field_t { free, taken, contended };
 19 | 	std::atomic<field_t> locked;
 20 | 	public:
 21 | 		futex_lock() : locked(free) {}
 22 | 		futex_lock(futex_lock&) = delete; /* TODO? */
 23 | 
 24 | 		/**
 25 | 		 * @brief take the lock
 26 | 		 * @details This function will issue a futex wait syscall
 27 | 		 *          while waiting for the lock.
 28 | 		 */
 29 | 		void lock() {
 30 | 			field_t c = free;
 31 | 			if(!locked.compare_exchange_strong(c, taken)) {
 32 | 				if(c != contended) {
 33 | 					c = locked.exchange(contended);
 34 | 				}
 35 | 				while(c != free) {
 36 | 					wait();
 37 | 					c = locked.exchange(contended);
 38 | 				}
 39 | 			}
 40 | 		}
 41 | 
 42 | 		/**
 43 | 		 * @brief release the lock
 44 | 		 * @details This will unlock the lock. If there are sleeping threads,
 45 | 		 *          they will also be woken up.
 46 | 		 */
 47 | 		void unlock() {
 48 | 			auto old = locked.exchange(free, std::memory_order_release);
 49 | 			if(old == contended) {
 50 | 				notify_all(); // notify_one instead?
 51 | 			}
 52 | 		}
 53 | 
 54 | 		/**
 55 | 		 * @brief non-blocking trylock.
 56 | 		 * @return true iff the lock has been taken
 57 | 		 */
 58 | 		bool try_lock() {
 59 | 			if(this->is_locked()) {
 60 | 				return false;
 61 | 			}
 62 | 			field_t c = free;
 63 | 			return locked.compare_exchange_strong(c, taken, std::memory_order_acq_rel, std::memory_order_relaxed);
 64 | 		}
 65 | 
 66 | 		/**
 67 | 		 * @brief blocking trylock.
 68 | 		 * @return true iff the lock has been taken,
 69 | 		 *         false after waiting for an unlock() operation on the lock.
 70 | 		 */
 71 | 		bool try_lock_or_wait() {
 72 | 			field_t c = locked.load(std::memory_order_acquire);
 73 | 			if(c == free) {
 74 | 				if(locked.compare_exchange_strong(c, taken, std::memory_order_acq_rel, std::memory_order_relaxed)) {
 75 | 					return true;
 76 | 				}
 77 | 			}
 78 | 			/* not free (or CAS failed because no longer free) */
 79 | 			if(c != contended) {
 80 | 				c = locked.exchange(contended);
 81 | 				if(c == free) {
 82 | 					return true;
 83 | 				}
 84 | 			}
 85 | 			/* contended (or was taken and swapped to contended) */
 86 | 			wait();
 87 | 			return false;
 88 | 		}
 89 | 
 90 | 		/**
 91 | 		 * @brief check lock state
 92 | 		 * @return true iff the lock is taken, false otherwise
 93 | 		 */
 94 | 		bool is_locked() {
 95 | 			return locked.load(std::memory_order_acquire) != free;
 96 | 		}
 97 | 
 98 | 	private:
 99 | 		/** @brief sleep until unlock() is called */
100 | 		void wait() {
101 | 			sys_futex(&locked, FUTEX_WAIT, contended, NULL, NULL, 0);
102 | 		}
103 | 
104 | 		/** @brief wake one thread from sleeping that waits on this lock */
105 | 		void notify_one() {
106 | 			sys_futex(&locked, FUTEX_WAKE, 1, NULL, NULL, 0);
107 | 		}
108 | 
109 | 		/** @brief wake all sleeping threads that wait on this lock */
110 | 		void notify_all() {
111 | 			static const int WAKE_AT_ONCE = 32768;
112 | 			int woken;
113 | 			do {
114 | 				woken = sys_futex(&locked, FUTEX_WAKE, WAKE_AT_ONCE, NULL, NULL, 0);
115 | 			} while (woken == WAKE_AT_ONCE);
116 | 		}
117 | 
118 | 		/**
119 | 		 * @brief wrapper for futex syscalls
120 | 		 * @param addr1 the address threads are observing
121 | 		 * @param op either FUTEX_WAIT or FUTEX_WAKE
122 | 		 * @param val1 if op==FUTEX_WAIT then the expected value for *addr1,
123 | 		 *             if op==FUTEX_WAKE then the number of threads to wake up
124 | 		 * @param timeout always NULL
125 | 		 * @param addr2 always NULL
126 | 		 * @param val3 always 0
127 | 		 * @return the return value of the futex syscall
128 | 		 */
129 | 		static long sys_futex(void *addr1, int op, int val1, struct timespec *timeout, void *addr2, int val3)
130 | 		{
131 | 			return syscall(SYS_futex, addr1, op, val1, timeout, addr2, val3);
132 | 		}
133 | 
134 | 		static_assert(sizeof(locked) == sizeof(field_t), "std::atomic<field_t> size differs from size of field_t. Your architecture does not support the futex_lock");
135 | };
136 | 
137 | #endif /* qd_futex_lock_hpp */
138 | 


--------------------------------------------------------------------------------
/queues/dual_buffer_queue.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef qd_dual_buffer_queue_hpp
  2 | #define qd_dual_buffer_queue_hpp qd_dual_buffer_queue_hpp
  3 | 
  4 | #include<algorithm>
  5 | #include<array>
  6 | #include<atomic>
  7 | #include<new>
  8 | #include "util/type_tools.hpp"
  9 | 
 10 | /**
 11 |  * @brief policies for atomic instructions to use
 12 |  */
 13 | enum class atomic_instruction_policy_t {use_fetch_and_add, use_compare_and_swap};
 14 | 
 15 | static inline void no_op(char*) {}
 16 | 
 17 | /**
 18 |  * @brief a entry-based tantrum queue with a fixed buffer for parameters
 19 |  * @tparam ENTRIES the number of entries
 20 |  * @tparam ARRAY_SIZE the buffer's size in bytes
 21 |  */
 22 | template<unsigned ENTRIES, unsigned int ENTRY_SIZE=16, atomic_instruction_policy_t atomic_instruction_policy=atomic_instruction_policy_t::use_fetch_and_add>
 23 | class dual_buffer_queue {
 24 | 	/** type for sizes of queue entries */
 25 | 	typedef unsigned s_type;
 26 | 	/** type for the size field for queue entries, loads must not be optimized away in flush */
 27 | 	typedef std::atomic<char*> sizetype;
 28 | 
 29 | 	/** type for function pointers to be stored in this queue */
 30 | 	typedef void(*ftype)(char*);
 31 | 
 32 | 	public:
 33 | 		/** constants for current state of the queue */
 34 | 		enum class status { OPEN=0, SUCCESS=0, FULL, CLOSED };
 35 | 
 36 | 		dual_buffer_queue() : counter(ENTRIES) {}
 37 | 		/** opens the queue */
 38 | 		void open() {
 39 | 			counter.store(0, std::memory_order_relaxed);
 40 | 		}
 41 | 
 42 | 		void forwardall(unsigned) {};
 43 | 		template<typename P, typename... Ts>
 44 | 		void forwardall(unsigned offset, P&& p, Ts&&... ts) {
 45 | 			auto ptr = reinterpret_cast<P*>(&buffer_array[offset]);
 46 | 			new (ptr) P(std::forward<P>(p));
 47 | 			forwardall(offset+sizeof(p), std::forward<Ts>(ts)...);
 48 | 		}
 49 | 		/**
 50 | 		 * @brief enqueues an entry
 51 | 		 * @tparam P return type of associated function
 52 | 		 * @param op wrapper function for associated function
 53 | 		 * @return SUCCESS on successful storing in queue, FULL if the queue is full and CLOSED if the queue is closed explicitly
 54 | 		 */
 55 | 		template<typename... Ps>
 56 | 		status enqueue(ftype op, Ps*... ps) {
 57 | 			/* entry size = size of size + size of wrapper functor + size of promise + size of all parameters*/
 58 | 			constexpr unsigned size = aligned(sumsizes<Ps...>::size + sizeof(unsigned));
 59 | 			constexpr unsigned req_entries = (size+ENTRY_SIZE-1)/ENTRY_SIZE;
 60 | 			unsigned index = counter.load(std::memory_order_relaxed);
 61 | 			if(index > (ENTRIES - req_entries)) {
 62 | 				return status::CLOSED;
 63 | 			}
 64 | 			/* get memory in buffer */
 65 | 			if(atomic_instruction_policy == atomic_instruction_policy_t::use_fetch_and_add) {
 66 | 				index = counter.fetch_add(req_entries, std::memory_order_relaxed);
 67 | 			} else {
 68 | 				while(!counter.compare_exchange_weak(index, index+req_entries, std::memory_order_relaxed)) {
 69 | 					//qd::pause();
 70 | 				}
 71 | 			}
 72 | 			if(index <= ENTRIES - req_entries) {
 73 | 				/* enough memory available: move op, p and parameters to buffer, then set size of entry */
 74 | 				*reinterpret_cast<unsigned*>(&buffer_array[index*ENTRY_SIZE]) = req_entries;
 75 | 				forwardall(index*ENTRY_SIZE + sizeof(unsigned), std::move(*ps)...);
 76 | 				entries[index].f.store(op, std::memory_order_release);
 77 | 				return status::SUCCESS;
 78 | 			} else {
 79 | 				if(index < ENTRIES) {
 80 | 					*reinterpret_cast<unsigned*>(&buffer_array[index*ENTRY_SIZE]) = ENTRIES-index;
 81 | 					entries[index].f.store(&no_op, std::memory_order_release);
 82 | 				}
 83 | 				return status::FULL;
 84 | 			}
 85 | 		}
 86 | 
 87 | 		/** execute all stored operations, leave queue in closed state */
 88 | 		void flush() {
 89 | 			unsigned todo = 0;
 90 | 			bool open = true;
 91 | 			while(open) {
 92 | 				unsigned done = todo;
 93 | 				todo = counter.load(std::memory_order_relaxed);
 94 | 				if(todo == done) { /* close queue */
 95 | 					todo = counter.exchange(ENTRIES, std::memory_order_relaxed);
 96 | 					open = false;
 97 | 				}
 98 | 				if(todo >= ENTRIES) { /* queue closed */
 99 | 					todo = ENTRIES;
100 | 					open = false;
101 | 				}
102 | 				for(unsigned index = done; index < todo; ) {
103 | 					/* synchronization on entry size field: 0 until entry available */
104 | 					ftype fun = entries[index].f.load(std::memory_order_acquire);
105 | 					while(!fun) {
106 | 						qd::pause();
107 | 						fun = entries[index].f.load(std::memory_order_acquire);
108 | 					};
109 | 					/* call functor with pointer to promise (of unknown type) */
110 | 					unsigned req_entries = *reinterpret_cast<unsigned*>(&buffer_array[index*ENTRY_SIZE]);
111 | 					fun(&buffer_array[index*ENTRY_SIZE + sizeof(req_entries)]);
112 | 					index += req_entries;
113 | 				}
114 | 			}
115 | 			;
116 | 			if(todo > 0) {
117 | 				std::fill(reinterpret_cast<char*>(&entries[0]), reinterpret_cast<char*>(&entries[todo])-1, 0);
118 | 			}
119 | 		}
120 | 	private:
121 | 		/** counter for how much of the buffer is currently in use; offset to free area in buffer_array */
122 | 		std::atomic<s_type> counter;
123 | 		char pad[128];
124 | 		/** optimization flag: no writes when queue in known-closed state */
125 | 		struct entry_t {
126 | 			std::atomic<ftype> f;
127 | 		};
128 | 		std::array<entry_t, ENTRIES> entries;
129 | 		char pad3[128];
130 | 		/** the buffer for entries to this queue */
131 | 		std::array<char, ENTRIES*ENTRY_SIZE> buffer_array;
132 | 
133 | 		static constexpr s_type aligned(s_type size) {
134 | 			return size==0?1:size;
135 | 			//return (size<4*ENTRY_SIZE)?4*ENTRY_SIZE:size;
136 | 			//return ((size + sizeof(sizetype) - 1) / sizeof(sizetype)) * sizeof(sizetype); /* TODO: better way of computing a ceil? */
137 | 		}
138 | };
139 | 
140 | #endif /* qd_dual_buffer_queue_hpp */
141 | 


--------------------------------------------------------------------------------
/locks/ticket_futex_lock.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef qd_ticket_futex_lock_hpp
  2 | #define qd_ticket_futex_lock_hpp qd_futex_lock_hpp
  3 | 
  4 | #include<atomic>
  5 | #include<unistd.h>
  6 | #include<linux/futex.h>
  7 | #include<sys/syscall.h>
  8 | #include<thread>
  9 | #include "util/pause.hpp"
 10 | 
 11 | /**
 12 |  * @brief a ticket based lock with futex functionality
 13 |  * @details This lock cooperates with the Linux kernel when there is no chance on taking the lock.
 14 |  *          The sleep-waiting can be triggered by lock() and try_lock_or_wait().
 15 |  *          When the lock cannot be taken these two functions wait until the Linux kernel wakes them up again,
 16 |  *          which is triggered by unlock() operations.
 17 |  * @note lock() and unlock() function largely like a ticket lock,
 18 |  *       while try_lock() and try_lock_or_wait() are more similar to the futex lock.
 19 |  */
 20 | class ticket_futex_lock {
 21 | 	using field_t = int;
 22 | 	std::atomic<field_t> ticket;
 23 | 	char pad[128-sizeof(ticket)];
 24 | 	std::atomic<field_t> serving;
 25 | 	public:
 26 | 		ticket_futex_lock() : ticket(0), serving(0) {}
 27 | 		ticket_futex_lock(ticket_futex_lock&) = delete; /* TODO? */
 28 | 
 29 | 		/**
 30 | 		 * @brief take the lock
 31 | 		 * @details This function will issue a futex wait syscall
 32 | 		 *          while waiting for the lock.
 33 | 		 */
 34 | 		void lock() {
 35 | 			field_t mynum = ticket.fetch_add(2, std::memory_order_relaxed);
 36 | 			field_t current;
 37 | 			current = serving.load(std::memory_order_acquire);
 38 | 			while((current&~1) != mynum ) {
 39 | 				if((current & 1) != 1) {
 40 | 					if(!serving.compare_exchange_strong(current, current|1, std::memory_order_acq_rel)) {
 41 | 						continue;
 42 | 					} else {
 43 | 						current |= 1;
 44 | 					}
 45 | 				}
 46 | 				wait(current);
 47 | 				current = serving.load(std::memory_order_acquire);
 48 | 			}
 49 | 		}
 50 | 
 51 | 		/**
 52 | 		 * @brief release the lock
 53 | 		 * @details This will unlock the lock. If there are sleeping threads,
 54 | 		 *          they will also be woken up.
 55 | 		 */
 56 | 		void unlock() {
 57 | 			field_t mine = serving.load(std::memory_order_relaxed);
 58 | 			field_t current = serving.exchange((mine&~1)+2, std::memory_order_release);
 59 | 			if((current & 1) == 1) {
 60 | 				notify_all();
 61 | 			}
 62 | 		}
 63 | 
 64 | 		/**
 65 | 		 * @brief non-blocking trylock.
 66 | 		 * @return true iff the lock has been taken
 67 | 		 */
 68 | 		bool try_lock() {
 69 | 			field_t current_ticket = ticket.load(std::memory_order_relaxed);
 70 | 			field_t current_serving = serving.load(std::memory_order_acquire);
 71 | 			if(current_ticket != current_serving) {
 72 | 					return false;
 73 | 			}
 74 | 			return ticket.compare_exchange_strong(current_ticket, current_ticket+2);
 75 | 		}
 76 | 
 77 | 		/**
 78 | 		 * @brief blocking trylock.
 79 | 		 * @return true iff the lock has been taken,
 80 | 		 *         false after waiting for an unlock() operation on the lock.
 81 | 		 */
 82 | 		bool try_lock_or_wait() {
 83 | 			field_t current_ticket = ticket.load(std::memory_order_relaxed);
 84 | 			field_t current_serving = serving.load(std::memory_order_acquire);
 85 | 			field_t flag = current_serving & 1;
 86 | 			if((current_ticket|flag) == current_serving) {
 87 | 				if(ticket.compare_exchange_strong(current_ticket, current_ticket+2)) {
 88 | 					return true;
 89 | 				}
 90 | 			}
 91 | 			if(flag != 1) {
 92 | 				field_t flagged = current_serving|1;
 93 | 				if(serving.compare_exchange_strong(current_serving, flagged)) {
 94 | 						current_serving = flagged;
 95 | 				} else {
 96 | 						/* someone called unlock, no need to wait */
 97 | 						return false;
 98 | 				}
 99 | 			}
100 | 			wait(current_serving);
101 | 			return false;
102 | 		}
103 | 
104 | 		/**
105 | 		 * @brief check lock state
106 | 		 * @return true iff the lock is taken, false otherwise
107 | 		 */
108 | 		bool is_locked() {
109 | 			return ticket.load(std::memory_order_relaxed) != serving.load(std::memory_order_acquire);
110 | 		}
111 | 
112 | 	private:
113 | 		/**
114 | 		 * @brief sleep until unlock() is called
115 | 		 * @param addr the memory address to wait for a change on
116 | 		 * @param expected currently expected value at addr
117 | 		 */
118 | 		void wait(int expected) {
119 | 			sys_futex(&serving, FUTEX_WAIT, expected, NULL, NULL, 0);
120 | 		}
121 | 
122 | 		/**
123 | 		 * @brief wake one thread from sleeping that waits on this lock
124 | 		 * @param addr the memory address to wait for a change on
125 | 		 */
126 | 		void notify_one() {
127 | 			sys_futex(&serving, FUTEX_WAKE, 1, NULL, NULL, 0);
128 | 		}
129 | 
130 | 		/**
131 | 		 * @brief wake all sleeping threads that wait on this lock
132 | 		 * @param addr the memory address to wait for a change on
133 | 		 */
134 | 		void notify_all() {
135 | 			static const int WAKE_AT_ONCE = 32768;
136 | 			int woken;
137 | 			do {
138 | 				woken = sys_futex(&serving, FUTEX_WAKE, WAKE_AT_ONCE, NULL, NULL, 0);
139 | 			} while (woken == WAKE_AT_ONCE);
140 | 		}
141 | 
142 | 		/**
143 | 		 * @brief wrapper for futex syscalls
144 | 		 * @param addr1 the address threads are observing
145 | 		 * @param op either FUTEX_WAIT or FUTEX_WAKE
146 | 		 * @param val1 if op==FUTEX_WAIT then the expected value for *addr1,
147 | 		 *             if op==FUTEX_WAKE then the number of threads to wake up
148 | 		 * @param timeout always NULL
149 | 		 * @param addr2 always NULL
150 | 		 * @param val3 always 0
151 | 		 * @return the return value of the futex syscall
152 | 		 */
153 | 		static long sys_futex(void *addr1, int op, int val1, struct timespec *timeout, void *addr2, int val3)
154 | 		{
155 | 			return syscall(SYS_futex, addr1, op, val1, timeout, addr2, val3);
156 | 		}
157 | 
158 | 
159 | 		static_assert(sizeof(serving) == sizeof(field_t), "std::atomic<field_t> size differs from size of field_t. Your architecture does not support the ticket_futex_lock");
160 | };
161 | 
162 | #endif /* qd_ticket_futex_lock_hpp */
163 | 


--------------------------------------------------------------------------------
/qdlock.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef qd_qdlock_hpp
  2 | #define qd_qdlock_hpp qdlock_hpp
  3 | 
  4 | #include "qdlock_base.hpp"
  5 | 
  6 | 
  7 | /**
  8 |  * @brief queue delegation lock implementation
  9 |  * @tparam MLock mutual exclusion lock
 10 |  * @tparam DQueue delegation queue
 11 |  */
 12 | template<class MLock, class DQueue, starvation_policy_t starvation_policy=starvation_policy_t::starvation_free>
 13 | class qdlock_impl : private qdlock_base<MLock, DQueue, starvation_policy> {
 14 | 	typedef qdlock_base<MLock, DQueue, starvation_policy> base;
 15 | 	template<class, class> friend class qd_condition_variable_impl;
 16 | 	public:
 17 | 		typedef typename base::no_reader_sync reader_indicator_t;
 18 | 		typedef typename base::no_hierarchy_sync hierarchy_t;
 19 | #if 0
 20 | 		/**
 21 | 		 * @brief delegate function
 22 | 		 * @tparam R return type of delegated operation
 23 | 		 * @tparam Ps parameter types of delegated operation
 24 | 		 * @param f the delegated operation
 25 | 		 * @param ps the parameters for the delegated operation
 26 | 		 * @return a future for return value of delegated operation
 27 | 		 */
 28 | #endif
 29 | 
 30 | 		/* the following delegate_XX functions are all specified twice:
 31 | 		 * First for a templated version, where a function is explicitly
 32 | 		 * given in the template argument list. (Probably using a macro)
 33 | 		 * Second for a version where the function or functor is a
 34 | 		 * normal parameter to the delegate_XX function.
 35 | 		 *
 36 | 		 * XX specifies how futures are dealt with:
 37 | 		 * n  - no futures
 38 | 		 * f  - returns future for return value of delegated operation
 39 | 		 * p  - delegated operation returns void and takes a promise as
 40 | 		 *      first argument, which is passed here by the user.
 41 | 		 * fp - returns future for a type which is specified in the
 42 | 		 *      template parameter list. The delegated operation takes
 43 | 		 *      a promise for that type as its first argument.
 44 | 		 */
 45 | 
 46 | 		/* interface _n functions: Do not wait, do not provide a future. */
 47 | 		template<typename Function, Function f, typename... Ps>
 48 | 		void delegate_n(Ps&&... ps) {
 49 | 			/* template provides function address */
 50 | 			using promise_t = typename base::no_promise::promise;
 51 | 			using reader_sync_t = typename base::no_reader_sync;
 52 | 			base::template delegate<Function, f, promise_t, reader_sync_t, hierarchy_t, Ps...>(nullptr, std::forward<Ps>(ps)...);
 53 | 		}
 54 | 		template<typename Function, typename... Ps>
 55 | 		void delegate_n(Function&& f, Ps&&... ps) {
 56 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
 57 | 			using promise_t = typename base::no_promise::promise;
 58 | 			using reader_sync_t = typename base::no_reader_sync;
 59 | 			base::template delegate<std::nullptr_t, nullptr, promise_t, reader_sync_t, hierarchy_t, Function, Ps...>(nullptr, std::forward<Function>(f), std::forward<Ps>(ps)...);
 60 | 		}
 61 | 
 62 | 		/* interface _f functions: Provide a future for the return value of the delegated operation */
 63 | 		template<typename Function, Function f, typename... Ps>
 64 | 		auto delegate_f(Ps&&... ps)
 65 | 		-> typename base::template std_promise<decltype(f(ps...))>::future
 66 | 		{
 67 | 			using return_t = decltype(f(ps...));
 68 | 			using promise_factory = typename base::template std_promise<return_t>;
 69 | 			using promise_t = typename promise_factory::promise;;
 70 | 			using reader_sync_t = typename base::no_reader_sync;
 71 | 			auto result = promise_factory::create_promise();
 72 | 			auto future = promise_factory::create_future(result);
 73 | 			base::template delegate<Function, f, promise_t, reader_sync_t, hierarchy_t, Ps...>(std::move(result), std::forward<Ps>(ps)...);
 74 | 			return future;
 75 | 		}
 76 | 		template<typename Function, typename... Ps>
 77 | 		auto delegate_f(Function&& f, Ps&&... ps)
 78 | 		-> typename base::template std_promise<decltype(f(ps...))>::future
 79 | 		{
 80 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
 81 | 			using return_t = decltype(f(ps...));
 82 | 			using promise_factory = typename base::template std_promise<return_t>;
 83 | 			using promise_t = typename promise_factory::promise;;
 84 | 			using reader_sync_t = typename base::no_reader_sync;
 85 | 			auto result = promise_factory::create_promise();
 86 | 			auto future = promise_factory::create_future(result);
 87 | 			base::template delegate<std::nullptr_t, nullptr, promise_t, reader_sync_t, hierarchy_t, Function, Ps...>(std::move(result), std::forward<Function>(f), std::forward<Ps>(ps)...);
 88 | 			return future;
 89 | 		}
 90 | 
 91 | 		/* interface _p functions: User provides a promise, which is used explicitly by the delegated (void) function */
 92 | 		template<typename Function, Function f, typename Promise, typename... Ps>
 93 | 		auto delegate_p(Promise&& result, Ps&&... ps)
 94 | 		-> void
 95 | 		{
 96 | 			using no_promise = typename base::no_promise::promise;
 97 | 			using reader_sync_t = typename base::no_reader_sync;
 98 | 			base::template delegate<Function, f, no_promise, reader_sync_t, hierarchy_t, Ps...>(nullptr, std::forward<Promise>(result), std::forward<Ps>(ps)...);
 99 | 		}
100 | 		template<typename Function, typename Promise, typename... Ps>
101 | 		auto delegate_p(Function&& f, Promise&& result, Ps&&... ps)
102 | 		-> void
103 | 		{
104 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
105 | 			using no_promise = typename base::no_promise::promise;
106 | 			using reader_sync_t = typename base::no_reader_sync;
107 | 			base::template delegate<std::nullptr_t, nullptr, no_promise, reader_sync_t, hierarchy_t, Function, Promise, Ps...>(nullptr, std::forward<Function>(f), std::forward<Promise>(result), std::forward<Ps>(ps)...);
108 | 		}
109 | 
110 | 		/* interface _fp functions: Promise is generated here, but delegated function uses it explicitly. */
111 | 		template<typename Return, typename Function, Function f, typename... Ps>
112 | 		auto delegate_fp(Ps&&... ps)
113 | 		-> typename base::template std_promise<Return>::future
114 | 		{
115 | 			using promise_factory = typename base::template std_promise<Return>;
116 | 			using promise_t = typename promise_factory::promise;
117 | 			using no_promise = typename base::no_promise::promise;
118 | 			using reader_sync_t = typename base::no_reader_sync;
119 | 			auto result = promise_factory::create_promise();
120 | 			auto future = promise_factory::create_future(result);
121 | 			base::template delegate<Function, f, no_promise, reader_sync_t, hierarchy_t, promise_t, Ps...>(std::move(result), std::forward<Ps>(ps)...);
122 | 			return future;
123 | 		}
124 | 		template<typename Return, typename Function, typename... Ps>
125 | 		auto delegate_fp(Function&& f, Ps&&... ps)
126 | 		-> typename base::template std_promise<Return>::future
127 | 		{
128 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
129 | 			using promise_factory = typename base::template std_promise<Return>;
130 | 			using promise_t = typename promise_factory::promise;
131 | 			using no_promise = typename base::no_promise::promise;
132 | 			using reader_sync_t = typename base::no_reader_sync;
133 | 			auto result = promise_factory::create_promise();
134 | 			auto future = promise_factory::create_future(result);
135 | 			base::template delegate<std::nullptr_t, nullptr, no_promise, reader_sync_t, hierarchy_t, Function, promise_t, Ps...>(nullptr, std::forward<Function>(f), std::move(result), std::forward<Ps>(ps)...);
136 | 			return future;
137 | 		}
138 | 		
139 | 
140 | 		void lock() {
141 | 			this->mutex_lock.lock();
142 | 		}
143 | 		void unlock() {
144 | 			this->mutex_lock.unlock();
145 | 		}
146 | };
147 | 
148 | #endif /* qd_qdlock_hpp */
149 | 


--------------------------------------------------------------------------------
/mrqdlock.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef qd_mrqdlock_hpp
  2 | #define qd_mrqdlock_hpp qd_mrqdlock_hpp
  3 | 
  4 | #include<array>
  5 | #include<atomic>
  6 | #include "readindicator/reader_groups.hpp"
  7 | #include "util/pause.hpp"
  8 | #include "qdlock_base.hpp"
  9 | 
 10 | template<class MLock, class DQueue, class RIndicator, int READ_PATIENCE_LIMIT, starvation_policy_t starvation_policy=starvation_policy_t::starvation_free>
 11 | class mrqdlock_impl {
 12 | 	char pad1[128];
 13 | 	std::atomic<int> writeBarrier;
 14 | 	char pad2[128];
 15 | 	RIndicator reader_indicator;
 16 | 	char pad3[128];
 17 | 	typedef qdlock_base<MLock, DQueue, starvation_policy> base;
 18 | 	base myqdlock;
 19 | 	typedef mrqdlock_impl<MLock, DQueue, RIndicator, READ_PATIENCE_LIMIT, starvation_policy>* this_t;
 20 | 	struct reader_indicator_sync {
 21 | 		static void wait_writers(base* t) {
 22 | 			while(reinterpret_cast<this_t>(t->__data)->writeBarrier.load() > 0) {
 23 | 				qd::pause();
 24 | 			}
 25 | 		}
 26 | 		static void wait_readers(base* t) {
 27 | 			while(reinterpret_cast<this_t>(t->__data)->reader_indicator.query()) {
 28 | 				qd::pause();
 29 | 			}
 30 | 		}
 31 | 	};
 32 | 	typedef typename base::no_hierarchy_sync hierarchy_t;
 33 | 	public:
 34 | 		typedef RIndicator reader_indicator_t;
 35 | 		mrqdlock_impl() : writeBarrier(0) {
 36 | 			myqdlock.__data = reinterpret_cast<void*>(this);
 37 | 		} // TODO proper comment. YES THIS NEEDS TO BE INITIALIZED
 38 | 
 39 | 		/* the following delegate_XX functions are all specified twice:
 40 | 		 * First for a templated version, where a function is explicitly
 41 | 		 * given in the template argument list. (Probably using a macro)
 42 | 		 * Second for a version where the function or functor is a
 43 | 		 * normal parameter to the delegate_XX function.
 44 | 		 *
 45 | 		 * XX specifies how futures are dealt with:
 46 | 		 * n  - no futures
 47 | 		 * f  - returns future for return value of delegated operation
 48 | 		 * p  - delegated operation returns void and takes a promise as
 49 | 		 *      first argument, which is passed here by the user.
 50 | 		 * fp - returns future for a type which is specified in the
 51 | 		 *      template parameter list. The delegated operation takes
 52 | 		 *      a promise for that type as its first argument.
 53 | 		 */
 54 | 
 55 | 		/* interface _n functions: Do not wait, do not provide a future. */
 56 | 		template<typename Function, Function f, typename... Ps>
 57 | 		void delegate_n(Ps&&... ps) {
 58 | 			/* template provides function address */
 59 | 			using promise_t = typename base::no_promise::promise;
 60 | 			myqdlock.template delegate<Function, f, promise_t, reader_indicator_sync, hierarchy_t, Ps...>(nullptr, std::forward<Ps>(ps)...);
 61 | 		}
 62 | 		template<typename Function, typename... Ps>
 63 | 		void delegate_n(Function&& f, Ps&&... ps) {
 64 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
 65 | 			using promise_t = typename base::no_promise::promise;
 66 | 			myqdlock.template delegate<std::nullptr_t, nullptr, promise_t, reader_indicator_sync, hierarchy_t, Function, Ps...>(nullptr, std::forward<Function>(f), std::forward<Ps>(ps)...);
 67 | 		}
 68 | 		template<typename Function, Function f, typename... Ps>
 69 | 		auto delegate_f(Ps&&... ps)
 70 | 		-> typename base::template std_promise<decltype(f(ps...))>::future
 71 | 		{
 72 | 			using return_t = decltype(f(ps...));
 73 | 			using promise_factory = typename base::template std_promise<return_t>;
 74 | 			using promise_t = typename promise_factory::promise;;
 75 | 			auto result = promise_factory::create_promise();
 76 | 			auto future = promise_factory::create_future(result);
 77 | 			myqdlock.template delegate<Function, f, promise_t, reader_indicator_sync, hierarchy_t, Ps...>(std::move(result), std::forward<Ps>(ps)...);
 78 | 			return future;
 79 | 		}
 80 | 		
 81 | 		/* interface _f functions: Provide a future for the return value of the delegated operation */
 82 | 		template<typename Function, typename... Ps>
 83 | 		auto delegate_f(Function&& f, Ps&&... ps)
 84 | 		-> typename base::template std_promise<decltype(f(ps...))>::future
 85 | 		{
 86 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
 87 | 			using return_t = decltype(f(ps...));
 88 | 			using promise_factory = typename base::template std_promise<return_t>;
 89 | 			using promise_t = typename promise_factory::promise;;
 90 | 			auto result = promise_factory::create_promise();
 91 | 			auto future = promise_factory::create_future(result);
 92 | 			myqdlock.template delegate<std::nullptr_t, nullptr, promise_t, reader_indicator_sync, hierarchy_t, Function, Ps...>(std::move(result), std::forward<Function>(f), std::forward<Ps>(ps)...);
 93 | 			return future;
 94 | 		}
 95 | 		
 96 | 		/* interface _p functions: User provides a promise, which is used explicitly by the delegated (void) function */
 97 | 		template<typename Function, Function f, typename Promise, typename... Ps>
 98 | 		auto delegate_p(Promise&& result, Ps&&... ps)
 99 | 		-> void
100 | 		{
101 | 			myqdlock.template delegate<Function, f, Promise, reader_indicator_sync, hierarchy_t, Ps...>(std::forward<Promise>(result), std::forward<Ps>(ps)...);
102 | 		}
103 | 		template<typename Function, typename Promise, typename... Ps>
104 | 		auto delegate_p(Function&& f, Promise&& result, Ps&&... ps)
105 | 		-> void
106 | 		{
107 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
108 | 			myqdlock.template delegate<std::nullptr_t, nullptr, Promise, reader_indicator_sync, hierarchy_t, Function, Ps...>(std::forward<Promise>(result), std::forward<Function>(f), std::forward<Ps>(ps)...);
109 | 		}
110 | 
111 | 		/* interface _fp functions: Promise is generated here, but delegated function uses it explicitly. */
112 | 		template<typename Return, typename Function, Function f, typename... Ps>
113 | 		auto delegate_fp(Ps&&... ps)
114 | 		-> typename base::template std_promise<Return>::future
115 | 		{
116 | 			using promise_factory = typename base::template std_promise<Return>;
117 | 			using promise_t = typename base::no_promise::promise;
118 | 			auto result = promise_factory::create_promise();
119 | 			auto future = promise_factory::create_future(result);
120 | 			myqdlock.template delegate<Function, f, promise_t, reader_indicator_sync, hierarchy_t, Ps...>(std::move(result), std::forward<Ps>(ps)...);
121 | 			return future;
122 | 		}
123 | 		template<typename Return, typename Function, typename... Ps>
124 | 		auto delegate_fp(Function&& f, Ps&&... ps)
125 | 		-> typename base::template std_promise<Return>::future
126 | 		{
127 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
128 | 			using promise_factory = typename base::template std_promise<Return>;
129 | 			using promise_t = typename base::no_promise::promise;
130 | 			auto result = promise_factory::create_promise();
131 | 			auto future = promise_factory::create_future(result);
132 | 			myqdlock.template delegate<std::nullptr_t, nullptr, promise_t, reader_indicator_sync, hierarchy_t, Function, Ps...>(nullptr, std::forward<Function>(f), std::move(result), std::forward<Ps>(ps)...);
133 | 			return future;
134 | 		}
135 | 
136 | 		void lock() {
137 | 			while(writeBarrier.load() > 0) {
138 | 				qd::pause();
139 | 			}
140 | 			myqdlock.mutex_lock.lock();
141 | 			while(reader_indicator.query()) {
142 | 				qd::pause();
143 | 			}
144 | 		}
145 | 		void unlock() {
146 | 			myqdlock.mutex_lock.unlock();
147 | 		}
148 | 
149 | 		void rlock() {
150 | 			bool bRaised = false;
151 | 			int readPatience = 0;
152 | start:
153 | 			reader_indicator.arrive();
154 | 			if(myqdlock.mutex_lock.is_locked()) {
155 | 				reader_indicator.depart();
156 | 				while(myqdlock.mutex_lock.is_locked()) {
157 | 					if((readPatience == READ_PATIENCE_LIMIT) && !bRaised) {
158 | 						writeBarrier.fetch_add(1);
159 | 						bRaised = true;
160 | 					}
161 | 					readPatience += 1;
162 | 					qd::pause();
163 | 				}
164 | 				goto start;
165 | 			}
166 | 			if(bRaised) writeBarrier.fetch_sub(1);
167 | 		}
168 | 		void runlock() {
169 | 			reader_indicator.depart();
170 | 		}
171 | 
172 | };
173 | 
174 | #endif /* qd_mrqdlock_hpp */
175 | 


--------------------------------------------------------------------------------
/queues/buffer_queue.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef qd_buffer_queue_hpp
  2 | #define qd_buffer_queue_hpp qd_buffer_queue_hpp
  3 | 
  4 | #include<algorithm>
  5 | #include<array>
  6 | #include<atomic>
  7 | #include<map>
  8 | #include<stack>
  9 | #include "util/type_tools.hpp"
 10 | 
 11 | /**
 12 |  * @brief a buffer-based tantrum queue
 13 |  * @tparam ARRAY_SIZE the buffer's size in bytes
 14 |  */
 15 | template<unsigned long ARRAY_SIZE>
 16 | class buffer_queue {
 17 | 	/** type for sizes of queue entries */
 18 | 	typedef unsigned long s_type;
 19 | 	/** type for the size field for queue entries, loads must not be optimized away in flush */
 20 | 	typedef std::atomic<s_type> sizetype;
 21 | 
 22 | 	/** type for function pointers to be stored in this queue */
 23 | //	typedef std::function<void(char*)> ftype;
 24 | 	typedef void(*ftype)(char*);
 25 | 
 26 | 	struct entry_t {
 27 | 		std::atomic<int> fun;
 28 | 		char buf[28];
 29 | 	};
 30 | 	public:
 31 | 		/** constants for current state of the queue */
 32 | 		enum class status { OPEN=0, SUCCESS=0, FULL, CLOSED };
 33 | 
 34 | 		buffer_queue() : counter(ARRAY_SIZE), closed(status::CLOSED), sizes(new std::map<ftype, int>), functions(new std::map<int, ftype>), rev_functions(new std::map<ftype, int>), function_idx(1) {
 35 | 			std::fill(&buffer_array[0], &buffer_array[ARRAY_SIZE], 0);
 36 | 		}
 37 | 
 38 | 		~buffer_queue() {
 39 | 			delete sizes.load();
 40 | 			delete functions.load();
 41 | 			delete rev_functions.load();
 42 | 		}
 43 | 
 44 | 		/** opens the queue */
 45 | 		void open() {
 46 | 			counter.store(0, std::memory_order_relaxed);
 47 | 			closed.store(status::OPEN, std::memory_order_relaxed);
 48 | 		}
 49 | 
 50 | 		void forwardall(char*) {};
 51 | 		template<typename P, typename... Ts>
 52 | 		void forwardall(char* offset, P&& p, Ts&&... ts) {
 53 | 			auto ptr = reinterpret_cast<P*>(offset);
 54 | 			new (ptr) P(std::forward<P>(p));
 55 | 			forwardall(offset+sizeof(p), std::forward<Ts>(ts)...);
 56 | 		}
 57 | 		/**
 58 | 		 * @brief enqueues an entry
 59 | 		 * @tparam P return type of associated function
 60 | 		 * @param op wrapper function for associated function
 61 | 		 * @return SUCCESS on successful storing in queue, FULL if the queue is full and CLOSED if the queue is closed explicitly
 62 | 		 */
 63 | 		template<typename... Ps>
 64 | 		status enqueue(ftype op, Ps*... ps) {
 65 | 			/* entry size = size of size + size of wrapper functor + size of promise + size of all parameters*/
 66 | 			constexpr s_type size = aligned(sizeof(std::atomic<int>) + sumsizes<Ps...>::size);
 67 | 			if(!rev_functions.load(std::memory_order_relaxed)->count(op)) {
 68 | 				auto new_sizes = new std::map<ftype, int>;
 69 | 				auto current_sizes = sizes.load(std::memory_order_relaxed);
 70 | 				auto new_funs = new std::map<int, ftype>;
 71 | 				auto new_rfuns = new std::map<ftype, int>;
 72 | 				auto idx = function_idx.fetch_add(1);
 73 | 				auto current_funs = functions.load(std::memory_order_relaxed);
 74 | 				auto current_rfuns = rev_functions.load(std::memory_order_relaxed);
 75 | 				do {
 76 | 					*new_sizes = *current_sizes;
 77 | 					(*new_sizes)[op] = size;
 78 | 				} while(!sizes.compare_exchange_weak(current_sizes, new_sizes, std::memory_order_relaxed));
 79 | 				do {
 80 | 					*new_funs = *current_funs;
 81 | 					(*new_funs)[idx] = op;
 82 | 				} while(!functions.compare_exchange_weak(current_funs, new_funs, std::memory_order_relaxed));
 83 | 				do {
 84 | 					*new_rfuns = *current_rfuns;
 85 | 					(*new_rfuns)[op] = idx;
 86 | 				} while(!rev_functions.compare_exchange_weak(current_rfuns, new_rfuns, std::memory_order_relaxed));
 87 | 
 88 | 				std::lock_guard<std::mutex> l(to_free_lock);
 89 | 				to_free.push(current_sizes);
 90 | 				to_free.push(current_rfuns);
 91 | 				to_free2.push(current_funs);
 92 | 			}
 93 | 
 94 | 			auto current_status = closed.load(std::memory_order_relaxed);
 95 | 			if(current_status != status::OPEN) {
 96 | 				return current_status;
 97 | 			}
 98 | 			/* get memory in buffer */
 99 | 			s_type index = counter.fetch_add(size, std::memory_order_relaxed);
100 | 			auto entryp = reinterpret_cast<entry_t*>(&buffer_array[index]);
101 | 			if(index <= ARRAY_SIZE - size) {
102 | 				/* enough memory available: move op, p and parameters to buffer, then set size of entry */
103 | 				forwardall(&entryp->buf[0], std::move(*ps)...);
104 | 
105 | 				entryp->fun.store((*rev_functions.load(std::memory_order_relaxed))[op], std::memory_order_release);
106 | 				return status::SUCCESS;
107 | 			} else {
108 | 				/* not enough memory available: avoid deadlock in flush by setting special value */
109 | 				if(index < static_cast<s_type>(ARRAY_SIZE - sizeof(std::atomic<int>))) {
110 | 					entryp->fun.store(-1, std::memory_order_release); // TODO MEMORDER
111 | 				}
112 | 				closed.store(status::FULL, std::memory_order_relaxed);
113 | 				return status::FULL;
114 | 			}
115 | 		}
116 | 
117 | 		/** execute all stored operations, leave queue in closed state */
118 | 		void flush() {
119 | 			s_type todo = 0;
120 | 			bool open = true;
121 | 			while(open) {
122 | 				s_type done = todo;
123 | 				todo = counter.load(std::memory_order_relaxed);
124 | 				if(todo == done) { /* close queue */
125 | 					todo = counter.exchange(ARRAY_SIZE, std::memory_order_relaxed);
126 | 					open = false;
127 | 					closed.store(status::CLOSED, std::memory_order_relaxed);
128 | 				}
129 | 				if(todo >= static_cast<s_type>(ARRAY_SIZE)-sizeof(std::atomic<int>)) { /* queue closed */
130 | 					constexpr auto todomax = ARRAY_SIZE-static_cast<s_type>(sizeof(std::atomic<int>));
131 | 					todo = todomax;
132 | 					closed.store(status::CLOSED, std::memory_order_relaxed);
133 | 					open = false;
134 | 				}
135 | 				s_type last_size = 0;
136 | 				for(s_type index = done; index < todo; index+=last_size) {
137 | 					/* synchronization on entry size field: 0 until entry available */
138 | 					int fun_idx = 0;
139 | 					auto entryp = reinterpret_cast<entry_t*>(&buffer_array[index]);
140 | 					do {
141 | 						fun_idx = entryp->fun.load(std::memory_order_acquire);
142 | 					} while(fun_idx == 0);
143 | 					/* buffer full signal: everything done */
144 | 					if(fun_idx == -1) {
145 | 						std::fill(&buffer_array[index], &buffer_array[index+sizeof(std::atomic<int>)], 0);
146 | 						goto end;
147 | 					}
148 | 					ftype fun = (*functions.load(std::memory_order_relaxed))[fun_idx];
149 | 					/* get functor from buffer */ // TODO: move-construct?
150 | 					last_size = (*sizes.load(std::memory_order_relaxed))[fun];
151 | 
152 | 					/* call functor with pointer to promise (of unknown type) */
153 | 					fun(&entryp->buf[0]);
154 | 
155 | 					/* cleanup: call destructor of (now empty) functor and clear buffer area */
156 | //					fun->~ftype();
157 | 					std::fill(&buffer_array[index], &buffer_array[index+last_size], 0);
158 | 				}
159 | 			}
160 | end:
161 | 			std::lock_guard<std::mutex> l(to_free_lock);
162 | 			while(!to_free.empty()) {
163 | 				auto del = to_free.top();
164 | 				to_free.pop();
165 | 				delete del;
166 | 			}
167 | 			while(!to_free2.empty()) {
168 | 				auto del = to_free2.top();
169 | 				to_free2.pop();
170 | 				delete del;
171 | 			}
172 | 		}
173 | 	private:
174 | 		/** counter for how much of the buffer is currently in use; offset to free area in buffer_array */
175 | 		std::atomic<s_type> counter;
176 | 		char pad[128];
177 | 		/** optimization flag: no writes when queue in known-closed state */
178 | 		std::atomic<status> closed;
179 | 		char pad2[128];
180 | 		/** the buffer for entries to this queue */
181 | 		std::array<char, ARRAY_SIZE> buffer_array;
182 | 		char pad4[128];
183 | 		std::atomic<std::map<ftype, int>*> sizes;
184 | 		char pad45[128];
185 | 		std::atomic<std::map<int, ftype>*> functions;
186 | 		char pad5[128];
187 | 		std::atomic<std::map<ftype, int>*> rev_functions;
188 | 		char pad55[128];
189 | 		std::mutex to_free_lock;
190 | 		char pad6[128];
191 | 		std::stack<std::map<ftype, int>*> to_free;
192 | 		char pad65[128];
193 | 		std::stack<std::map<int, ftype>*> to_free2;
194 | 		char pad7[128];
195 | 		std::atomic<int> function_idx;
196 | 
197 | 		static constexpr s_type aligned(s_type size) {
198 | 			return size;
199 | 			//return ((size + sizeof(sizetype) - 1) / sizeof(sizetype)) * sizeof(sizetype); /* TODO: better way of computing a ceil? */
200 | 		}
201 | };
202 | 
203 | #endif /* qd_buffer_queue_hpp */
204 | 


--------------------------------------------------------------------------------
/locks/mcs_futex_lock.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef qd_mcs_futex_lock_hpp
  2 | #define qd_mcs_futex_lock_hpp qd_futex_lock_hpp
  3 | 
  4 | #include<atomic>
  5 | #include<unistd.h>
  6 | #include<linux/futex.h>
  7 | #include<map>
  8 | #include<stack>
  9 | #include<sys/syscall.h>
 10 | #include<sys/param.h>
 11 | #include<thread>
 12 | #include "util/pause.hpp"
 13 | 
 14 | struct mcs_node {
 15 | 	enum field_t { free, taken, contended };
 16 | 	std::atomic<field_t> is_locked;
 17 | 	std::atomic<mcs_node*> next;
 18 | 	mcs_node() : next(nullptr) {}
 19 | };
 20 | 
 21 | /**
 22 |  * @brief a MCS based lock with futex functionality
 23 |  * @details This lock cooperates with the Linux kernel when there is no chance on taking the lock.
 24 |  *          The sleep-waiting can be triggered by lock() and try_lock_or_wait().
 25 |  *          When the lock cannot be taken these two functions wait until the Linux kernel wakes them up again,
 26 |  *          which is triggered by unlock() operations.
 27 |  * @note lock() and unlock() function largely like an MCS lock,
 28 |  *       while try_lock() and try_lock_or_wait() are more similar to the futex lock.
 29 |  */
 30 | class mcs_futex_lock {
 31 | 	using mcs_node = ::mcs_node;
 32 | 	using field_t = size_t;
 33 | 	std::atomic<field_t> locked;
 34 | 	enum field_status { free=0, flag=0x1 };
 35 | 	thread_local static std::map<mcs_futex_lock*, mcs_node> mcs_node_store;
 36 | 	public:
 37 | 		mcs_futex_lock() : locked(free) {}
 38 | 		mcs_futex_lock(mcs_futex_lock&) = delete; /* TODO? */
 39 | 
 40 | 		/**
 41 | 		 * @brief take the lock
 42 | 		 * @details This function will issue a futex wait syscall
 43 | 		 *          while waiting for the lock.
 44 | 		 */
 45 | 		void lock() {
 46 | 			mcs_node* mynode = &mcs_node_store[this];
 47 | 			field_t current = this->locked.load(std::memory_order_relaxed);
 48 | 			bool success;
 49 | 			field_t f;
 50 | 			do {
 51 | 				f = current & flag;
 52 | 				success = this->locked.compare_exchange_strong(current, reinterpret_cast<field_t>(mynode)|f, std::memory_order_acq_rel);
 53 | 			} while(!success);
 54 | 			mcs_node* found = reinterpret_cast<mcs_node*>(current ^ f);
 55 | 			if(found != nullptr) {
 56 | 				mynode->is_locked.store(mcs_node::taken, std::memory_order_release);
 57 | 				found->next = mynode;
 58 | 				for(int i = 0; i < 512; i++) {
 59 | 					if(mynode->is_locked.load(std::memory_order_acquire) == mcs_node::free) {
 60 | 						break;
 61 | 					}
 62 | 					qd::pause();
 63 | 				}
 64 | 				if(mynode->is_locked.load(std::memory_order_acquire) == mcs_node::taken) {
 65 | 					mcs_node::field_t l = mcs_node::taken;
 66 | 					mynode->is_locked.compare_exchange_strong(l, mcs_node::contended, std::memory_order_acq_rel);
 67 | 					if(l == mcs_node::free) {
 68 | 						return;
 69 | 					}
 70 | 				}
 71 | 				while(mynode->is_locked.load(std::memory_order_acquire) != mcs_node::free) {
 72 | 					wait(reinterpret_cast<int*>(&mynode->is_locked), static_cast<int>(mcs_node::contended));
 73 | 				}
 74 | 			}
 75 | 		}
 76 | 
 77 | 		/**
 78 | 		 * @brief release the lock
 79 | 		 * @details This will unlock the lock. If there are sleeping threads,
 80 | 		 *          they will also be woken up.
 81 | 		 */
 82 | 		void unlock() {
 83 | 			mcs_node* mynode = &mcs_node_store[this];
 84 | 			/* thread_local mynode is set while locked */
 85 | 			field_t c = reinterpret_cast<field_t>(mynode);
 86 | 			if(mynode->next == nullptr) {
 87 | 				bool sleep = (locked & flag) == flag;
 88 | 				if(sleep) {
 89 | 					c |= flag;
 90 | 				}
 91 | 				if(this->locked.compare_exchange_strong(c, free, std::memory_order_acq_rel)) {
 92 | 					if(sleep) {
 93 | 						notify_all(reinterpret_cast<int*>(&locked)); /* TODO one instead? */
 94 | 					}
 95 | 					return;
 96 | 				} else if((c & ~flag) == reinterpret_cast<field_t>(mynode)) {
 97 | 					unlock();
 98 | 					return;
 99 | 				}
100 | 			}
101 | 			while(mynode->next == nullptr) {
102 | 				/* wait for nextpointer */
103 | 				qd::pause();
104 | 			}
105 | 			mcs_node* next = mynode->next.load(); /* TODO */
106 | 
107 | 			int status = next->is_locked.exchange(mcs_node::free, std::memory_order_acq_rel);
108 | 			if(status == mcs_node::contended) {
109 | 				notify_one(reinterpret_cast<int*>(&next->is_locked)); /* there can only be one waiting */
110 | 			}
111 | 			mynode->next.store(nullptr, std::memory_order_relaxed);
112 | 		}
113 | 
114 | 		/**
115 | 		 * @brief non-blocking trylock.
116 | 		 * @return true iff the lock has been taken
117 | 		 */
118 | 		bool try_lock() {
119 | 			if(this->is_locked()) {
120 | 				return false;
121 | 			}
122 | 			field_t c = free;
123 | 			mcs_node* mynode = &mcs_node_store[this];
124 | 			return this->locked.compare_exchange_strong(c, reinterpret_cast<field_t>(mynode), std::memory_order_acq_rel);
125 | 		}
126 | 
127 | 		/**
128 | 		 * @brief blocking trylock.
129 | 		 * @return true iff the lock has been taken,
130 | 		 *         false after waiting for an unlock() operation on the lock.
131 | 		 */
132 | 		bool try_lock_or_wait() {
133 | 			field_t c = locked.load(std::memory_order_acquire); /*TODO acq or relaxed*/
134 | 			mcs_node* mynode = &mcs_node_store[this];
135 | 			field_t flagged = c | flag;
136 | 			while((c & flag) != flag) {
137 | 				if(c == free) {
138 | 					if(this->locked.compare_exchange_strong(c, reinterpret_cast<field_t>(mynode), std::memory_order_acq_rel)) {
139 | 						return true;
140 | 					}
141 | 					continue;
142 | 				}
143 | 				flagged = c | flag;
144 | 				if(this->locked.compare_exchange_weak(c,flagged, std::memory_order_acq_rel)) {
145 | 					break;
146 | 				}
147 | 			}
148 | 			/* get futexptr to least-significant 32 bit of address, assumes little-endian */
149 | 			int* futexptr = reinterpret_cast<int*>(&locked);
150 | 			wait(futexptr, reinterpret_cast<size_t>(flagged));
151 | 			return false;
152 | 		}
153 | 
154 | 		/**
155 | 		 * @brief check lock state
156 | 		 * @return true iff the lock is taken, false otherwise
157 | 		 */
158 | 		bool is_locked() {
159 | 			return locked.load(std::memory_order_acquire) != free;
160 | 		}
161 | 
162 | 		/**
163 | 		 * @brief wake threads that are waiting for a lock handover
164 | 		 */
165 | 		void wake() {
166 | 			mcs_node* mynode = &mcs_node_store[this];
167 | 			field_t c = reinterpret_cast<field_t>(mynode);
168 | 			field_t flagged = c | flag;
169 | 			if(locked == flagged) {
170 | 				notify_all(reinterpret_cast<int*>(&locked));
171 | 			}
172 | 		}
173 | 
174 | 	private:
175 | 		/**
176 | 		 * @brief sleep until unlock() is called
177 | 		 * @param addr the memory address to wait for a change on
178 | 		 * @param expected currently expected value at addr
179 | 		 */
180 | 		void wait(int* addr, int expected) {
181 | 			sys_futex(addr, FUTEX_WAIT, expected, NULL, NULL, 0);
182 | 		}
183 | 
184 | 		/**
185 | 		 * @brief wake one thread from sleeping that waits on this lock
186 | 		 * @param addr the memory address to wait for a change on
187 | 		 */
188 | 		void notify_one(int* addr) {
189 | 			sys_futex(addr, FUTEX_WAKE, 1, NULL, NULL, 0);
190 | 		}
191 | 
192 | 		/**
193 | 		 * @brief wake all sleeping threads that wait on this lock
194 | 		 * @param addr the memory address to wait for a change on
195 | 		 */
196 | 		void notify_all(int* addr) {
197 | 			static const int WAKE_AT_ONCE = 32768;
198 | 			int woken;
199 | 			do {
200 | 				woken = sys_futex(addr, FUTEX_WAKE, WAKE_AT_ONCE, NULL, NULL, 0);
201 | 			} while (woken == WAKE_AT_ONCE);
202 | 		}
203 | 
204 | 		/**
205 | 		 * @brief wrapper for futex syscalls
206 | 		 * @param addr1 the address threads are observing
207 | 		 * @param op either FUTEX_WAIT or FUTEX_WAKE
208 | 		 * @param val1 if op==FUTEX_WAIT then the expected value for *addr1,
209 | 		 *             if op==FUTEX_WAKE then the number of threads to wake up
210 | 		 * @param timeout always NULL
211 | 		 * @param addr2 always NULL
212 | 		 * @param val3 always 0
213 | 		 * @return the return value of the futex syscall
214 | 		 */
215 | 		static long sys_futex(void *addr1, int op, int val1, struct timespec *timeout, void *addr2, int val3)
216 | 		{
217 | 			return syscall(SYS_futex, addr1, op, val1, timeout, addr2, val3);
218 | 		}
219 | 
220 | 
221 | 		static_assert(sizeof(locked) == sizeof(field_t), "std::atomic<field_t> size differs from size of field_t. Your architecture does not support the MCS futex_lock");
222 | 		static_assert(__BYTE_ORDER == __LITTLE_ENDIAN, "Your architecture's endianness is currently not supported. Please report this as a bug.");
223 | };
224 | 
225 | #endif /* qd_mcs_futex_lock_hpp */
226 | 


--------------------------------------------------------------------------------
/hqdlock.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef qd_hqdlock_hpp
  2 | #define qd_hqdlock_hpp qd_hqdlock_hpp
  3 | 
  4 | #include <vector>
  5 | 
  6 | #include <sched.h>
  7 | #include <unistd.h>
  8 | #ifdef QD_USE_LIBNUMA
  9 | #include <numa.h>
 10 | #endif
 11 | #include "qdlock_base.hpp"
 12 | 
 13 | /**
 14 |  * @brief policies for thread pinning
 15 |  */
 16 | enum class pinning_policy_t {free_threads, pinned_threads};
 17 | 
 18 | /**
 19 |  * @brief hierarchical queue delegation lock implementation
 20 |  * @tparam GMLock global mutual exclusion lock
 21 |  * @tparam NMLock node-local mutual exclusion lock
 22 |  * @tparam DQueue delegation queue
 23 |  */
 24 | template<class GMLock, class NMLock, class DQueue, pinning_policy_t pinning_policy=pinning_policy_t::free_threads, starvation_policy_t starvation_policy=starvation_policy_t::starvation_free>
 25 | class hqdlock_impl {
 26 | 	template<class, class> friend class qd_condition_variable_impl;
 27 | 	
 28 | 	typedef qdlock_base<NMLock, DQueue, starvation_policy> base;
 29 | 	typedef hqdlock_impl<GMLock, NMLock, DQueue, pinning_policy, starvation_policy>* this_t;
 30 | 	struct hierarchy_sync {
 31 | 		static void lock(base* t) {
 32 | 			reinterpret_cast<this_t>(t->__data)->global_lock.lock();
 33 | 		};
 34 | 		static void unlock(base* t) {
 35 | 			reinterpret_cast<this_t>(t->__data)->global_lock.unlock();
 36 | 		};
 37 | 	};
 38 | 	typedef hierarchy_sync hierarchy_t;
 39 | 	typedef typename base::no_reader_sync reader_indicator_t;
 40 | 	
 41 | 	GMLock global_lock;
 42 | 	char pad1[128];
 43 | 	int numanodes;
 44 | 	std::vector<int> numa_mapping;
 45 | 	base* qdlocks;
 46 | 	thread_local static bool node_id_init;
 47 | 	thread_local static int node_id;
 48 | 	base& get_qd() {
 49 | 		int idx;
 50 | 		if(pinning_policy == pinning_policy_t::free_threads) {
 51 | 			int cpu = sched_getcpu();
 52 | 			idx = numa_mapping[cpu];
 53 | 		} else {
 54 | 			if(!node_id_init) {
 55 | 				node_id = sched_getcpu();
 56 | 				node_id_init = true;
 57 | 			}
 58 | 			idx = numa_mapping[node_id];
 59 | 		}
 60 | 		return qdlocks[idx];
 61 | 	}
 62 | 	public:
 63 | 
 64 | 		hqdlock_impl() {
 65 | 			int num_cpus = sysconf(_SC_NPROCESSORS_CONF); // sane default
 66 | 			numa_mapping.resize(num_cpus, 0);
 67 | 			#ifdef QD_USE_LIBNUMA
 68 | 			/* use libnuma only if it is actually available */
 69 | 			if(numa_available() != -1) {
 70 | 				numanodes = numa_num_configured_nodes();
 71 | 				/* Initialize the NUMA map */
 72 | 				for (int i = 0; i < num_cpus; ++i) {
 73 | 					numa_mapping[i] = numa_node_of_cpu(i);
 74 | 				}
 75 | 			}
 76 | 			#endif
 77 | 			/* initialize hierarchy components */
 78 | 			qdlocks = new base[numanodes];
 79 | 			for(int a = 0; a < numanodes; a++) {
 80 | 				qdlocks[a].__data = reinterpret_cast<void*>(this);
 81 | 			}
 82 | 		}
 83 | 		/* the following delegate_XX functions are all specified twice:
 84 | 		 * First for a templated version, where a function is explicitly
 85 | 		 * given in the template argument list. (Probably using a macro)
 86 | 		 * Second for a version where the function or functor is a
 87 | 		 * normal parameter to the delegate_XX function.
 88 | 		 *
 89 | 		 * XX specifies how futures are dealt with:
 90 | 		 * n  - no futures
 91 | 		 * f  - returns future for return value of delegated operation
 92 | 		 * p  - delegated operation returns void and takes a promise as
 93 | 		 *      first argument, which is passed here by the user.
 94 | 		 * fp - returns future for a type which is specified in the
 95 | 		 *      template parameter list. The delegated operation takes
 96 | 		 *      a promise for that type as its first argument.
 97 | 		 */
 98 | 
 99 | 		/* interface _n functions: Do not wait, do not provide a future. */
100 | 		template<typename Function, Function f, typename... Ps>
101 | 		void delegate_n(Ps&&... ps) {
102 | 			/* template provides function address */
103 | 			using promise_t = typename base::no_promise::promise;
104 | 			using reader_sync_t = typename base::no_reader_sync;
105 | 			get_qd().template delegate<Function, f, promise_t, reader_sync_t, hierarchy_t, Ps...>(nullptr, std::forward<Ps>(ps)...);
106 | 		}
107 | 		template<typename Function, typename... Ps>
108 | 		void delegate_n(Function&& f, Ps&&... ps) {
109 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
110 | 			using promise_t = typename base::no_promise::promise;
111 | 			using reader_sync_t = typename base::no_reader_sync;
112 | 			get_qd().template delegate<std::nullptr_t, nullptr, promise_t, reader_sync_t, hierarchy_t, Function, Ps...>(nullptr, std::forward<Function>(f), std::forward<Ps>(ps)...);
113 | 		}
114 | 
115 | 		/* interface _f functions: Provide a future for the return value of the delegated operation */
116 | 		template<typename Function, Function f, typename... Ps>
117 | 		auto delegate_f(Ps&&... ps)
118 | 		-> typename base::template std_promise<decltype(f(ps...))>::future
119 | 		{
120 | 			using return_t = decltype(f(ps...));
121 | 			using promise_factory = typename base::template std_promise<return_t>;
122 | 			using promise_t = typename promise_factory::promise;;
123 | 			using reader_sync_t = typename base::no_reader_sync;
124 | 			auto result = promise_factory::create_promise();
125 | 			auto future = promise_factory::create_future(result);
126 | 			get_qd().template delegate<Function, f, promise_t, reader_sync_t, hierarchy_t, Ps...>(std::move(result), std::forward<Ps>(ps)...);
127 | 			return future;
128 | 		}
129 | 		template<typename Function, typename... Ps>
130 | 		auto delegate_f(Function&& f, Ps&&... ps)
131 | 		-> typename base::template std_promise<decltype(f(ps...))>::future
132 | 		{
133 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
134 | 			using return_t = decltype(f(ps...));
135 | 			using promise_factory = typename base::template std_promise<return_t>;
136 | 			using promise_t = typename promise_factory::promise;;
137 | 			using reader_sync_t = typename base::no_reader_sync;
138 | 			auto result = promise_factory::create_promise();
139 | 			auto future = promise_factory::create_future(result);
140 | 			get_qd().template delegate<std::nullptr_t, nullptr, promise_t, reader_sync_t, hierarchy_t, Function, Ps...>(std::move(result), std::forward<Function>(f), std::forward<Ps>(ps)...);
141 | 			return future;
142 | 		}
143 | 
144 | 		/* interface _p functions: User provides a promise, which is used explicitly by the delegated (void) function */
145 | 		template<typename Function, Function f, typename Promise, typename... Ps>
146 | 		auto delegate_p(Promise&& result, Ps&&... ps)
147 | 		-> void
148 | 		{
149 | 			using no_promise = typename base::no_promise::promise;
150 | 			using reader_sync_t = typename base::no_reader_sync;
151 | 			get_qd().template delegate<Function, f, no_promise, reader_sync_t, hierarchy_t, Ps...>(nullptr, std::forward<Promise>(result), std::forward<Ps>(ps)...);
152 | 		}
153 | 		template<typename Function, typename Promise, typename... Ps>
154 | 		auto delegate_p(Function&& f, Promise&& result, Ps&&... ps)
155 | 		-> void
156 | 		{
157 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
158 | 			using no_promise = typename base::no_promise::promise;
159 | 			using reader_sync_t = typename base::no_reader_sync;
160 | 			get_qd().template delegate<std::nullptr_t, nullptr, no_promise, reader_sync_t, hierarchy_t, Function, Promise, Ps...>(nullptr, std::forward<Function>(f), std::forward<Promise>(result), std::forward<Ps>(ps)...);
161 | 		}
162 | 
163 | 		/* interface _fp functions: Promise is generated here, but delegated function uses it explicitly. */
164 | 		template<typename Return, typename Function, Function f, typename... Ps>
165 | 		auto delegate_fp(Ps&&... ps)
166 | 		-> typename base::template std_promise<Return>::future
167 | 		{
168 | 			using promise_factory = typename base::template std_promise<Return>;
169 | 			using promise_t = typename promise_factory::promise;
170 | 			using no_promise = typename base::no_promise::promise;
171 | 			using reader_sync_t = typename base::no_reader_sync;
172 | 			auto result = promise_factory::create_promise();
173 | 			auto future = promise_factory::create_future(result);
174 | 			get_qd().template delegate<Function, f, no_promise, reader_sync_t, hierarchy_t, promise_t, Ps...>(std::move(result), std::forward<Ps>(ps)...);
175 | 			return future;
176 | 		}
177 | 		template<typename Return, typename Function, typename... Ps>
178 | 		auto delegate_fp(Function&& f, Ps&&... ps)
179 | 		-> typename base::template std_promise<Return>::future
180 | 		{
181 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
182 | 			using promise_factory = typename base::template std_promise<Return>;
183 | 			using promise_t = typename promise_factory::promise;
184 | 			using no_promise = typename base::no_promise::promise;
185 | 			using reader_sync_t = typename base::no_reader_sync;
186 | 			auto result = promise_factory::create_promise();
187 | 			auto future = promise_factory::create_future(result);
188 | 			get_qd().template delegate<std::nullptr_t, nullptr, no_promise, reader_sync_t, hierarchy_t, Function, promise_t, Ps...>(nullptr, std::forward<Function>(f), std::move(result), std::forward<Ps>(ps)...);
189 | 			return future;
190 | 		}
191 | 		
192 | 		/* TODO? the following functions could also implement a cohort lock */
193 | 		void lock() {
194 | 			get_qd().mutex_lock.lock();
195 | 			this->global_lock.lock();
196 | 		}
197 | 		void unlock() {
198 | 			this->global_lock.unlock();
199 | 			get_qd().mutex_lock.unlock();
200 | 		}
201 | };
202 | 
203 | template<class GMLock, class NMLock, class DQueue, pinning_policy_t pinning_policy, starvation_policy_t starvation_policy>
204 | thread_local bool hqdlock_impl<GMLock, NMLock, DQueue, pinning_policy, starvation_policy>::node_id_init = false;
205 | template<class GMLock, class NMLock, class DQueue, pinning_policy_t pinning_policy, starvation_policy_t starvation_policy>
206 | thread_local int hqdlock_impl<GMLock, NMLock, DQueue, pinning_policy, starvation_policy>::node_id;
207 | 
208 | template<class GMLock, class NMLock, class DQueue, class RIndicator, int READ_PATIENCE_LIMIT, pinning_policy_t pinning_policy=pinning_policy_t::free_threads, starvation_policy_t starvation_policy=starvation_policy_t::starvation_free>
209 | class mrhqdlock_impl {
210 | 	char pad1[128];
211 | 	std::atomic<int> writeBarrier;
212 | 	char pad2[128];
213 | 	RIndicator reader_indicator;
214 | 	char pad3[128];
215 | 	typedef mrhqdlock_impl<GMLock, NMLock, DQueue, RIndicator, READ_PATIENCE_LIMIT, pinning_policy, starvation_policy>* this_t;
216 | 	typedef qdlock_base<NMLock, DQueue, starvation_policy> base;
217 | 	struct reader_indicator_sync {
218 | 		static void wait_writers(base* t) {
219 | 			while(reinterpret_cast<this_t>(t->__data)->writeBarrier.load() > 0) {
220 | 				qd::pause();
221 | 			}
222 | 		}
223 | 		static void wait_readers(base* t) {
224 | 			while(reinterpret_cast<this_t>(t->__data)->reader_indicator.query()) {
225 | 				qd::pause();
226 | 			}
227 | 		}
228 | 	};
229 | 	struct hierarchy_sync {
230 | 		static void lock(base* t) {
231 | 			reinterpret_cast<this_t>(t->__data)->global_lock.lock();
232 | 		};
233 | 		static void unlock(base* t) {
234 | 			reinterpret_cast<this_t>(t->__data)->global_lock.unlock();
235 | 		};
236 | 	};
237 | 	typedef hierarchy_sync hierarchy_t;
238 | 	
239 | 	GMLock global_lock;
240 | 	char pad4[128];
241 | 	int numanodes;
242 | 	std::vector<int> numa_mapping;
243 | 	base* qdlocks;
244 | 	thread_local static bool node_id_init;
245 | 	thread_local static int node_id;
246 | 	base& get_qd() {
247 | 		int idx;
248 | 		if(pinning_policy == pinning_policy_t::free_threads) {
249 | 			int cpu = sched_getcpu();
250 | 			idx = numa_mapping[cpu];
251 | 		} else {
252 | 			if(!node_id_init) {
253 | 				node_id = sched_getcpu();
254 | 				node_id_init = true;
255 | 			}
256 | 			idx = numa_mapping[node_id];
257 | 		}
258 | 		return qdlocks[idx];
259 | 	}
260 | 	public:
261 | 
262 | 		mrhqdlock_impl() : writeBarrier(0) {
263 | 			int num_cpus = sysconf(_SC_NPROCESSORS_CONF); // sane default
264 | 			numa_mapping.resize(num_cpus, 0);
265 | 			#ifdef QD_USE_LIBNUMA
266 | 			/* use libnuma only if it is actually available */
267 | 			if(numa_available() != -1) {
268 | 				numanodes = numa_num_configured_nodes();
269 | 				/* Initialize the NUMA map */
270 | 				for (int i = 0; i < num_cpus; ++i) {
271 | 					numa_mapping[i] = numa_node_of_cpu(i);
272 | 				}
273 | 			}
274 | 			#endif
275 | 			/* initialize hierarchy components */
276 | 			qdlocks = new base[numanodes];
277 | 			for(int a = 0; a < numanodes; a++) {
278 | 				qdlocks[a].__data = reinterpret_cast<void*>(this);
279 | 			}
280 | 		}
281 | 		typedef reader_indicator_sync reader_sync_t;
282 | 
283 | 		/* the following delegate_XX functions are all specified twice:
284 | 		 * First for a templated version, where a function is explicitly
285 | 		 * given in the template argument list. (Probably using a macro)
286 | 		 * Second for a version where the function or functor is a
287 | 		 * normal parameter to the delegate_XX function.
288 | 		 *
289 | 		 * XX specifies how futures are dealt with:
290 | 		 * n  - no futures
291 | 		 * f  - returns future for return value of delegated operation
292 | 		 * p  - delegated operation returns void and takes a promise as
293 | 		 *      first argument, which is passed here by the user.
294 | 		 * fp - returns future for a type which is specified in the
295 | 		 *      template parameter list. The delegated operation takes
296 | 		 *      a promise for that type as its first argument.
297 | 		 */
298 | 
299 | 		/* interface _n functions: Do not wait, do not provide a future. */
300 | 		template<typename Function, Function f, typename... Ps>
301 | 		void delegate_n(Ps&&... ps) {
302 | 			/* template provides function address */
303 | 			using promise_t = typename base::no_promise::promise;
304 | 			get_qd().template delegate<Function, f, promise_t, reader_sync_t, hierarchy_t, Ps...>(nullptr, std::forward<Ps>(ps)...);
305 | 		}
306 | 		template<typename Function, typename... Ps>
307 | 		void delegate_n(Function&& f, Ps&&... ps) {
308 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
309 | 			using promise_t = typename base::no_promise::promise;
310 | 			get_qd().template delegate<std::nullptr_t, nullptr, promise_t, reader_sync_t, hierarchy_t, Function, Ps...>(nullptr, std::forward<Function>(f), std::forward<Ps>(ps)...);
311 | 		}
312 | 		template<typename Function, Function f, typename... Ps>
313 | 		auto delegate_f(Ps&&... ps)
314 | 		-> typename base::template std_promise<decltype(f(ps...))>::future
315 | 		{
316 | 			using return_t = decltype(f(ps...));
317 | 			using promise_factory = typename base::template std_promise<return_t>;
318 | 			using promise_t = typename promise_factory::promise;;
319 | 			auto result = promise_factory::create_promise();
320 | 			auto future = promise_factory::create_future(result);
321 | 			get_qd().template delegate<Function, f, promise_t, reader_sync_t, hierarchy_t, Ps...>(std::move(result), std::forward<Ps>(ps)...);
322 | 			return future;
323 | 		}
324 | 		
325 | 		/* interface _f functions: Provide a future for the return value of the delegated operation */
326 | 		template<typename Function, typename... Ps>
327 | 		auto delegate_f(Function&& f, Ps&&... ps)
328 | 		-> typename base::template std_promise<decltype(f(ps...))>::future
329 | 		{
330 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
331 | 			using return_t = decltype(f(ps...));
332 | 			using promise_factory = typename base::template std_promise<return_t>;
333 | 			using promise_t = typename promise_factory::promise;;
334 | 			auto result = promise_factory::create_promise();
335 | 			auto future = promise_factory::create_future(result);
336 | 			get_qd().template delegate<std::nullptr_t, nullptr, promise_t, reader_sync_t, hierarchy_t, Function, Ps...>(std::move(result), std::forward<Function>(f), std::forward<Ps>(ps)...);
337 | 			return future;
338 | 		}
339 | 		
340 | 		/* interface _p functions: User provides a promise, which is used explicitly by the delegated (void) function */
341 | 		template<typename Function, Function f, typename Promise, typename... Ps>
342 | 		auto delegate_p(Promise&& result, Ps&&... ps)
343 | 		-> void
344 | 		{
345 | 			using no_promise = typename base::no_promise::promise;
346 | 			get_qd().template delegate<Function, f, no_promise, reader_sync_t, hierarchy_t, Ps...>(nullptr, std::forward<Promise>(result), std::forward<Ps>(ps)...);
347 | 		}
348 | 		template<typename Function, typename Promise, typename... Ps>
349 | 		auto delegate_p(Function&& f, Promise&& result, Ps&&... ps)
350 | 		-> void
351 | 		{
352 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
353 | 			using no_promise = typename base::no_promise::promise;
354 | 			get_qd().template delegate<std::nullptr_t, nullptr, no_promise, reader_sync_t, hierarchy_t, Function, Promise, Ps...>(nullptr, std::forward<Function>(f), std::forward<Promise>(result), std::forward<Ps>(ps)...);
355 | 		}
356 | 
357 | 		/* interface _fp functions: Promise is generated here, but delegated function uses it explicitly. */
358 | 		template<typename Return, typename Function, Function f, typename... Ps>
359 | 		auto delegate_fp(Ps&&... ps)
360 | 		-> typename base::template std_promise<Return>::future
361 | 		{
362 | 			using promise_factory = typename base::template std_promise<Return>;
363 | 			using promise_t = typename base::no_promise::promise;
364 | 			auto result = promise_factory::create_promise();
365 | 			auto future = promise_factory::create_future(result);
366 | 			get_qd().template delegate<Function, f, promise_t, reader_sync_t, hierarchy_t, promise_t, Ps...>(std::move(result), std::forward<Ps>(ps)...);
367 | 			return future;
368 | 		}
369 | 		template<typename Return, typename Function, typename... Ps>
370 | 		auto delegate_fp(Function&& f, Ps&&... ps)
371 | 		-> typename base::template std_promise<Return>::future
372 | 		{
373 | 			/* type of functor/function ptr stored in f, set template function pointer to NULL */
374 | 			using promise_factory = typename base::template std_promise<Return>;
375 | 			using promise_t = typename base::no_promise::promise;
376 | 			auto result = promise_factory::create_promise();
377 | 			auto future = promise_factory::create_future(result);
378 | 			get_qd().template delegate<std::nullptr_t, nullptr, promise_t, reader_sync_t, hierarchy_t, Function, promise_t, Ps...>(nullptr, std::forward<Function>(f), std::move(result), std::forward<Ps>(ps)...);
379 | 			return future;
380 | 		}
381 | 
382 | 		void lock() {
383 | 			while(writeBarrier.load() > 0) {
384 | 				qd::pause();
385 | 			}
386 | 			get_qd().mutex_lock.lock();
387 | 			this->global_lock.lock();
388 | 			while(reader_indicator.query()) {
389 | 				qd::pause();
390 | 			}
391 | 		}
392 | 		void unlock() {
393 | 			this->global_lock.unlock();
394 | 			get_qd().mutex_lock.unlock();
395 | 		}
396 | 
397 | 		void rlock() {
398 | 			bool bRaised = false;
399 | 			int readPatience = 0;
400 | start:
401 | 			reader_indicator.arrive();
402 | 			if(this->global_lock.is_locked()) {
403 | 				reader_indicator.depart();
404 | 				while(this->global_lock.is_locked()) { // TODO is this sufficient?
405 | 					qd::pause();
406 | 					if((readPatience == READ_PATIENCE_LIMIT) && !bRaised) {
407 | 						writeBarrier.fetch_add(1);
408 | 						bRaised = true;
409 | 					}
410 | 					readPatience += 1;
411 | 				}
412 | 				goto start;
413 | 			}
414 | 			if(bRaised) writeBarrier.fetch_sub(1);
415 | 		}
416 | 		void runlock() {
417 | 			reader_indicator.depart();
418 | 		}
419 | 
420 | };
421 | 
422 | template<class GMLock, class NMLock, class DQueue, class RIndicator, int READ_PATIENCE_LIMIT, pinning_policy_t pinning_policy, starvation_policy_t starvation_policy>
423 | thread_local bool mrhqdlock_impl<GMLock, NMLock, DQueue, RIndicator, READ_PATIENCE_LIMIT, pinning_policy, starvation_policy>::node_id_init = false;
424 | template<class GMLock, class NMLock, class DQueue, class RIndicator, int READ_PATIENCE_LIMIT, pinning_policy_t pinning_policy, starvation_policy_t starvation_policy>
425 | thread_local int mrhqdlock_impl<GMLock, NMLock, DQueue, RIndicator, READ_PATIENCE_LIMIT, pinning_policy, starvation_policy>::node_id;
426 | 
427 | #endif /* qd_hqdlock_hpp */
428 | 


--------------------------------------------------------------------------------
/qdlock_base.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef qdlock_base_hpp
  2 | #define qdlock_base_hpp qdlock_base_hpp
  3 | 
  4 | #include<cstddef>
  5 | #include<future>
  6 | #include<type_traits>
  7 | 
  8 | #include "util/pause.hpp"
  9 | #include "waiting_future.hpp"
 10 | 
 11 | 
 12 | /* unpacking promise from delegation queue */
 13 | template<typename R, template<class> class Promise = std::promise>
 14 | class unpack_promise : public Promise<R> {
 15 | 	public:
 16 | 		unpack_promise(char* pptr) : Promise<R>(std::move(*reinterpret_cast<Promise<R>*>(pptr))) {}
 17 | 		~unpack_promise() {} /* automatically destructs superclass */
 18 | 		unpack_promise() = delete; /* meaningless */
 19 | 		unpack_promise(unpack_promise&) = delete; /* not possible */
 20 | 		unpack_promise(unpack_promise&&) = delete; /* actually implementable, TODO for now */
 21 | };
 22 | 
 23 | 
 24 | /* Next is a block of templates for delegated function wrappers.
 25 |  * They unpack the promise (using a wrapper) and fulfill it in various ways.
 26 |  * These are
 27 |  * 1) normal functions
 28 |  * 2) member functions and functors
 29 |  * A) with the function pointer known to the template (pointer known at compile time)
 30 |  * B) with the function pointer specified at runtime (signature known at compile time)
 31 |  * a) returning a future of some type
 32 |  * b) returning a future of void
 33 |  * c) returning void (fire and forget delegation)
 34 |  */
 35 | 
 36 | /* case 2Aa */
 37 | template<typename Types, typename Function, Function f, typename O, typename... Ps>
 38 | auto delegated_function_future(char* pptr, O&& o, Ps&&... ps)
 39 | -> typename
 40 | 	std::enable_if<
 41 | 		std::is_same<types<>, Types>::value
 42 | 		&& !std::is_same<Function, std::nullptr_t>::value
 43 | 		&& std::is_member_function_pointer<Function>::value
 44 | 	, void>::type
 45 | {
 46 | 	typedef decltype(f(ps...)) R;
 47 | 	unpack_promise<R> p(pptr);
 48 | 	p.set_value((o.*f)(std::forward<Ps>(ps)...));
 49 | }
 50 | 
 51 | /* case 2Ba */
 52 | template<typename Types, typename Ignored, Ignored i, typename Function, typename O, typename... Ps>
 53 | auto delegated_function_future(char* pptr, Function&& f, O&& o, Ps&&... ps)
 54 | -> typename
 55 | 	std::enable_if<
 56 | 		std::is_same<types<>, Types>::value
 57 | 		&& std::is_same<Ignored, std::nullptr_t>::value
 58 | 		&& std::is_member_function_pointer<Function>::value
 59 | 	, void>::type
 60 | {
 61 | 	typedef decltype(f(ps...)) R;
 62 | 	unpack_promise<R> p(pptr);
 63 | 	p.set_value((o.*f)(std::forward<Ps>(ps)...));
 64 | }
 65 | 
 66 | /* case 1Aa */
 67 | template<typename Types, typename Function, Function f, typename... Ps>
 68 | auto delegated_function_future(char* pptr, Ps&&... ps)
 69 | -> typename std::enable_if<std::is_same<types<>, Types>::value && !std::is_same<Function, std::nullptr_t>::value, void>::type
 70 | {
 71 | 	typedef decltype(f(ps...)) R;
 72 | 	unpack_promise<R> p(pptr);
 73 | 	p.set_value(f(std::forward<Ps>(ps)...));
 74 | }
 75 | 
 76 | /* case 1Ba */
 77 | template<typename Types, typename Ignored, Ignored i, typename Function, typename... Ps>
 78 | auto delegated_function_future(char* pptr, Function&& f, Ps&&... ps)
 79 | -> typename
 80 | 	std::enable_if<
 81 | 		std::is_same<types<>, Types>::value
 82 | 		&& std::is_same<Ignored, std::nullptr_t>::value
 83 | 		&& !std::is_member_function_pointer<Function>::value
 84 | 	, void>::type
 85 | {
 86 | 	typedef decltype(f(ps...)) R;
 87 | 	unpack_promise<R> p(pptr);
 88 | 	p.set_value(f(std::forward<Ps>(ps)...));
 89 | }
 90 | 
 91 | /* cases Aa?, unrolling */
 92 | template<typename Types, typename Function, Function f, typename... Ps>
 93 | auto delegated_function_future(char* buf, Ps&&... ps)
 94 | -> typename std::enable_if<!std::is_same<types<>, Types>::value && !std::is_same<Function, std::nullptr_t>::value, void>::type
 95 | {
 96 | 	typedef typename Types::type T;
 97 | 	auto ptr = reinterpret_cast<T*>(buf);
 98 | 	delegated_function_future<typename Types::tail, Function, f>(buf+sizeof(T), std::forward<Ps>(ps)..., std::forward<T>(*ptr));
 99 | }
100 | 
101 | /* cases Ba?, unrolling */
102 | template<typename Types, typename Ignored, std::nullptr_t i, typename... Ps>
103 | auto delegated_function_future(char* buf, Ps&&... ps)
104 | -> typename
105 | 	std::enable_if<
106 | 		true
107 | 		//std::is_same<types<>, typename Types::tail>::value
108 | 		&& std::is_same<Ignored, std::nullptr_t>::value
109 | 	, void>::type
110 | {
111 | 	typedef typename Types::type T;
112 | 	auto ptr = reinterpret_cast<T*>(buf);
113 | 	delegated_function_future<typename Types::tail, Ignored, i>(buf+sizeof(T), std::forward<Ps>(ps)..., std::forward<T>(*ptr));
114 | }
115 | 
116 | /* case 2Ab */
117 | template<typename Types, typename Function, Function f, typename O, typename... Ps>
118 | auto delegated_void_function_future(char* pptr, O&& o, Ps&&... ps)
119 | -> typename
120 | 	std::enable_if<
121 | 		std::is_same<types<>, Types>::value
122 | 		&& !std::is_same<Function, std::nullptr_t>::value
123 | 		&& std::is_member_function_pointer<Function>::value
124 | 	, void>::type
125 | {
126 | 	typedef decltype(f(ps...)) R;
127 | 	static_assert(std::is_same<R, void>::value, "void code path used for non-void function");
128 | 	unpack_promise<R> p(pptr);
129 | 	(o.*f)(std::forward<Ps>(ps)...);
130 | 	p.set_value();
131 | }
132 | 
133 | /* case 2Bb */
134 | template<typename Types, typename Ignored, Ignored i, typename Function, typename O, typename... Ps>
135 | auto delegated_void_function_future(char* pptr, Function&& f, O&& o, Ps&&... ps)
136 | -> typename
137 | 	std::enable_if<
138 | 		std::is_same<types<>, Types>::value
139 | 		&& std::is_same<Ignored, std::nullptr_t>::value
140 | 		&& std::is_member_function_pointer<Function>::value
141 | 	, void>::type
142 | {
143 | 	typedef decltype(f(ps...)) R;
144 | 	static_assert(std::is_same<R, void>::value, "void code path used for non-void function");
145 | 	unpack_promise<R> p(pptr);
146 | 	(o.*f)(std::forward<Ps>(ps)...);
147 | 	p.set_value();
148 | }
149 | /** wrapper function for void operations */
150 | /* 1Ab */
151 | template<typename Types, typename Function, Function f, typename... Ps>
152 | auto delegated_void_function_future(char* pptr, Ps&&... ps)
153 | -> typename
154 | 	std::enable_if<
155 | 		std::is_same<types<>, Types>::value
156 | 	, void>::type
157 | {
158 | 	typedef decltype(f(ps...)) R;
159 | 	static_assert(std::is_same<R, void>::value, "void code path used for non-void function");
160 | 	unpack_promise<R> p(pptr);
161 | 	f(ps...);
162 | 	p.set_value();
163 | }
164 | /* case 1Bb */
165 | template<typename Types, typename Ignored, Ignored i, typename Function, typename... Ps>
166 | auto delegated_void_function_future(char* pptr, Function&& f, Ps&&... ps)
167 | -> typename
168 | 	std::enable_if<
169 | 		std::is_same<types<>, Types>::value
170 | 		&& std::is_same<Ignored, std::nullptr_t>::value
171 | 		&& !std::is_member_function_pointer<Function>::value
172 | 	, void>::type
173 | {
174 | 	typedef decltype(f(ps...)) R;
175 | 	static_assert(std::is_same<R, void>::value, "void code path used for non-void function");
176 | 	unpack_promise<R> p(pptr);
177 | 	f(std::forward<Ps>(ps)...);
178 | 	p.set_value();
179 | }
180 | 
181 | /* Ab unrolling */
182 | template<typename Types, typename Function, Function f, typename... Ps>
183 | auto delegated_void_function_future(char* buf, Ps&&... ps)
184 | -> typename
185 | 	std::enable_if<
186 | 		!std::is_same<types<>, Types>::value
187 | 		&& !std::is_same<Function, std::nullptr_t>::value
188 | 	, void>::type
189 | {
190 | 	typedef typename Types::type T;
191 | 	auto ptr = reinterpret_cast<T*>(buf);
192 | 	delegated_void_function_future<typename Types::tail, Function, f>(buf+sizeof(T), std::forward<Ps>(ps)..., std::forward<T>(*ptr));
193 | }
194 | /* cases Bb, unrolling */
195 | template<typename Types, typename Ignored, std::nullptr_t i, typename... Ps>
196 | auto delegated_void_function_future(char* buf, Ps&&... ps)
197 | -> typename
198 | 	std::enable_if<
199 | 		std::is_same<types<>, typename Types::tail>::value
200 | 		&& std::is_same<Ignored, std::nullptr_t>::value
201 | 	, void>::type
202 | {
203 | 	typedef typename Types::type T;
204 | 	auto ptr = reinterpret_cast<T*>(buf);
205 | 	delegated_void_function_future<typename Types::tail, Ignored, i>(buf+sizeof(T), std::forward<Ps>(ps)..., std::forward<T>(*ptr));
206 | }
207 | 
208 | /** wrapper function for operations without associated future */
209 | /* case 2Ac */
210 | template<typename Types, typename Function, Function f, typename O, typename... Ps>
211 | auto delegated_function_nofuture(char*, O&& o, Ps&&... ps)
212 | -> typename
213 | 	std::enable_if<
214 | 		std::is_same<types<>, Types>::value
215 | 		&& !std::is_same<Function, std::nullptr_t>::value
216 | 		&& std::is_member_function_pointer<Function>::value
217 | 	, void>::type
218 | {
219 | 	(o.*f)(std::forward<Ps>(ps)...);
220 | }
221 | 
222 | /* case 2Bc */
223 | template<typename Types, typename Ignored, Ignored i, typename Function, typename O, typename... Ps>
224 | auto delegated_function_nofuture(char*, Function&& f, O&& o, Ps&&... ps)
225 | -> typename
226 | 	std::enable_if<
227 | 		std::is_same<types<>, Types>::value
228 | 		&& std::is_same<Ignored, std::nullptr_t>::value
229 | 		&& std::is_member_function_pointer<Function>::value
230 | 	, void>::type
231 | {
232 | 	(o.*f)(std::forward<Ps>(ps)...);
233 | }
234 | 
235 | /* case 1Ac */
236 | template<typename Types, typename Function, Function f, typename... Ps>
237 | auto delegated_function_nofuture(char*, Ps&&... ps)
238 | -> typename std::enable_if<std::is_same<types<>, Types>::value && !std::is_same<Function, std::nullptr_t>::value, void>::type
239 | {
240 | 	f(std::forward<Ps>(ps)...);
241 | }
242 | 
243 | /* case 1Bc */
244 | template<typename Types, typename Ignored, Ignored i, typename Function, typename... Ps>
245 | auto delegated_function_nofuture(char*, Function&& f, Ps&&... ps)
246 | -> typename
247 | 	std::enable_if<
248 | 		std::is_same<types<>, Types>::value
249 | 		&& std::is_same<Ignored, std::nullptr_t>::value
250 | 		&& !std::is_member_function_pointer<Function>::value
251 | 	, void>::type
252 | {
253 | 	f(std::forward<Ps>(ps)...);
254 | }
255 | 
256 | /* cases Ac, unrolling */
257 | template<typename Types, typename Function, Function f, typename... Ps>
258 | auto delegated_function_nofuture(char* buf, Ps&&... ps)
259 | -> typename std::enable_if<!std::is_same<types<>, Types>::value && !std::is_same<Function, std::nullptr_t>::value, void>::type
260 | {
261 | 	typedef typename Types::type T;
262 | 	auto ptr = reinterpret_cast<T*>(buf);
263 | 	delegated_function_nofuture<typename Types::tail, Function, f>(buf+sizeof(T), std::forward<Ps>(ps)..., std::forward<T>(*ptr));
264 | }
265 | 
266 | /* cases Bc, unrolling */
267 | template<typename Types, typename Ignored, std::nullptr_t i, typename... Ps>
268 | auto delegated_function_nofuture(char* buf, Ps&&... ps)
269 | -> typename
270 | 	std::enable_if<
271 | 		true
272 | 		//std::is_same<types<>, typename Types::tail>::value
273 | 		&& std::is_same<Ignored, std::nullptr_t>::value
274 | 	, void>::type
275 | {
276 | 	typedef typename Types::type T;
277 | 	auto ptr = reinterpret_cast<T*>(buf);
278 | 	delegated_function_nofuture<typename Types::tail, Ignored, i>(buf+sizeof(T), std::forward<Ps>(ps)..., std::forward<T>(*ptr));
279 | }
280 | 
281 | /**
282 |  * @brief policies for starvation freedom
283 |  */
284 | enum class starvation_policy_t {may_starve, starvation_free};
285 | 
286 | /**
287 |  * @brief queue delegation base class
288 |  * @tparam MLock mutual exclusion lock
289 |  * @tparam DQueue delegation queue
290 |  */
291 | template<class MLock, class DQueue, starvation_policy_t starvation_policy=starvation_policy_t::starvation_free>
292 | class qdlock_base {
293 | 	public:
294 | 		void* __data;
295 | 		char pad1[128];
296 | 		MLock mutex_lock;
297 | 		char pad2[128];
298 | 	protected:
299 | 		DQueue delegation_queue;
300 | 
301 | 		/** executes the operation */
302 | 
303 | 		/* case 1Aa */
304 | 		template<typename Function, Function f, typename Promise, typename... Ps>
305 | 		auto execute(Promise r, Ps&&... ps)
306 | 		-> typename
307 | 			std::enable_if<
308 | 				!std::is_same<Function, std::nullptr_t>::value
309 | 				&& !std::is_same<void, decltype(f(ps...))>::value
310 | 				&& std::is_function<typename std::remove_pointer<Function>::type>::value
311 | 			, void>::type
312 | 		{
313 | //			static_assert(std::is_same<R, decltype(f(ps...))>::value, "promise and function have different return types");
314 | 			r.set_value(f(std::move(ps)...));
315 | 		}
316 | 
317 | 		/** alternative for operations with a promise, case member function pointer in template, object specified */
318 | 		/* case 2Aa */
319 | 		template<typename Function, Function f, typename R, typename O, typename... Ps>
320 | 		auto execute(std::promise<typename std::result_of<decltype(&O::Function)(O, Ps...)>::type> r, O&& o, Ps&&... ps)
321 | 		-> typename
322 | 			std::enable_if<
323 | 				!std::is_same<Function, std::nullptr_t>::value
324 | 				&& std::is_member_function_pointer<Function>::value
325 | //				&& std::is_same< decltype(O::Function(ps...)), decltype(o.*f(ps...))>::value
326 | 			, void>::type
327 | 		{
328 | 		//	static_assert(std::is_same<R, decltype(f(ps...))>::value, "promise and function have different return types");
329 | 			r.set_value((o.*f)(std::move(ps)...));
330 | 		}
331 | 
332 | 		/** alternative for operations with a promise, case function pointer specified */
333 | 		/* case 1Ba */
334 | 		template<typename Ignored, Ignored i, typename R, typename Function, typename... Ps>
335 | 		auto execute(std::promise<typename std::result_of<Function(Ps...)>::type> r, Function&& f, Ps&&... ps)
336 | 		-> typename
337 | 			std::enable_if<
338 | 				std::is_same<Ignored, std::nullptr_t>::value
339 | 				&& !std::is_same<void, typename std::result_of<Function(Ps...)>::type>::value
340 | 			, void>::type
341 | 		{
342 | 		//	static_assert(std::is_same<R, decltype(f(ps...))>::value, "promise and function have different return types");
343 | 			static_assert(std::is_same<Ignored, std::nullptr_t>::value, "functors cannot be used when specifying a function");
344 | 			r.set_value(f(std::move(ps)...));
345 | 		}
346 | 
347 | 		/** alternative for operations with a promise, case member function pointer and object specified */
348 | 		/* case 2Ba */
349 | 		template<typename Ignored, Ignored i, typename R, typename Function, typename O, typename... Ps>
350 | 		auto execute(std::promise<typename std::result_of<Function(Ps...)>::type> r, Function&& f, O&& o, Ps&&... ps)
351 | 		-> typename
352 | 			std::enable_if<
353 | 				std::is_same<Ignored, std::nullptr_t>::value
354 | 				&& std::is_member_function_pointer<Function>::value
355 | 			, void>::type
356 | 		{
357 | 		//	static_assert(std::is_same<R, decltype(f(ps...))>::value, "promise and function have different return types");
358 | 			static_assert(std::is_same<Ignored, std::nullptr_t>::value, "functors cannot be used when specifying a function");
359 | 			r.set_value((o.*f)(std::move(ps)...));
360 | 		}
361 | 
362 | 		/** alternative for operations which return void */
363 | 		/* case 1Ab */
364 | 		template<typename Function, Function f, typename Promise, typename... Ps>
365 | 		auto execute(std::promise<void> r, Ps&&... ps)
366 | 		-> typename
367 | 			std::enable_if<
368 | 				!std::is_same<Function, std::nullptr_t>::value
369 | 				&& std::is_same<void, decltype(f(ps...))>::value
370 | 				&& std::is_function<typename std::remove_pointer<Function>::type>::value
371 | 			, void>::type
372 | 		{
373 | 			f(std::move(ps)...);
374 | 			r.set_value();
375 | 		}
376 | 
377 | 		/** alternative for operations with a promise, case member function pointer in template, object specified */
378 | 		/* case 2Ab */
379 | 		template<typename Function, Function f, typename R, typename O, typename... Ps>
380 | 		auto execute(std::promise<void> r, O&& o, Ps&&... ps)
381 | 		-> typename
382 | 			std::enable_if<
383 | 				!std::is_same<Function, std::nullptr_t>::value
384 | 				&& std::is_member_function_pointer<Function>::value
385 | //				&& std::is_same< decltype(O::Function(ps...)), decltype(o.*f(ps...))>::value
386 | 			, void>::type
387 | 		{
388 | 		//	static_assert(std::is_same<R, decltype(f(ps...))>::value, "promise and function have different return types");
389 | 			(o.*f)(std::move(ps)...);
390 | 			r.set_value();
391 | 		}
392 | 
393 | 		/** alternative for operations with a promise, case function pointer specified */
394 | 		/* case 1Bb */
395 | 		template<typename Ignored, Ignored i, typename R, typename Function, typename... Ps>
396 | 		auto execute(std::promise<void> r, Function&& f, Ps&&... ps)
397 | 		-> typename std::enable_if<std::is_same<Ignored, std::nullptr_t>::value, void>::type
398 | 		{
399 | 		//	static_assert(std::is_same<R, decltype(f(ps...))>::value, "promise and function have different return types");
400 | 			static_assert(std::is_same<Ignored, std::nullptr_t>::value, "functors cannot be used when specifying a function");
401 | 			f(std::move(ps)...);
402 | 			r.set_value();
403 | 		}
404 | 
405 | 		/** alternative for operations with a promise, case member function pointer and object specified */
406 | 		/* case 2Bb */
407 | 		template<typename Ignored, Ignored i, typename R, typename Function, typename O, typename... Ps>
408 | 		auto execute(std::promise<void> r, Function&& f, O&& o, Ps&&... ps)
409 | 		-> typename
410 | 			std::enable_if<
411 | 				std::is_same<Ignored, std::nullptr_t>::value
412 | 				&& std::is_member_function_pointer<Function>::value
413 | 			, void>::type
414 | 		{
415 | 		//	static_assert(std::is_same<R, decltype(f(ps...))>::value, "promise and function have different return types");
416 | 			static_assert(std::is_same<Ignored, std::nullptr_t>::value, "functors cannot be used when specifying a function");
417 | 			(o.*f)(std::move(ps)...);
418 | 			r.set_value();
419 | 		}
420 | 
421 | 		/** alternative for operations without a promise, case function pointer in template */
422 | 		/* case 1Ac */
423 | 		template<typename Function, Function f, typename Promise, typename... Ps>
424 | 		auto execute(std::nullptr_t, Ps&&... ps)
425 | 		-> typename
426 | 			std::enable_if<
427 | 				!std::is_same<Function, std::nullptr_t>::value
428 | 				&& std::is_function<typename std::remove_pointer<Function>::type>::value
429 | 			, void>::type
430 | 		{
431 | 			f(std::move(ps)...);
432 | 		}
433 | 
434 | 		/** alternative for operations without a promise, case member function pointer in template, object specified */
435 | 		/* case 2Ac */
436 | 		template<typename Function, Function f, typename Promise, typename O, typename... Ps>
437 | 		auto execute(std::nullptr_t, O&& o, Ps&&... ps)
438 | 		-> typename
439 | 			std::enable_if<
440 | 				!std::is_same<Function, std::nullptr_t>::value
441 | 				&& std::is_member_function_pointer<Function>::value
442 | //				&& std::is_same< decltype(O::Function(ps...)), decltype(o.*f(ps...))>::value
443 | 			, void>::type
444 | 		{
445 | 			(o.*f)(std::move(ps)...);
446 | 		}
447 | 
448 | 		/** alternative for operations without a promise, case function pointer specified */
449 | 		/* case 1Bc */
450 | 		template<typename Ignored, Ignored i, typename Promise, typename Function, typename... Ps>
451 | 		auto execute(std::nullptr_t, Function&& f, Ps&&... ps)
452 | 		-> typename std::enable_if<std::is_same<Ignored, std::nullptr_t>::value, void>::type
453 | 		{
454 | 			static_assert(std::is_same<Ignored, std::nullptr_t>::value, "functors cannot be used when specifying a function");
455 | 			f(std::move(ps)...);
456 | 		}
457 | 
458 | 		/** alternative for operations without a promise, case member function pointer and object specified */
459 | 		/* case 2Bc */
460 | 		template<typename Ignored, Ignored i, typename Promise, typename Function, typename O, typename... Ps>
461 | 		auto execute(std::nullptr_t, Function&& f, O&& o, Ps&&... ps)
462 | 		-> typename
463 | 			std::enable_if<
464 | 				std::is_same<Ignored, std::nullptr_t>::value
465 | 				&& std::is_member_function_pointer<Function>::value
466 | 			, void>::type
467 | 		{
468 | 			static_assert(std::is_same<Ignored, std::nullptr_t>::value, "functors cannot be used when specifying a function");
469 | 			(o.*f)(std::move(ps)...);
470 | 		}
471 | 
472 | 
473 | 		/* ENQUEUE IMPLEMENTATIONS */
474 | 
475 | 		/** maybe enqueues the operation */
476 | 		/* case Aa */
477 | 		template<typename Function, Function f, typename R, typename... Ps>
478 | 		auto enqueue(std::promise<R>* r, Ps*... ps)
479 | 		-> typename
480 | 			std::enable_if<
481 | 				!std::is_same<Function, std::nullptr_t>::value
482 | 				&& !std::is_same<R, void>::value
483 | 			, typename DQueue::status>::type
484 | 		{
485 | 			static_assert(std::is_same<R, decltype(f((*ps)...))>::value, "promise and function have different return types");
486 | 			void (*d)(char*) = delegated_function_future<types<Ps...>, Function, f>;
487 | 			return delegation_queue.enqueue(d, std::move(ps)..., std::move(r));
488 | 		}
489 | 
490 | 		/** alternative with returning a result, case function specified as argument */
491 | 		/* case Ba */
492 | 		template<typename Ignored, std::nullptr_t i, typename R, typename Function, typename... Ps>
493 | 		auto enqueue(std::promise<R>* r, Function* f, Ps*... ps)
494 | 		-> typename
495 | 			std::enable_if<
496 | 				std::is_same<Ignored, std::nullptr_t>::value
497 | 				&& !std::is_same<R, void>::value
498 | 			, typename DQueue::status>::type
499 | 		{
500 | 			static_assert(std::is_same<R, decltype((*f)(*ps...))>::value, "promise and function have different return types");
501 | 			void (*d)(char*) = delegated_function_future<types<Function, Ps...>, std::nullptr_t, nullptr>;
502 | 			return delegation_queue.enqueue(d, std::move(f), std::move(ps)..., std::move(r));
503 | 		}
504 | 
505 | 		/** alternative for operations which return void */
506 | 		/* case Ab */
507 | 		template<typename Function, Function f, typename... Ps>
508 | 		auto enqueue(std::promise<void>* r, Ps*... ps)
509 | 		-> typename DQueue::status {
510 | 			void (*d)(char*) = delegated_void_function_future<types<Ps...>, Function, f>;
511 | 			return delegation_queue.enqueue(d, ps..., std::move(r));
512 | 		}
513 | 		/** alternative with returning a void, case function specified as argument */
514 | 		/* case Bb */
515 | 		template<typename Ignored, std::nullptr_t i, typename R, typename... Ps>
516 | 		auto enqueue(std::promise<void>* r, Ps*... ps)
517 | 		-> typename
518 | 			std::enable_if<
519 | 				std::is_same<Ignored, std::nullptr_t>::value
520 | 			, typename DQueue::status>::type
521 | 		{
522 | 			static_assert(std::is_same<R, decltype(f(ps...))>::value, "promise and function have different return types");
523 | 			void (*d)(char*) = delegated_void_function_future<types<Ps...>, std::nullptr_t, nullptr>;
524 | 			return delegation_queue.enqueue(d, std::move(ps...), std::move(r));
525 | 		}
526 | 
527 | 
528 | 		/** alternative without returning a result, case function specified in template */
529 | 		/* case Ac */
530 | 		template<typename Function, Function f, typename... Ps>
531 | 		auto enqueue(std::nullptr_t*, Ps*... ps)
532 | 		-> typename std::enable_if<!std::is_same<Function, std::nullptr_t>::value, typename DQueue::status>::type
533 | 		{
534 | 			void (*d)(char*) = delegated_function_nofuture<types<Ps...>, Function, f>;
535 | 			return delegation_queue.enqueue(d, ps...);
536 | 		}
537 | 
538 | 		/** alternative without returning a result, case function specified as argument */
539 | 		/* case Bc */
540 | 		template<typename Ignored, std::nullptr_t i, typename... Ps>
541 | 		auto enqueue(std::nullptr_t*, Ps*... ps)
542 | 		-> typename DQueue::status
543 | 		{
544 | 			void (*d)(char*) = delegated_function_nofuture<types<Ps...>, std::nullptr_t, nullptr>;
545 | 			return delegation_queue.enqueue(d, ps...);
546 | 		}
547 | 
548 | 	public:
549 | 		struct no_promise {
550 | 			typedef std::nullptr_t promise;
551 | 			typedef std::nullptr_t future;
552 | 			static promise create_promise(promise**) {
553 | 				return nullptr;
554 | 			}
555 | 			static future create_future(promise) {
556 | 				return nullptr;
557 | 			}
558 | 		};
559 | 
560 | 		struct no_reader_sync {
561 | 			static void wait_writers(qdlock_base<MLock, DQueue, starvation_policy>*) {};
562 | 			static void wait_readers(qdlock_base<MLock, DQueue, starvation_policy>*) {};
563 | 		};
564 | 		struct no_hierarchy_sync {
565 | 			static void lock(qdlock_base<MLock, DQueue, starvation_policy>*) {};
566 | 			static void unlock(qdlock_base<MLock, DQueue, starvation_policy>*) {};
567 | 		};
568 | #if 0
569 | 		template<typename T>
570 | 		class wrapped_promise : public std::promise<T> {
571 | 			wrapped_promise** co_owner;
572 | 			public:
573 | 				wrapped_promise() = delete;
574 | 				wrapped_promise(wrapped_promise** ptr) : co_owner(ptr) {
575 | 					*co_owner = this;
576 | 				}
577 | 				wrapped_promise(wrapped_promise&) = delete;
578 | 				wrapped_promise(wrapped_promise&& rhs) : std::promise<T>(std::move(rhs)), co_owner(rhs.co_owner), active(rhs.active) {
579 | 					if(active) *co_owner = this;
580 | 				}
581 | 				wrapped_promise& operator=(wrapped_promise&) = delete;
582 | 				wrapped_promise& operator=(wrapped_promise&& rhs) {
583 | 					std::promise<T>::operator=(std::move(rhs));
584 | 					co_owner = rhs.co_owner;
585 | 					active = rhs.active;
586 | 					if(active) *co_owner = this;
587 | 					return *this;
588 | 				}
589 | 		};
590 | #endif
591 | 
592 | 		template<typename R>
593 | 		struct std_promise {
594 | 			typedef std::promise<R> promise;
595 | 			typedef waiting_future<R> future;
596 | 			static promise create_promise() {
597 | 				return promise();
598 | 			}
599 | 			static future create_future(promise& p) {
600 | 				return p.get_future();
601 | 			}
602 | 		};
603 | 	private:
604 | 		template<typename Function, Function f, typename Promise, typename RSync, typename HSync, typename... Ps>
605 | 		auto helper(Promise&& result, Ps&&... ps)
606 | 		-> void {
607 | 			HSync::lock(this);
608 | 			this->delegation_queue.open();
609 | 			RSync::wait_readers(this);
610 | 			execute<Function, f, Promise, Ps...>(std::move(result), std::forward<Ps>(ps)...);
611 | 			this->delegation_queue.flush();
612 | 			HSync::unlock(this);
613 | 		}
614 | 
615 | 		template<typename Function, Function f, typename Promise, typename RSync, typename HSync, typename... Ps>
616 | 		auto executor(Promise&& result, Ps&&... ps)
617 | 		-> void {
618 | 			HSync::lock(this);
619 | 			RSync::wait_readers(this);
620 | 			execute<Function, f, Promise, Ps...>(std::move(result), std::forward<Ps>(ps)...);
621 | 			HSync::unlock(this);
622 | 		}
623 | 
624 | 		template<typename Function, Function f, typename Promise, typename... Ps>
625 | 		bool try_enqueue(int attempts, Promise* result, Ps... ps) {
626 | 			typename DQueue::status status = DQueue::status::CLOSED;
627 | 			for(int i = 1; i <= attempts; i++) {
628 | 				status = enqueue<Function, f>(result, ps...);
629 | 				if(status == DQueue::status::SUCCESS) {
630 | 					return true;
631 | 				} else if(status == DQueue::status::FULL) {
632 | 					qd::pause();
633 | 					//break;
634 | 				} else {
635 | 					qd::pause();
636 | 				}
637 | 			}
638 | 			return false;
639 | 		}
640 | 	public:
641 | 		//-> typename std::conditional<std::is_same<std::nullptr_t, typename Promise::promise>::value, void, typename Promise::future>::type
642 | 		template<typename Function, Function f, typename Promise, typename RSync, typename HSync, typename... Ps>
643 | 		auto delegate(Promise&& result, Ps&&... ps)
644 | 		-> void
645 | 		{
646 | 			RSync::wait_writers(this);
647 | 
648 | 			if(this->mutex_lock.try_lock()) {
649 | 				executor<Function, f, Promise, RSync, HSync, Ps...>(std::move(result), std::forward<Ps>(ps)...);
650 | 				this->mutex_lock.unlock();
651 | 				return;
652 | 			}
653 | 			/* for guaranteed starvation freedom add a limit here */
654 | 			for(unsigned int retries = 1; (starvation_policy == starvation_policy_t::may_starve) || retries < 512; retries++) {
655 | 				/* retry enqueueing a couple of times if CLOSED */
656 | 				if (try_enqueue<Function, f>(1, &result, (&ps)...)) {
657 | 					return;
658 | 				}
659 | 				bool lock_acquired;
660 | 				/** @todo magic number 127 */
661 | 				if(retries % (127 + 1) == 0) {
662 | 					lock_acquired = this->mutex_lock.try_lock_or_wait();
663 | 				} else {
664 | 					lock_acquired = this->mutex_lock.try_lock();
665 | 				}
666 | 				if(lock_acquired) {
667 | 					helper<Function, f, Promise, RSync, HSync, Ps...>(std::move(result), std::forward<Ps>(ps)...);
668 | 					this->mutex_lock.unlock();
669 | 					return;
670 | 				}
671 | 				std::this_thread::yield();
672 | 			}
673 | 			/* if starvation_policy is may_starve, then this is dead code, only relevant for the starvation_free variant */
674 | 			this->mutex_lock.lock();
675 | 			executor<Function, f, Promise, RSync, HSync, Ps...>(std::move(result), std::forward<Ps>(ps)...);
676 | 			this->mutex_lock.unlock();
677 | 		}
678 | };
679 | 
680 | #endif /* qdlock_base_hpp */
681 | 


--------------------------------------------------------------------------------