├── .gitignore
├── CMakeLists.txt
├── Fifo1.hpp
├── Fifo2.hpp
├── Fifo3.hpp
├── Fifo3a.hpp
├── Fifo4.hpp
├── Fifo4a.hpp
├── Fifo4b.hpp
├── Fifo5.hpp
├── Fifo5a.hpp
├── Fifo5b.hpp
├── LICENSE
├── Mutex.hpp
├── README.md
├── TryLock.hpp
├── bench.cpp
├── bench.hpp
├── bench_all.cpp
├── boost_lockfree.cpp
├── fifo1.cpp
├── fifo2.cpp
├── fifo3.cpp
├── fifo3a.cpp
├── fifo4.cpp
├── fifo4a.cpp
├── fifo4b.cpp
├── fifo5.cpp
├── fifo5a.cpp
├── fifo5b.cpp
├── mutex.cpp
├── rigtorp.cpp
├── rigtorp.hpp
├── run_all.sh
├── tryLock.cpp
└── unitTests.cpp


/.gitignore:
--------------------------------------------------------------------------------
1 | build
2 | *.swp
3 | 


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required(VERSION 3.22)
 2 | set(CMAKE_C_COMPILER /usr/bin/gcc-12)
 3 | set(CMAKE_CXX_COMPILER /usr/bin/g++-12)
 4 | 
 5 | project(cppcon2023)
 6 | 
 7 | cmake_path(GET PROJECT_BINARY_DIR STEM buildDir)
 8 | string(TOLOWER ${buildDir} buildDir)
 9 | if(buildDir STREQUAL "release")
10 |     set(buildType RelWithDebInfo)
11 | elseif(buildDir STREQUAL "debug")
12 |     set(buildType Debug)
13 | else()
14 |     set(buildType Debug)
15 | endif()
16 | if(NOT CMAKE_BUILD_TYPE)
17 |     set(CMAKE_BUILD_TYPE ${buildType})
18 | endif()
19 | 
20 | set(BOOST_ROOT /mnt/c/Users/Charles/AppData/Local/lxss/rootfs/usr/)
21 | find_package(Boost REQUIRED)
22 | 
23 | set(CMAKE_CXX_STANDARD 23)
24 | set(CMAKE_CXX_STANDARD_REQUIRED true)
25 | set(CMAKE_CXX_EXTENSIONS OFF)
26 | add_compile_options(-Wall -Werror -Wextra -Wconversion)
27 | add_compile_options(-Wno-unused-parameter)
28 | 
29 | function(add_fifo fifo)
30 |     add_executable(${fifo} ${CMAKE_SOURCE_DIR}/${fifo}.cpp)
31 |     target_link_libraries(${fifo} pthread)
32 | 
33 |     add_executable(${fifo}.tsan ${CMAKE_SOURCE_DIR}/${fifo}.cpp)
34 |     target_compile_options(${fifo}.tsan PRIVATE -fsanitize=thread)
35 |     target_link_libraries(${fifo}.tsan PRIVATE pthread tsan)
36 | endfunction()
37 | 
38 | 
39 | add_fifo(fifo1)
40 | add_fifo(fifo2)
41 | add_fifo(fifo3)
42 | add_fifo(fifo3a)
43 | add_fifo(fifo4)
44 | add_fifo(fifo4a)
45 | add_fifo(fifo4b)
46 | add_fifo(fifo5)
47 | add_fifo(fifo5a)
48 | add_fifo(fifo5b)
49 | add_fifo(boost_lockfree)
50 | add_fifo(rigtorp)
51 | target_compile_options(rigtorp PRIVATE -Wno-interference-size)
52 | target_compile_options(rigtorp.tsan PRIVATE -Wno-interference-size)
53 | add_fifo(mutex)
54 | add_fifo(tryLock)
55 | 
56 | 
57 | include(GoogleTest)
58 | enable_testing()
59 | 
60 | add_executable(unitTests unitTests.cpp)
61 | target_compile_options(unitTests PRIVATE -fsanitize=undefined -fsanitize=address)
62 | target_link_libraries(unitTests PRIVATE asan ubsan gtest gtest_main)
63 | target_link_options(unitTests PRIVATE -fsanitize=undefined -fsanitize=address)
64 | gtest_add_tests(TARGET unitTests)
65 | 
66 | 
67 | find_package(benchmark REQUIRED)
68 | add_executable(bench bench.cpp)
69 | target_compile_options(bench PRIVATE -Wno-interference-size)
70 | target_link_libraries(bench PRIVATE benchmark::benchmark)
71 | 
72 | add_executable(bench_all bench_all.cpp)
73 | target_compile_options(bench_all PRIVATE -Wno-interference-size)
74 | 


--------------------------------------------------------------------------------
/Fifo1.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <cassert>
 4 | #include <cstdlib>
 5 | #include <memory>
 6 | 
 7 | 
 8 | /// Non-threadsafe circular FIFO; has data races
 9 | template<typename T, typename Alloc = std::allocator<T>>
10 | class Fifo1 : private Alloc
11 | {
12 | public:
13 |     using value_type = T;
14 |     using allocator_traits = std::allocator_traits<Alloc>;
15 |     using size_type = typename allocator_traits::size_type;
16 | 
17 |     explicit Fifo1(size_type capacity, Alloc const& alloc = Alloc{})
18 |         : Alloc{alloc}
19 |         , capacity_{capacity}
20 |         , ring_{allocator_traits::allocate(*this, capacity)}
21 |     {}
22 | 
23 |     // For consistency with other fifos
24 |     Fifo1(Fifo1 const&) = delete;
25 |     Fifo1& operator=(Fifo1 const&) = delete;
26 |     Fifo1(Fifo1&&) = delete;
27 |     Fifo1& operator=(Fifo1&&) = delete;
28 | 
29 |     ~Fifo1() {
30 |         while(not empty()) {
31 |             ring_[popCursor_ % capacity_].~T();
32 |             ++popCursor_;
33 |         }
34 |         allocator_traits::deallocate(*this, ring_, capacity_);
35 |     }
36 | 
37 | 
38 |     /// Returns the number of elements in the fifo
39 |     auto size() const noexcept {
40 |         assert(popCursor_ <= pushCursor_);
41 |         return pushCursor_ - popCursor_;
42 |     }
43 | 
44 |     /// Returns whether the container has no elements
45 |     auto empty() const noexcept { return size() == 0; }
46 | 
47 |     /// Returns whether the container has capacity_() elements
48 |     auto full() const noexcept { return size() == capacity(); }
49 | 
50 |     /// Returns the number of elements that can be held in the fifo
51 |     auto capacity() const noexcept { return capacity_; }
52 | 
53 | 
54 |     /// Push one object onto the fifo.
55 |     /// @return `true` if the operation is successful; `false` if fifo is full.
56 |     auto push(T const& value) {
57 |         if (full()) {
58 |             return false;
59 |         }
60 |         new (&ring_[pushCursor_ % capacity_]) T(value);
61 |         ++pushCursor_;
62 |         return true;
63 |     }
64 | 
65 |     /// Pop one object from the fifo.
66 |     /// @return `true` if the pop operation is successful; `false` if fifo is empty.
67 |     auto pop(T& value) {
68 |         if (empty()) {
69 |             return false;
70 |         }
71 |         value = ring_[popCursor_ % capacity_];
72 |         ring_[popCursor_ % capacity_].~T();
73 |         ++popCursor_;
74 |         return true;
75 |     }
76 | 
77 | private:
78 |     size_type capacity_;
79 |     T* ring_;
80 |     size_type pushCursor_{};
81 |     size_type popCursor_{};
82 | };
83 | 


--------------------------------------------------------------------------------
/Fifo2.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <atomic>
 4 | #include <cassert>
 5 | #include <memory>
 6 | 
 7 | 
 8 | /// Threadsafe but flawed circular FIFO
 9 | template<typename T, typename Alloc = std::allocator<T>>
10 | class Fifo2 : private Alloc
11 | {
12 | public:
13 |     using value_type = T;
14 |     using allocator_traits = std::allocator_traits<Alloc>;
15 |     using size_type = typename allocator_traits::size_type;
16 | 
17 |     explicit Fifo2(size_type capacity, Alloc const& alloc = Alloc{})
18 |         : Alloc{alloc}
19 |         , capacity_{capacity}
20 |         , ring_{allocator_traits::allocate(*this, capacity)}
21 |     {}
22 | 
23 |     ~Fifo2() {
24 |         while(not empty()) {
25 |             ring_[popCursor_ % capacity_].~T();
26 |             ++popCursor_;
27 |         }
28 |         allocator_traits::deallocate(*this, ring_, capacity_);
29 |     }
30 | 
31 |     /// Returns the number of elements in the fifo
32 |     auto size() const noexcept {
33 |         assert(popCursor_ <= pushCursor_);
34 |         return pushCursor_ - popCursor_;
35 |     }
36 | 
37 |     /// Returns whether the container has no elements
38 |     auto empty() const noexcept { return size() == 0; }
39 | 
40 |     /// Returns whether the container has capacity_() elements
41 |     auto full() const noexcept { return size() == capacity(); }
42 | 
43 |     /// Returns the number of elements that can be held in the fifo
44 |     auto capacity() const noexcept { return capacity_; }
45 | 
46 | 
47 | 
48 |     /// Push one object onto the fifo.
49 |     /// @return `true` if the operation is successful; `false` if fifo is full.
50 |     auto push(T const& value) {
51 |         if (full()) {
52 |             return false;
53 |         }
54 |         new (&ring_[pushCursor_ % capacity_]) T(value);
55 |         ++pushCursor_;
56 |         return true;
57 |     }
58 | 
59 |     /// Pop one object from the fifo.
60 |     /// @return `true` if the pop operation is successful; `false` if fifo is empty.
61 |     auto pop(T& value) {
62 |         if (empty()) {
63 |             return false;
64 |         }
65 |         value = ring_[popCursor_ % capacity_];
66 |         ring_[popCursor_ % capacity_].~T();
67 |         ++popCursor_;
68 |         return true;
69 |     }
70 | 
71 | private:
72 |     size_type capacity_;
73 |     T* ring_;
74 | 
75 |     using CursorType = std::atomic<size_type>;
76 |     static_assert(CursorType::is_always_lock_free);
77 | 
78 |     /// Loaded and stored by the push thread; loaded by the pop thread
79 |     CursorType pushCursor_;
80 | 
81 |     /// Loaded and stored by the pop thread; loaded by the push thread
82 |     CursorType popCursor_;
83 | };
84 | 


--------------------------------------------------------------------------------
/Fifo3.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <atomic>
  4 | #include <cassert>
  5 | #include <memory>
  6 | #include <new>
  7 | 
  8 | 
  9 | /// Threadsafe, efficient circular FIFO
 10 | template<typename T, typename Alloc = std::allocator<T>>
 11 | class Fifo3 : private Alloc
 12 | {
 13 | public:
 14 |     using value_type = T;
 15 |     using allocator_traits = std::allocator_traits<Alloc>;
 16 |     using size_type = typename allocator_traits::size_type;
 17 | 
 18 |     explicit Fifo3(size_type capacity, Alloc const& alloc = Alloc{})
 19 |         : Alloc{alloc}
 20 |         , capacity_{capacity}
 21 |         , ring_{allocator_traits::allocate(*this, capacity)}
 22 |     {}
 23 | 
 24 |     ~Fifo3() {
 25 |         while(not empty()) {
 26 |             element(popCursor_)->~T();
 27 |             ++popCursor_;
 28 |         }
 29 |         allocator_traits::deallocate(*this, ring_, capacity_);
 30 |     }
 31 | 
 32 | 
 33 |     /// Returns the number of elements in the fifo
 34 |     auto size() const noexcept {
 35 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 36 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
 37 | 
 38 |         assert(popCursor <= pushCursor);
 39 |         return pushCursor - popCursor;
 40 |     }
 41 | 
 42 |     /// Returns whether the container has no elements
 43 |     auto empty() const noexcept { return size() == 0; }
 44 | 
 45 |     /// Returns whether the container has capacity_() elements
 46 |     auto full() const noexcept { return size() == capacity(); }
 47 | 
 48 |     /// Returns the number of elements that can be held in the fifo
 49 |     auto capacity() const noexcept { return capacity_; }
 50 | 
 51 | 
 52 |     /// Push one object onto the fifo.
 53 |     /// @return `true` if the operation is successful; `false` if fifo is full.
 54 |     auto push(T const& value) {
 55 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 56 |         auto popCursor = popCursor_.load(std::memory_order_acquire);
 57 |         if (full(pushCursor, popCursor)) {
 58 |             return false;
 59 |         }
 60 |         new (element(pushCursor)) T(value);
 61 |         pushCursor_.store(pushCursor + 1, std::memory_order_release);
 62 |         return true;
 63 |     }
 64 | 
 65 |     /// Pop one object from the fifo.
 66 |     /// @return `true` if the pop operation is successful; `false` if fifo is empty.
 67 |     auto pop(T& value) {
 68 |         auto pushCursor = pushCursor_.load(std::memory_order_acquire);
 69 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
 70 |         if (empty(pushCursor, popCursor)) {
 71 |             return false;
 72 |         }
 73 |         value = *element(popCursor);
 74 |         element(popCursor)->~T();
 75 |         popCursor_.store(popCursor + 1, std::memory_order_release);
 76 |         return true;
 77 |     }
 78 | 
 79 | private:
 80 |     auto full(size_type pushCursor, size_type popCursor) const noexcept {
 81 |         return (pushCursor - popCursor) == capacity_;
 82 |     }
 83 |     static auto empty(size_type pushCursor, size_type popCursor) noexcept {
 84 |         return pushCursor == popCursor;
 85 |     }
 86 |     auto element(size_type cursor) noexcept {
 87 |         return &ring_[cursor % capacity_];
 88 |     }
 89 | 
 90 | private:
 91 |     size_type capacity_;
 92 |     T* ring_;
 93 | 
 94 |     using CursorType = std::atomic<size_type>;
 95 |     static_assert(CursorType::is_always_lock_free);
 96 | 
 97 |     // N.B. std::hardware_destructive_interference_size is not used directly
 98 |     // error: use of ‘std::hardware_destructive_interference_size’ [-Werror=interference-size]
 99 |     // note: its value can vary between compiler versions or with different ‘-mtune’ or ‘-mcpu’ flags
100 |     // note: if this use is part of a public ABI, change it to instead use a constant variable you define
101 |     // note: the default value for the current CPU tuning is 64 bytes
102 |     // note: you can stabilize this value with ‘--param hardware_destructive_interference_size=64’, or disable this warning with ‘-Wno-interference-size’
103 |     static constexpr auto hardware_destructive_interference_size = size_type{64};
104 | 
105 |     /// Loaded and stored by the push thread; loaded by the pop thread
106 |     alignas(hardware_destructive_interference_size) CursorType pushCursor_;
107 | 
108 |     /// Loaded and stored by the pop thread; loaded by the push thread
109 |     alignas(hardware_destructive_interference_size) CursorType popCursor_;
110 | 
111 |     // Padding to avoid false sharing with adjacent objects
112 |     char padding_[hardware_destructive_interference_size - sizeof(size_type)];
113 | };
114 | 


--------------------------------------------------------------------------------
/Fifo3a.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <atomic>
  4 | #include <cassert>
  5 | #include <memory>
  6 | #include <new>
  7 | 
  8 | 
  9 | /**
 10 |  * Threadsafe, efficient circular FIFO with constrained cursors
 11 |  *
 12 |  * This Fifo is useful when you need to constrain the cursor ranges. For
 13 |  * example say if the sizeof(CursorType) is 8 or 15. The cursors may
 14 |  * take on any value up to the fifo's capacity + 1. Furthermore, there
 15 |  * are no calculations where an intermediate cursor value is larger then
 16 |  * this number. And, finally, the cursors are never negative.
 17 |  *
 18 |  * The problem that must be resolved is how to distinguish an empty fifo
 19 |  * from a full one and still meet the above constraints. First, define
 20 |  * an empty fifo as when the pushCursor and popCursor are equal. We
 21 |  * cannot define a full fifo as pushCursor == popCursor + capacity.
 22 |  * Firstly, the intermediate value popCursor + capacity can overflow if
 23 |  * a signed cursor is used and if the cursors are constrained to
 24 |  * [0..capacity) there is no distiction between the full definition and
 25 |  * the empty definition.
 26 |  *
 27 |  * To resolve this we introduce the idea of a sentinal element by
 28 |  * allocating one more element than the capacity of the fifo and define
 29 |  * a full fifo as when the cursors are "one apart". That is,
 30 |  *
 31 |  * @code
 32 |  *    | pushCursor < popCursor:  pushCursor == popCursor - 1
 33 |  *    | popCursor < pushCursor:  popCursor == pushCursor - capacity
 34 |  *    |                   else:  false
 35 |  * @endcode
 36 |  */
 37 | template<typename T, typename Alloc = std::allocator<T>>
 38 | class Fifo3a : private Alloc
 39 | {
 40 | public:
 41 |     using value_type = T;
 42 |     using allocator_traits = std::allocator_traits<Alloc>;
 43 |     using size_type = typename allocator_traits::size_type;
 44 | 
 45 |     explicit Fifo3a(size_type capacity, Alloc const& alloc = Alloc{})
 46 |         : Alloc{alloc}
 47 |         , capacity_{capacity}
 48 |         , ring_{allocator_traits::allocate(*this, capacity + 1)}
 49 |     {}
 50 | 
 51 |     ~Fifo3a() {
 52 |         // TODO fix shouldn't matter for benchmark since it waits until
 53 |         // the fifo is empty and only need if destructors have side
 54 |         // effects.
 55 |         // while(not empty()) {
 56 |         //     ring_[popCursor_ & mask_].~T();
 57 |         //     ++popCursor_;
 58 |         // }
 59 |         allocator_traits::deallocate(*this, ring_, capacity_ + 1);
 60 |     }
 61 | 
 62 |     /// Returns the number of elements in the fifo
 63 |     auto size() const noexcept {
 64 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 65 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
 66 |         if (popCursor <= pushCursor) {
 67 |             return pushCursor - popCursor;
 68 |         } else {
 69 |             return capacity_ - (popCursor - (pushCursor + 1));
 70 |         }
 71 |     }
 72 | 
 73 |     /// Returns whether the container has no elements
 74 |     auto empty() const noexcept {
 75 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 76 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
 77 |         return empty(pushCursor, popCursor);
 78 |     }
 79 | 
 80 |     /// Returns whether the container has capacity() elements
 81 |     auto full() const noexcept {
 82 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 83 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
 84 |         return full(pushCursor, popCursor);
 85 |     }
 86 | 
 87 |     /// Returns the number of elements that can be held in the fifo
 88 |     auto capacity() const noexcept { return capacity_; }
 89 | 
 90 | 
 91 |     /// Push one object onto the fifo.
 92 |     /// @return `true` if the operation is successful; `false` if fifo is full.
 93 |     auto push(T const& value) {
 94 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 95 |         auto popCursor = popCursor_.load(std::memory_order_acquire);
 96 |         if (full(pushCursor, popCursor)) {
 97 |             return false;
 98 |         }
 99 |         new (&ring_[pushCursor]) T(value);
100 |         if (pushCursor == capacity_) {
101 |             pushCursor_.store(0, std::memory_order_release);
102 |         } else {
103 |             pushCursor_.store(pushCursor + 1, std::memory_order_release);
104 |         }
105 |         return true;
106 |     }
107 | 
108 |     /// Pop one object from the fifo.
109 |     /// @return `true` if the pop operation is successful; `false` if fifo is empty.
110 |     auto pop(T& value) {
111 |         auto pushCursor = pushCursor_.load(std::memory_order_acquire);
112 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
113 |         if (empty(pushCursor, popCursor)) {
114 |             return false;
115 |         }
116 |         value = ring_[popCursor];
117 |         ring_[popCursor].~T();
118 |         if (popCursor == capacity_) {
119 |             popCursor_.store(0, std::memory_order_release);
120 |         } else {
121 |             popCursor_.store(popCursor + 1, std::memory_order_release);
122 |         }
123 |         return true;
124 |     }
125 | 
126 | private:
127 |     auto full(size_type pushCursor, size_type popCursor) const noexcept {
128 |         if (pushCursor < popCursor) {
129 |             return pushCursor == popCursor - 1;
130 |         } else if (popCursor < pushCursor) {
131 |             return  popCursor == pushCursor - capacity_;
132 |         } else {
133 |             return false;
134 |         }
135 |     }
136 |     static auto empty(size_type pushCursor, size_type popCursor) noexcept {
137 |         return pushCursor == popCursor;
138 |     }
139 | 
140 | private:
141 |     size_type capacity_;
142 |     T* ring_;
143 | 
144 |     using CursorType = std::atomic<size_type>;
145 |     static_assert(CursorType::is_always_lock_free);
146 | 
147 |     // See Fifo3 for reason std::hardware_destructive_interference_size is not used directly
148 |     static constexpr auto hardware_destructive_interference_size = size_type{64};
149 | 
150 |     /// Loaded and stored by the push thread; loaded by the pop thread
151 |     alignas(hardware_destructive_interference_size) CursorType pushCursor_;
152 | 
153 |     /// Loaded and stored by the pop thread; loaded by the push thread
154 |     alignas(hardware_destructive_interference_size) CursorType popCursor_;
155 | 
156 |     // Padding to avoid false sharing with adjacent objects
157 |     char padding_[hardware_destructive_interference_size - sizeof(CursorType)];
158 | };
159 | 


--------------------------------------------------------------------------------
/Fifo4.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <atomic>
  4 | #include <cassert>
  5 | #include <memory>
  6 | #include <new>
  7 | 
  8 | #include <sanitizer/tsan_interface.h>
  9 | 
 10 | 
 11 | /// Threadsafe, efficient circular FIFO with cached cursors
 12 | template<typename T, typename Alloc = std::allocator<T>>
 13 | class Fifo4 : private Alloc
 14 | {
 15 | public:
 16 |     using value_type = T;
 17 |     using allocator_traits = std::allocator_traits<Alloc>;
 18 |     using size_type = typename allocator_traits::size_type;
 19 | 
 20 |     explicit Fifo4(size_type capacity, Alloc const& alloc = Alloc{})
 21 |         : Alloc{alloc}
 22 |         , capacity_{capacity}
 23 |         , ring_{allocator_traits::allocate(*this, capacity)}
 24 |     {}
 25 | 
 26 |     ~Fifo4() {
 27 |         while(not empty()) {
 28 |             element(popCursor_)->~T();
 29 |             ++popCursor_;
 30 |         }
 31 |         allocator_traits::deallocate(*this, ring_, capacity_);
 32 |     }
 33 | 
 34 |     /// Returns the number of elements in the fifo
 35 |     auto size() const noexcept {
 36 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 37 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
 38 | 
 39 |         assert(popCursor <= pushCursor);
 40 |         return pushCursor - popCursor;
 41 |     }
 42 | 
 43 |     /// Returns whether the container has no elements
 44 |     auto empty() const noexcept { return size() == 0; }
 45 | 
 46 |     /// Returns whether the container has capacity_() elements
 47 |     auto full() const noexcept { return size() == capacity(); }
 48 | 
 49 |     /// Returns the number of elements that can be held in the fifo
 50 |     auto capacity() const noexcept { return capacity_; }
 51 | 
 52 | 
 53 |     /// Push one object onto the fifo.
 54 |     /// @return `true` if the operation is successful; `false` if fifo is full.
 55 |     auto push(T const& value) {
 56 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 57 |         if (full(pushCursor, popCursorCached_)) {
 58 |             popCursorCached_ = popCursor_.load(std::memory_order_acquire);
 59 |             // popCursorCached_ = popCursor_.load(std::memory_order_relaxed);
 60 |             if (full(pushCursor, popCursorCached_)) {
 61 |                 return false;
 62 |             }
 63 |         }
 64 | 
 65 |         //__tsan_acquire(&popCursor_);
 66 |         // std::atomic_thread_fence(std::memory_order_acquire);
 67 |         new (element(pushCursor)) T(value);
 68 |         pushCursor_.store(pushCursor + 1, std::memory_order_release);
 69 |         return true;
 70 |     }
 71 | 
 72 |     /// Pop one object from the fifo.
 73 |     /// @return `true` if the pop operation is successful; `false` if fifo is empty.
 74 |     auto pop(T& value) {
 75 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
 76 |         if (empty(pushCursorCached_, popCursor)) {
 77 |             pushCursorCached_ = pushCursor_.load(std::memory_order_acquire);
 78 |             // pushCursorCached_ = pushCursor_.load(std::memory_order_relaxed);
 79 |             if (empty(pushCursorCached_, popCursor)) {
 80 |                 return false;
 81 |             }
 82 |         }
 83 | 
 84 |         // __tsan_acquire(&pushCursor_);
 85 |         // std::atomic_thread_fence(std::memory_order_acquire);
 86 |         value = *element(popCursor);
 87 |         element(popCursor)->~T();
 88 |         popCursor_.store(popCursor + 1, std::memory_order_release);
 89 |         return true;
 90 |     }
 91 | 
 92 | private:
 93 |     auto full(size_type pushCursor, size_type popCursor) const noexcept {
 94 |         return (pushCursor - popCursor) == capacity_;
 95 |     }
 96 |     static auto empty(size_type pushCursor, size_type popCursor) noexcept {
 97 |         return pushCursor == popCursor;
 98 |     }
 99 |     auto element(size_type cursor) noexcept {
100 |         return &ring_[cursor % capacity_];
101 |     }
102 | 
103 | private:
104 |     size_type capacity_;
105 |     T* ring_;
106 | 
107 |     using CursorType = std::atomic<size_type>;
108 |     static_assert(CursorType::is_always_lock_free);
109 | 
110 |     // See Fifo3 for reason std::hardware_destructive_interference_size is not used directly
111 |     static constexpr auto hardware_destructive_interference_size = size_type{64};
112 | 
113 |     /// Loaded and stored by the push thread; loaded by the pop thread
114 |     alignas(hardware_destructive_interference_size) CursorType pushCursor_;
115 | 
116 |     /// Exclusive to the push thread
117 |     alignas(hardware_destructive_interference_size) size_type popCursorCached_{};
118 | 
119 |     /// Loaded and stored by the pop thread; loaded by the push thread
120 |     alignas(hardware_destructive_interference_size) CursorType popCursor_;
121 | 
122 |     /// Exclusive to the pop thread
123 |     alignas(hardware_destructive_interference_size) size_type pushCursorCached_{};
124 | 
125 |     // Padding to avoid false sharing with adjacent objects
126 |     char padding_[hardware_destructive_interference_size - sizeof(size_type)];
127 | };
128 | 


--------------------------------------------------------------------------------
/Fifo4a.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <atomic>
  4 | #include <cassert>
  5 | #include <memory>
  6 | #include <new>
  7 | 
  8 | 
  9 | /// Threadsafe, efficient circular FIFO with cached cursors; bitwise AND vs remainder
 10 | template<typename T, typename Alloc = std::allocator<T>>
 11 | class Fifo4a : private Alloc
 12 | {
 13 | public:
 14 |     using value_type = T;
 15 |     using allocator_traits = std::allocator_traits<Alloc>;
 16 |     using size_type = typename allocator_traits::size_type;
 17 | 
 18 |     explicit Fifo4a(size_type capacity, Alloc const& alloc = Alloc{})
 19 |         : Alloc{alloc}
 20 |         , mask_{capacity - 1}
 21 |         , ring_{allocator_traits::allocate(*this, capacity)}
 22 |     {}
 23 | 
 24 |     ~Fifo4a() {
 25 |         while(not empty()) {
 26 |             element(popCursor_)->~T();
 27 |             ++popCursor_;
 28 |         }
 29 |         allocator_traits::deallocate(*this, ring_, capacity());
 30 |     }
 31 | 
 32 |     /// Returns the number of elements in the fifo
 33 |     auto size() const noexcept {
 34 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 35 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
 36 | 
 37 |         assert(popCursor <= pushCursor);
 38 |         return pushCursor - popCursor;
 39 |     }
 40 | 
 41 |     /// Returns whether the container has no elements
 42 |     auto empty() const noexcept { return size() == 0; }
 43 | 
 44 |     /// Returns whether the container has capacity() elements
 45 |     auto full() const noexcept { return size() == capacity(); }
 46 | 
 47 |     /// Returns the number of elements that can be held in the fifo
 48 |     auto capacity() const noexcept { return mask_ + 1; }
 49 | 
 50 | 
 51 |     /// Push one object onto the fifo.
 52 |     /// @return `true` if the operation is successful; `false` if fifo is full.
 53 |     auto push(T const& value) {
 54 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 55 |         if (full(pushCursor, popCursorCached_)) {
 56 |             popCursorCached_ = popCursor_.load(std::memory_order_acquire);
 57 |             if (full(pushCursor, popCursorCached_)) {
 58 |                 return false;
 59 |             }
 60 |         }
 61 | 
 62 |         new (element(pushCursor)) T(value);
 63 |         pushCursor_.store(pushCursor + 1, std::memory_order_release);
 64 |         return true;
 65 |     }
 66 | 
 67 |     /// Pop one object from the fifo.
 68 |     /// @return `true` if the pop operation is successful; `false` if fifo is empty.
 69 |     auto pop(T& value) {
 70 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
 71 |         if (empty(pushCursorCached_, popCursor)) {
 72 |             pushCursorCached_ = pushCursor_.load(std::memory_order_acquire);
 73 |             if (empty(pushCursorCached_, popCursor)) {
 74 |                 return false;
 75 |             }
 76 |         }
 77 | 
 78 |         value = *element(popCursor);
 79 |         element(popCursor)->~T();
 80 |         popCursor_.store(popCursor + 1, std::memory_order_release);
 81 |         return true;
 82 |     }
 83 | 
 84 | private:
 85 |     auto full(size_type pushCursor, size_type popCursor) const noexcept {
 86 |         return (pushCursor - popCursor) == capacity();
 87 |     }
 88 |     static auto empty(size_type pushCursor, size_type popCursor) noexcept {
 89 |         return pushCursor == popCursor;
 90 |     }
 91 |     auto element(size_type cursor) noexcept {
 92 |         return &ring_[cursor & mask_];
 93 |     }
 94 | 
 95 | private:
 96 |     size_type mask_;
 97 |     T* ring_;
 98 | 
 99 |     using CursorType = std::atomic<size_type>;
100 |     static_assert(CursorType::is_always_lock_free);
101 | 
102 |     // See Fifo3 for reason std::hardware_destructive_interference_size is not used directly
103 |     static constexpr auto hardware_destructive_interference_size = size_type{64};
104 | 
105 |     /// Loaded and stored by the push thread; loaded by the pop thread
106 |     alignas(hardware_destructive_interference_size) CursorType pushCursor_;
107 | 
108 |     /// Exclusive to the push thread
109 |     alignas(hardware_destructive_interference_size) size_type popCursorCached_{};
110 | 
111 |     /// Loaded and stored by the pop thread; loaded by the push thread
112 |     alignas(hardware_destructive_interference_size) CursorType popCursor_;
113 | 
114 |     /// Exclusive to the pop thread
115 |     alignas(hardware_destructive_interference_size) size_type pushCursorCached_{};
116 | 
117 |     // Padding to avoid false sharing with adjacent objects
118 |     char padding_[hardware_destructive_interference_size - sizeof(size_type)];
119 | };
120 | 


--------------------------------------------------------------------------------
/Fifo4b.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <atomic>
  4 | #include <cassert>
  5 | #include <memory>
  6 | #include <new>
  7 | 
  8 | 
  9 | /**
 10 |  * Fifo3a with cached cursors; a threadsafe, efficient circular FIFO
 11 |  * with constrained cursors.
 12 |  *
 13 |  * This Fifo is useful when you need to constrain the cursor ranges. For
 14 |  * example say if the sizeof(CursorType) is 8 or 16. The cursors may
 15 |  * take on any value up to the fifo's capacity + 1. Furthermore, there
 16 |  * are no calculations where an intermediate cursor value is larger then
 17 |  * this number. And, finally, the cursors are never negative.
 18 |  *
 19 |  * The problem that must be resolved is how to distinguish an empty fifo
 20 |  * from a full one and still meet the above constraints. First, define
 21 |  * an empty fifo as when the pushCursor and popCursor are equal. We
 22 |  * cannot define a full fifo as pushCursor == popCursor + capacity as we
 23 |  * did with most of the others. Firstly, the intermediate value
 24 |  * popCursor + capacity can overflow if a signed cursor is used and if
 25 |  * the cursors are constrained to [0..capacity) there is no distiction
 26 |  * between the full definition and the empty definition.
 27 |  *
 28 |  * To resolve this we introduce the idea of a sentinal element by
 29 |  * allocating one more element than the capacity of the fifo and define
 30 |  * a full fifo as when the cursors are "one apart". That is,
 31 |  *
 32 |  * @code
 33 |  *    | pushCursor < popCursor:  pushCursor == popCursor - 1
 34 |  *    | popCursor < pushCursor:  popCursor == pushCursor - capacity
 35 |  *    |                   else:  false
 36 |  * @endcode
 37 |  */
 38 | template<typename T, typename Alloc = std::allocator<T>>
 39 | class Fifo4b : private Alloc
 40 | {
 41 | public:
 42 |     using value_type = T;
 43 |     using allocator_traits = std::allocator_traits<Alloc>;
 44 |     using size_type = typename allocator_traits::size_type;
 45 | 
 46 |     explicit Fifo4b(size_type capacity, Alloc const& alloc = Alloc{})
 47 |         : Alloc{alloc}
 48 |         , capacity_{capacity}
 49 |         , ring_{allocator_traits::allocate(*this, capacity + 1)}
 50 |     {}
 51 | 
 52 |     ~Fifo4b() {
 53 |         // TODO fix shouldn't matter for benchmark since it waits until
 54 |         // the fifo is empty and only need if destructors have side
 55 |         // effects.
 56 |         // while(not empty()) {
 57 |         //     ring_[popCursor_ & mask_].~T();
 58 |         //     ++popCursor_;
 59 |         // }
 60 |         allocator_traits::deallocate(*this, ring_, capacity_ + 1);
 61 |     }
 62 | 
 63 |     /// Returns the number of elements in the fifo
 64 |     auto size() const noexcept {
 65 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 66 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
 67 |         if (popCursor <= pushCursor) {
 68 |             return pushCursor - popCursor;
 69 |         } else {
 70 |             return capacity_ - (popCursor - (pushCursor + 1));
 71 |         }
 72 |     }
 73 | 
 74 |     /// Returns whether the container has no elements
 75 |     auto empty() const noexcept {
 76 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 77 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
 78 |         return empty(pushCursor, popCursor);
 79 |     }
 80 | 
 81 |     /// Returns whether the container has capacity() elements
 82 |     auto full() const noexcept {
 83 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 84 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
 85 |         return full(pushCursor, popCursor);
 86 |     }
 87 | 
 88 |     /// Returns the number of elements that can be held in the fifo
 89 |     auto capacity() const noexcept { return capacity_; }
 90 | 
 91 | 
 92 |     /// Push one object onto the fifo.
 93 |     /// @return `true` if the operation is successful; `false` if fifo is full.
 94 |     auto push(T const& value) {
 95 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 96 |         if (full(pushCursor, popCursorCached_)) {
 97 |             popCursorCached_ = popCursor_.load(std::memory_order_acquire);
 98 |             if (full(pushCursor, popCursorCached_)) {
 99 |                 return false;
100 |             }
101 |         }
102 | 
103 |         new (&ring_[pushCursor]) T(value);
104 |         if (pushCursor == capacity_) {
105 |             pushCursor_.store(0, std::memory_order_release);
106 |         } else {
107 |             pushCursor_.store(pushCursor + 1, std::memory_order_release);
108 |         }
109 |         return true;
110 |     }
111 | 
112 |     /// Pop one object from the fifo.
113 |     /// @return `true` if the pop operation is successful; `false` if fifo is empty.
114 |     auto pop(T& value) {
115 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
116 |         if (empty(pushCursorCached_, popCursor)) {
117 |             pushCursorCached_ = pushCursor_.load(std::memory_order_acquire);
118 |             if (empty(pushCursorCached_, popCursor)) {
119 |                 return false;
120 |             }
121 |         }
122 | 
123 |         value = ring_[popCursor];
124 |         ring_[popCursor].~T();
125 |         if (popCursor == capacity_) {
126 |             popCursor_.store(0, std::memory_order_release);
127 |         } else {
128 |             popCursor_.store(popCursor + 1, std::memory_order_release);
129 |         }
130 |         return true;
131 |     }
132 | 
133 | private:
134 |     auto full(size_type pushCursor, size_type popCursor) const noexcept {
135 |         if (pushCursor < popCursor) {
136 |             return pushCursor == popCursor - 1;
137 |         } else if (popCursor < pushCursor) {
138 |             return  popCursor == pushCursor - capacity_;
139 |         } else {
140 |             return false;
141 |         }
142 |     }
143 |     static auto empty(size_type pushCursor, size_type popCursor) noexcept {
144 |         return pushCursor == popCursor;
145 |     }
146 | 
147 | private:
148 |     size_type capacity_;
149 |     T* ring_;
150 | 
151 |     using CursorType = std::atomic<size_type>;
152 |     static_assert(CursorType::is_always_lock_free);
153 | 
154 |     // See Fifo3 for reason std::hardware_destructive_interference_size is not used directly
155 |     static constexpr auto hardware_destructive_interference_size = size_type{64};
156 | 
157 |     /// Loaded and stored by the push thread; loaded by the pop thread
158 |     alignas(hardware_destructive_interference_size) CursorType pushCursor_;
159 | 
160 |     /// Exclusive to the push thread
161 |     alignas(hardware_destructive_interference_size) size_type popCursorCached_{};
162 | 
163 |     /// Loaded and stored by the pop thread; loaded by the push thread
164 |     alignas(hardware_destructive_interference_size) CursorType popCursor_;
165 | 
166 |     /// Exclusive to the pop thread
167 |     alignas(hardware_destructive_interference_size) size_type pushCursorCached_{};
168 | 
169 |     // Padding to avoid false sharing with adjacent objects
170 |     char padding_[hardware_destructive_interference_size - sizeof(size_type)];
171 | };
172 | 


--------------------------------------------------------------------------------
/Fifo5.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <atomic>
  4 | #include <cassert>
  5 | #include <cstring>
  6 | #include <memory>
  7 | #include <new>
  8 | #include <type_traits>
  9 | 
 10 | /// A trait used to optimize the number of bytes copied. Specialize this
 11 | /// on the type used to parameterize the Fifo5 to implement the
 12 | /// optimization. The general template returns `sizeof(T)`.
 13 | template<typename T>
 14 | struct ValueSizeTraits
 15 | {
 16 |     using value_type = T;
 17 |     static std::size_t size(value_type const& value) { return sizeof(value_type); }
 18 | };
 19 | 
 20 | 
 21 | /// Require trivial, add ValueSizeTraits, pusher and popper to Fifo4
 22 | template<typename T, typename Alloc = std::allocator<T>>
 23 |     requires std::is_trivial_v<T>
 24 | class Fifo5 : private Alloc
 25 | {
 26 | public:
 27 |     using value_type = T;
 28 |     using allocator_traits = std::allocator_traits<Alloc>;
 29 |     using size_type = typename allocator_traits::size_type;
 30 | 
 31 |     explicit Fifo5(size_type capacity, Alloc const& alloc = Alloc{})
 32 |         : Alloc{alloc}
 33 |         , capacity_{capacity}
 34 |         , ring_{allocator_traits::allocate(*this, capacity)}
 35 |     {}
 36 | 
 37 |     ~Fifo5() {
 38 |         allocator_traits::deallocate(*this, ring_, capacity());
 39 |     }
 40 | 
 41 | 
 42 |     /// Returns the number of elements in the fifo
 43 |     auto size() const noexcept {
 44 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 45 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
 46 | 
 47 |         assert(popCursor <= pushCursor);
 48 |         return pushCursor - popCursor;
 49 |     }
 50 | 
 51 |     /// Returns whether the container has no elements
 52 |     auto empty() const noexcept { return size() == 0; }
 53 | 
 54 |     /// Returns whether the container has capacity_() elements
 55 |     auto full() const noexcept { return size() == capacity(); }
 56 | 
 57 |     /// Returns the number of elements that can be held in the fifo
 58 |     auto capacity() const noexcept { return capacity_; }
 59 | 
 60 | 
 61 |     /// An RAII proxy object returned by push(). Allows the caller to
 62 |     /// manipulate value_type's members directly in the fifo's ring. The
 63 |     /// actual push happens when the pusher goes out of scope.
 64 |     class pusher_t
 65 |     {
 66 |     public:
 67 |         pusher_t() = default;
 68 |         explicit pusher_t(Fifo5* fifo, size_type cursor) noexcept : fifo_{fifo}, cursor_{cursor} {}
 69 | 
 70 |         pusher_t(pusher_t const&) = delete;
 71 |         pusher_t& operator=(pusher_t const&) = delete;
 72 | 
 73 |         pusher_t(pusher_t&& other) noexcept
 74 |             : fifo_{std::move(other.fifo_)}
 75 |             , cursor_{std::move(other.cursor_)} {
 76 |             other.release();
 77 |         }
 78 |         pusher_t& operator=(pusher_t&& other) noexcept {
 79 |             fifo_ = std::move(other.fifo_);
 80 |             cursor_ = std::move(other.cursor_);
 81 |             other.release();
 82 |             return *this;
 83 |         }
 84 | 
 85 |         ~pusher_t() {
 86 |             if (fifo_) {
 87 |                 fifo_->pushCursor_.store(cursor_ + 1, std::memory_order_release);
 88 |             }
 89 |         }
 90 | 
 91 |         /// If called the actual push operation will not be called when the
 92 |         /// pusher_t goes out of scope. Operations on the pusher_t instance
 93 |         /// after release has been called are undefined.
 94 |         void release() noexcept { fifo_ = {}; }
 95 | 
 96 |         /// Return whether or not the pusher_t is active.
 97 |         explicit operator bool() const noexcept { return fifo_; }
 98 | 
 99 |         /// @name Direct access to the fifo's ring
100 |         ///@{
101 |         value_type* get() noexcept { return fifo_->element(cursor_); }
102 |         value_type const* get() const noexcept { return fifo_->element(cursor_); }
103 | 
104 |         value_type& operator*() noexcept { return *get(); }
105 |         value_type const& operator*() const noexcept { return *get(); }
106 | 
107 |         value_type* operator->() noexcept { return get(); }
108 |         value_type const* operator->() const noexcept { return get(); }
109 |         ///@}
110 | 
111 |         /// Copy-assign a `value_type` to the pusher. Prefer to use this
112 |         /// form rather than assigning directly to a value_type&. It takes
113 |         /// advantage of ValueSizeTraits.
114 |         pusher_t& operator=(value_type const& value) noexcept {
115 |             std::memcpy(get(), std::addressof(value), ValueSizeTraits<value_type>::size(value));
116 |             return *this;
117 |         }
118 | 
119 |     private:
120 |         Fifo5* fifo_{};
121 |         size_type cursor_;
122 |     };
123 |     friend class pusher_t;
124 | 
125 |     /// Optionally push one object onto a file via a pusher.
126 |     /// @return a pointer to pusher_t.
127 |     pusher_t push() noexcept {
128 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
129 |         if (full(pushCursor, popCursorCached_)) {
130 |             popCursorCached_ = popCursor_.load(std::memory_order_acquire);
131 |             if (full(pushCursor, popCursorCached_)) {
132 |                 return pusher_t{};
133 |             }
134 |         }
135 |         return pusher_t(this, pushCursor);
136 |     }
137 | 
138 |     /// Push one object onto the fifo.
139 |     /// @return `true` if the operation is successful; `false` if fifo is full.
140 |     auto push(T const& value) noexcept {
141 |         if (auto pusher = push(); pusher) {
142 |             pusher = value;
143 |             return true;
144 |         }
145 |         return false;
146 |     }
147 | 
148 |     /// An RAII proxy object returned by pop(). Allows the caller to
149 |     /// manipulate value_type members directly in the fifo's ring. The
150 |     // /actual pop happens when the popper goes out of scope.
151 |     class popper_t
152 |     {
153 |     public:
154 |         popper_t() = default;
155 |         explicit popper_t(Fifo5* fifo, size_type cursor) noexcept : fifo_{fifo}, cursor_{cursor} {}
156 | 
157 |         popper_t(popper_t const&) = delete;
158 |         popper_t& operator=(popper_t const&) = delete;
159 | 
160 |         popper_t(popper_t&& other) noexcept
161 |             : fifo_{std::move(other.fifo_)}
162 |             , cursor_{std::move(other.cursor_)} {
163 |             other.release();
164 |         }
165 |         popper_t& operator=(popper_t&& other) noexcept {
166 |             fifo_ = std::move(other.fifo_);
167 |             cursor_ = std::move(other.cursor_);
168 |             other.release();
169 |             return *this;
170 |         }
171 | 
172 |         ~popper_t() {
173 |             if (fifo_) {
174 |                 fifo_->popCursor_.store(cursor_ + 1, std::memory_order_release);
175 |             }
176 |         }
177 | 
178 |         /// If called the actual pop operation will not be called when the
179 |         /// popper_t goes out of scope. Operations on the popper_t instance
180 |         /// after release has been called are undefined.
181 |         void release() noexcept { fifo_ = {}; }
182 | 
183 |         /// Return whether or not the popper_t is active.
184 |         explicit operator bool() const noexcept { return fifo_; }
185 | 
186 |         /// @name Direct access to the fifo's ring
187 |         ///@{
188 |         value_type* get() noexcept { return fifo_->element(cursor_); }
189 |         value_type const* get() const noexcept { return fifo_->element(cursor_); }
190 | 
191 |         value_type& operator*() noexcept { return *get(); }
192 |         value_type const& operator*() const noexcept { return *get(); }
193 | 
194 |         value_type* operator->() noexcept { return get(); }
195 |         value_type const* operator->() const noexcept { return get(); }
196 |         ///@}
197 | 
198 |     private:
199 |         Fifo5* fifo_{};
200 |         size_type cursor_;
201 |     };
202 |     friend popper_t;
203 | 
204 |     auto pop() noexcept {
205 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
206 |         if (empty(pushCursorCached_, popCursor)) {
207 |             pushCursorCached_ = pushCursor_.load(std::memory_order_acquire);
208 |             if (empty(pushCursorCached_, popCursor)) {
209 |                 return popper_t{};
210 |             }
211 |         }
212 |         return popper_t(this, popCursor);
213 |     };
214 | 
215 |     /// Pop one object from the fifo.
216 |     /// @return `true` if the pop operation is successful; `false` if fifo is empty.
217 |     auto pop(T& value) noexcept {
218 |         if (auto popper = pop(); popper) {
219 |             value = *popper;
220 |             return true;
221 |         }
222 |         return false;
223 |     }
224 | 
225 | private:
226 |     auto full(size_type pushCursor, size_type popCursor) const noexcept {
227 |         assert(popCursor <= pushCursor);
228 |         return (pushCursor - popCursor) == capacity();
229 |     }
230 |     static auto empty(size_type pushCursor, size_type popCursor) noexcept {
231 |         return pushCursor == popCursor;
232 |     }
233 | 
234 |     auto* element(size_type cursor) noexcept { return &ring_[cursor % capacity_]; }
235 |     auto const* element(size_type cursor) const noexcept { return &ring_[cursor % capacity_]; }
236 | 
237 | private:
238 |     size_type capacity_;
239 |     T* ring_;
240 | 
241 |     using CursorType = std::atomic<size_type>;
242 |     static_assert(CursorType::is_always_lock_free);
243 | 
244 |     // https://stackoverflow.com/questions/39680206/understanding-stdhardware-destructive-interference-size-and-stdhardware-cons
245 |     // See Fifo3.hpp for reason why std::hardware_destructive_interference_size is not used directly
246 |     static constexpr auto hardware_destructive_interference_size = size_type{64};
247 | 
248 |     /// Loaded and stored by the push thread; loaded by the pop thread
249 |     alignas(hardware_destructive_interference_size) CursorType pushCursor_;
250 | 
251 |     /// Exclusive to the push thread
252 |     alignas(hardware_destructive_interference_size) size_type popCursorCached_{};
253 | 
254 |     /// Loaded and stored by the pop thread; loaded by the push thread
255 |     alignas(hardware_destructive_interference_size) CursorType popCursor_;
256 | 
257 |     /// Exclusive to the pop thread
258 |     alignas(hardware_destructive_interference_size) size_type pushCursorCached_{};
259 | 
260 |     // Padding to avoid false sharing with adjacent objects
261 |     char padding_[hardware_destructive_interference_size - sizeof(size_type)];
262 | };
263 | 


--------------------------------------------------------------------------------
/Fifo5a.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <atomic>
  4 | #include <cassert>
  5 | #include <cstring>
  6 | #include <memory>
  7 | #include <new>
  8 | #include <type_traits>
  9 | 
 10 | // For ValueSizeTraits
 11 | #include "Fifo5.hpp"
 12 | 
 13 | 
 14 | /// Require trivial, add ValueSizeTraits, pusher and popper to Fifo4;
 15 | /// bitwise AND vs remainder
 16 | template<typename T, typename Alloc = std::allocator<T>>
 17 |     requires std::is_trivial_v<T>
 18 | class Fifo5a : private Alloc
 19 | {
 20 | public:
 21 |     using value_type = T;
 22 |     using allocator_traits = std::allocator_traits<Alloc>;
 23 |     using size_type = typename allocator_traits::size_type;
 24 | 
 25 |     explicit Fifo5a(size_type capacity, Alloc const& alloc = Alloc{})
 26 |         : Alloc{alloc}
 27 |         , mask_{capacity - 1}
 28 |         , ring_{allocator_traits::allocate(*this, capacity)} {
 29 |         assert((capacity & mask_) == 0);
 30 |     }
 31 | 
 32 |     ~Fifo5a() {
 33 |         allocator_traits::deallocate(*this, ring_, capacity());
 34 |     }
 35 | 
 36 | 
 37 |     /// Returns the number of elements in the fifo
 38 |     auto size() const noexcept {
 39 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
 40 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
 41 | 
 42 |         assert(popCursor <= pushCursor);
 43 |         return pushCursor - popCursor;
 44 |     }
 45 | 
 46 |     /// Returns whether the container has no elements
 47 |     auto empty() const noexcept { return size() == 0; }
 48 | 
 49 |     /// Returns whether the container has capacity_() elements
 50 |     auto full() const noexcept { return size() == capacity(); }
 51 | 
 52 |     /// Returns the number of elements that can be held in the fifo
 53 |     auto capacity() const noexcept { return mask_ + 1; }
 54 | 
 55 | 
 56 |     /// An RAII proxy object returned by push(). Allows the caller to
 57 |     /// manipulate value_type's members directly in the fifo's ring. The
 58 |     /// actual push happens when the pusher goes out of scope.
 59 |     class pusher_t
 60 |     {
 61 |     public:
 62 |         pusher_t() = default;
 63 |         explicit pusher_t(Fifo5a* fifo, size_type cursor) noexcept : fifo_{fifo}, cursor_{cursor} {}
 64 | 
 65 |         pusher_t(pusher_t const&) = delete;
 66 |         pusher_t& operator=(pusher_t const&) = delete;
 67 | 
 68 |         pusher_t(pusher_t&& other) noexcept
 69 |             : fifo_{std::move(other.fifo_)}
 70 |             , cursor_{std::move(other.cursor_)} {
 71 |             other.release();
 72 |         }
 73 |         pusher_t& operator=(pusher_t&& other) noexcept {
 74 |             fifo_ = std::move(other.fifo_);
 75 |             cursor_ = std::move(other.cursor_);
 76 |             other.release();
 77 |             return *this;
 78 |         }
 79 | 
 80 |         ~pusher_t() {
 81 |             if (fifo_) {
 82 |                 fifo_->pushCursor_.store(cursor_ + 1, std::memory_order_release);
 83 |             }
 84 |         }
 85 | 
 86 |         /// If called the actual push operation will not be called when the
 87 |         /// pusher_t goes out of scope. Operations on the pusher_t instance
 88 |         /// after release has been called are undefined.
 89 |         void release() noexcept { fifo_ = {}; }
 90 | 
 91 |         /// Return whether or not the pusher_t is active.
 92 |         explicit operator bool() const noexcept { return fifo_; }
 93 | 
 94 |         /// @name Direct access to the fifo's ring
 95 |         ///@{
 96 |         value_type* get() noexcept { return fifo_->element(cursor_); }
 97 |         value_type const* get() const noexcept { return fifo_->element(cursor_); }
 98 | 
 99 |         value_type& operator*() noexcept { return *get(); }
100 |         value_type const& operator*() const noexcept { return *get(); }
101 | 
102 |         value_type* operator->() noexcept { return get(); }
103 |         value_type const* operator->() const noexcept { return get(); }
104 |         ///@}
105 | 
106 |         /// Copy-assign a `value_type` to the pusher. Prefer to use this
107 |         /// form rather than assigning directly to a value_type&. It takes
108 |         /// advantage of ValueSizeTraits.
109 |         pusher_t& operator=(value_type const& value) noexcept {
110 |             std::memcpy(get(), std::addressof(value), ValueSizeTraits<value_type>::size(value));
111 |             return *this;
112 |         }
113 | 
114 |     private:
115 |         Fifo5a* fifo_{};
116 |         size_type cursor_;
117 |     };
118 |     friend class pusher_t;
119 | 
120 |     /// Optionally push one object onto a file via a pusher.
121 |     /// @return a pointer to pusher_t.
122 |     pusher_t push() noexcept {
123 |         auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
124 |         if (full(pushCursor, popCursorCached_)) {
125 |             popCursorCached_ = popCursor_.load(std::memory_order_acquire);
126 |             if (full(pushCursor, popCursorCached_)) {
127 |                 return pusher_t{};
128 |             }
129 |         }
130 |         return pusher_t(this, pushCursor);
131 |     }
132 | 
133 |     /// Push one object onto the fifo.
134 |     /// @return `true` if the operation is successful; `false` if fifo is full.
135 |     auto push(T const& value) noexcept {
136 |         if (auto pusher = push(); pusher) {
137 |             pusher = value;
138 |             return true;
139 |         }
140 |         return false;
141 |     }
142 | 
143 |     /// An RAII proxy object returned by pop(). Allows the caller to
144 |     /// manipulate value_type members directly in the fifo's ring. The
145 |     // /actual pop happens when the popper goes out of scope.
146 |     class popper_t
147 |     {
148 |     public:
149 |         popper_t() = default;
150 |         explicit popper_t(Fifo5a* fifo, size_type cursor) noexcept : fifo_{fifo}, cursor_{cursor} {}
151 | 
152 |         popper_t(popper_t const&) = delete;
153 |         popper_t& operator=(popper_t const&) = delete;
154 | 
155 |         popper_t(popper_t&& other) noexcept
156 |             : fifo_{std::move(other.fifo_)}
157 |             , cursor_{std::move(other.cursor_)} {
158 |             other.release();
159 |         }
160 |         popper_t& operator=(popper_t&& other) noexcept {
161 |             fifo_ = std::move(other.fifo_);
162 |             cursor_ = std::move(other.cursor_);
163 |             other.release();
164 |             return *this;
165 |         }
166 | 
167 |         ~popper_t() {
168 |             if (fifo_) {
169 |                 fifo_->popCursor_.store(cursor_ + 1, std::memory_order_release);
170 |             }
171 |         }
172 | 
173 |         /// If called the actual pop operation will not be called when the
174 |         /// popper_t goes out of scope. Operations on the popper_t instance
175 |         /// after release has been called are undefined.
176 |         void release() noexcept { fifo_ = {}; }
177 | 
178 |         /// Return whether or not the popper_t is active.
179 |         explicit operator bool() const noexcept { return fifo_; }
180 | 
181 |         /// @name Direct access to the fifo's ring
182 |         ///@{
183 |         value_type* get() noexcept { return fifo_->element(cursor_); }
184 |         value_type const* get() const noexcept { return fifo_->element(cursor_); }
185 | 
186 |         value_type& operator*() noexcept { return *get(); }
187 |         value_type const& operator*() const noexcept { return *get(); }
188 | 
189 |         value_type* operator->() noexcept { return get(); }
190 |         value_type const* operator->() const noexcept { return get(); }
191 |         ///@}
192 | 
193 |     private:
194 |         Fifo5a* fifo_{};
195 |         size_type cursor_;
196 |     };
197 |     friend popper_t;
198 | 
199 |     auto pop() noexcept {
200 |         auto popCursor = popCursor_.load(std::memory_order_relaxed);
201 |         if (empty(pushCursorCached_, popCursor)) {
202 |             pushCursorCached_ = pushCursor_.load(std::memory_order_acquire);
203 |             if (empty(pushCursorCached_, popCursor)) {
204 |                 return popper_t{};
205 |             }
206 |         }
207 |         return popper_t(this, popCursor);
208 |     };
209 | 
210 |     /// Pop one object from the fifo.
211 |     /// @return `true` if the pop operation is successful; `false` if fifo is empty.
212 |     auto pop(T& value) noexcept {
213 |         if (auto popper = pop(); popper) {
214 |             value = *popper;
215 |             return true;
216 |         }
217 |         return false;
218 |     }
219 | 
220 | private:
221 |     auto full(size_type pushCursor, size_type popCursor) const noexcept {
222 |         assert(popCursor <= pushCursor);
223 |         return (pushCursor - popCursor) == capacity();
224 |     }
225 |     static auto empty(size_type pushCursor, size_type popCursor) noexcept {
226 |         return pushCursor == popCursor;
227 |     }
228 | 
229 |     auto* element(size_type cursor) noexcept { return &ring_[cursor & mask_]; }
230 |     auto const* element(size_type cursor) const noexcept { return &ring_[cursor & mask_]; }
231 | 
232 | private:
233 |     size_type mask_;
234 |     T* ring_;
235 | 
236 |     using CursorType = std::atomic<size_type>;
237 |     static_assert(CursorType::is_always_lock_free);
238 | 
239 |     // https://stackoverflow.com/questions/39680206/understanding-stdhardware-destructive-interference-size-and-stdhardware-cons
240 |     // See Fifo3.hpp for reason why std::hardware_destructive_interference_size is not used directly
241 |     static constexpr auto hardware_destructive_interference_size = size_type{64};
242 | 
243 |     /// Loaded and stored by the push thread; loaded by the pop thread
244 |     alignas(hardware_destructive_interference_size) CursorType pushCursor_;
245 | 
246 |     /// Exclusive to the push thread
247 |     alignas(hardware_destructive_interference_size) size_type popCursorCached_{};
248 | 
249 |     /// Loaded and stored by the pop thread; loaded by the push thread
250 |     alignas(hardware_destructive_interference_size) CursorType popCursor_;
251 | 
252 |     /// Exclusive to the pop thread
253 |     alignas(hardware_destructive_interference_size) size_type pushCursorCached_{};
254 | 
255 |     // Padding to avoid false sharing with adjacent objects
256 |     char padding_[hardware_destructive_interference_size - sizeof(size_type)];
257 | };
258 | 


--------------------------------------------------------------------------------
/Fifo5b.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <atomic>
  4 | #include <cassert>
  5 | #include <cstring>
  6 | #include <memory>
  7 | #include <new>
  8 | #include <type_traits>
  9 | 
 10 | // For ValueSizeTraits
 11 | #include "Fifo5.hpp"
 12 | 
 13 | 
 14 | /// Like Fifo5a except uses atomic_ref
 15 | template<typename T, typename Alloc = std::allocator<T>>
 16 |     requires std::is_trivial_v<T>
 17 | class Fifo5b : private Alloc
 18 | {
 19 | public:
 20 |     using value_type = T;
 21 |     using allocator_traits = std::allocator_traits<Alloc>;
 22 |     using size_type = typename allocator_traits::size_type;
 23 | 
 24 |     explicit Fifo5b(size_type capacity, Alloc const& alloc = Alloc{})
 25 |         : Alloc{alloc}
 26 |         , mask_{capacity - 1}
 27 |         , ring_{allocator_traits::allocate(*this, capacity)} {
 28 |         assert((capacity & mask_) == 0);
 29 |     }
 30 | 
 31 |     // For consistency with the other fifos
 32 |     Fifo5b(Fifo5b const&) = delete;
 33 |     Fifo5b(Fifo5b&&) = delete;
 34 | 
 35 |     ~Fifo5b() {
 36 |         allocator_traits::deallocate(*this, ring_, capacity());
 37 |     }
 38 | 
 39 | 
 40 |     /// Returns the number of elements in the fifo
 41 |     auto size() const noexcept {
 42 |         auto pushCursor = pushCursorRef_.load(std::memory_order_relaxed);
 43 |         auto popCursor = popCursorRef_.load(std::memory_order_relaxed);
 44 | 
 45 |         assert(popCursor <= pushCursor);
 46 |         return pushCursor - popCursor;
 47 |     }
 48 | 
 49 |     /// Returns whether the container has no elements
 50 |     auto empty() const noexcept { return size() == 0; }
 51 | 
 52 |     /// Returns whether the container has capacity_() elements
 53 |     auto full() const noexcept { return size() == capacity(); }
 54 | 
 55 |     /// Returns the number of elements that can be held in the fifo
 56 |     auto capacity() const noexcept { return mask_ + 1; }
 57 | 
 58 | 
 59 |     /// An RAII proxy object returned by push(). Allows the caller to
 60 |     /// manipulate value_type's members directly in the fifo's ring. The
 61 |     /// actual push happens when the pusher goes out of scope.
 62 |     class pusher_t
 63 |     {
 64 |     public:
 65 |         pusher_t() = default;
 66 |         explicit pusher_t(Fifo5b* fifo, size_type cursor) noexcept : fifo_{fifo}, cursor_{cursor} {}
 67 | 
 68 |         pusher_t(pusher_t const&) = delete;
 69 |         pusher_t& operator=(pusher_t const&) = delete;
 70 | 
 71 |         pusher_t(pusher_t&& other) noexcept
 72 |             : fifo_{std::move(other.fifo_)}
 73 |             , cursor_{std::move(other.cursor_)} {
 74 |             other.release();
 75 |         }
 76 |         pusher_t& operator=(pusher_t&& other) noexcept {
 77 |             fifo_ = std::move(other.fifo_);
 78 |             cursor_ = std::move(other.cursor_);
 79 |             other.release();
 80 |             return *this;
 81 |         }
 82 | 
 83 |         ~pusher_t() {
 84 |             if (fifo_) {
 85 |                 fifo_->pushCursorRef_.store(cursor_ + 1, std::memory_order_release);
 86 |             }
 87 |         }
 88 | 
 89 |         /// If called the actual push operation will not be called when the
 90 |         /// pusher_t goes out of scope. Operations on the pusher_t instance
 91 |         /// after release has been called are undefined.
 92 |         void release() noexcept { fifo_ = {}; }
 93 | 
 94 |         /// Return whether or not the pusher_t is active.
 95 |         explicit operator bool() const noexcept { return fifo_; }
 96 | 
 97 |         /// @name Direct access to the fifo's ring
 98 |         ///@{
 99 |         value_type* get() noexcept { return fifo_->element(cursor_); }
100 |         value_type const* get() const noexcept { return fifo_->element(cursor_); }
101 | 
102 |         value_type& operator*() noexcept { return *get(); }
103 |         value_type const& operator*() const noexcept { return *get(); }
104 | 
105 |         value_type* operator->() noexcept { return get(); }
106 |         value_type const* operator->() const noexcept { return get(); }
107 |         ///@}
108 | 
109 |         /// Copy-assign a `value_type` to the pusher. Prefer to use this
110 |         /// form rather than assigning directly to a value_type&. It takes
111 |         /// advantage of ValueSizeTraits.
112 |         pusher_t& operator=(value_type const& value) noexcept {
113 |             std::memcpy(get(), std::addressof(value), ValueSizeTraits<value_type>::size(value));
114 |             return *this;
115 |         }
116 | 
117 |     private:
118 |         Fifo5b* fifo_{};
119 |         size_type cursor_;
120 |     };
121 |     friend class pusher_t;
122 | 
123 |     /// Optionally push one object onto a file via a pusher.
124 |     /// @return a pointer to pusher_t.
125 |     pusher_t push() noexcept {
126 |         auto pushCursor = pushCursor_;
127 |         if (full(pushCursor, popCursorCached_)) {
128 |             // popCursorCached_ = popCursor_.load(std::memory_order_acquire);
129 |             popCursorCached_ = popCursorRef_.load(std::memory_order_acquire);
130 |             if (full(pushCursor, popCursorCached_)) {
131 |                 return pusher_t{};
132 |             }
133 |         }
134 |         return pusher_t(this, pushCursor);
135 |     }
136 | 
137 |     /// Push one object onto the fifo.
138 |     /// @return `true` if the operation is successful; `false` if fifo is full.
139 |     auto push(T const& value) noexcept {
140 |         if (auto pusher = push(); pusher) {
141 |             pusher = value;
142 |             return true;
143 |         }
144 |         return false;
145 |     }
146 | 
147 |     /// An RAII proxy object returned by pop(). Allows the caller to
148 |     /// manipulate value_type members directly in the fifo's ring. The
149 |     // /actual pop happens when the popper goes out of scope.
150 |     class popper_t
151 |     {
152 |     public:
153 |         popper_t() = default;
154 |         explicit popper_t(Fifo5b* fifo, size_type cursor) noexcept : fifo_{fifo}, cursor_{cursor} {}
155 | 
156 |         popper_t(popper_t const&) = delete;
157 |         popper_t& operator=(popper_t const&) = delete;
158 | 
159 |         popper_t(popper_t&& other) noexcept
160 |             : fifo_{std::move(other.fifo_)}
161 |             , cursor_{std::move(other.cursor_)} {
162 |             other.release();
163 |         }
164 |         popper_t& operator=(popper_t&& other) noexcept {
165 |             fifo_ = std::move(other.fifo_);
166 |             cursor_ = std::move(other.cursor_);
167 |             other.release();
168 |             return *this;
169 |         }
170 | 
171 |         ~popper_t() {
172 |             if (fifo_) {
173 |                 fifo_->popCursorRef_.store(cursor_ + 1, std::memory_order_release);
174 |             }
175 |         }
176 | 
177 |         /// If called the actual pop operation will not be called when the
178 |         /// popper_t goes out of scope. Operations on the popper_t instance
179 |         /// after release has been called are undefined.
180 |         void release() noexcept { fifo_ = {}; }
181 | 
182 |         /// Return whether or not the popper_t is active.
183 |         explicit operator bool() const noexcept { return fifo_; }
184 | 
185 |         /// @name Direct access to the fifo's ring
186 |         ///@{
187 |         value_type* get() noexcept { return fifo_->element(cursor_); }
188 |         value_type const* get() const noexcept { return fifo_->element(cursor_); }
189 | 
190 |         value_type& operator*() noexcept { return *get(); }
191 |         value_type const& operator*() const noexcept { return *get(); }
192 | 
193 |         value_type* operator->() noexcept { return get(); }
194 |         value_type const* operator->() const noexcept { return get(); }
195 |         ///@}
196 | 
197 |     private:
198 |         Fifo5b* fifo_{};
199 |         size_type cursor_;
200 |     };
201 |     friend popper_t;
202 | 
203 |     auto pop() noexcept {
204 |         auto popCursor = popCursor_;
205 |         if (empty(pushCursorCached_, popCursor)) {
206 |             pushCursorCached_ = pushCursorRef_.load(std::memory_order_acquire);
207 |             if (empty(pushCursorCached_, popCursor)) {
208 |                 return popper_t{};
209 |             }
210 |         }
211 |         return popper_t(this, popCursor);
212 |     };
213 | 
214 |     /// Pop one object from the fifo.
215 |     /// @return `true` if the pop operation is successful; `false` if fifo is empty.
216 |     auto pop(T& value) noexcept {
217 |         if (auto popper = pop(); popper) {
218 |             value = *popper;
219 |             return true;
220 |         }
221 |         return false;
222 |     }
223 | 
224 | private:
225 |     auto full(size_type pushCursor, size_type popCursor) const noexcept {
226 |         assert(popCursor <= pushCursor);
227 |         return (pushCursor - popCursor) == capacity();
228 |     }
229 |     static auto empty(size_type pushCursor, size_type popCursor) noexcept {
230 |         return pushCursor == popCursor;
231 |     }
232 | 
233 |     auto* element(size_type cursor) noexcept { return &ring_[cursor & mask_]; }
234 |     auto const* element(size_type cursor) const noexcept { return &ring_[cursor & mask_]; }
235 | 
236 | private:
237 |     size_type mask_;
238 |     T* ring_;
239 | 
240 |     using CursorType = size_type;
241 |     using CursorRefType = std::atomic_ref<CursorType>;
242 | 
243 |     // https://stackoverflow.com/questions/39680206/understanding-stdhardware-destructive-interference-size-and-stdhardware-cons
244 |     // See Fifo3.hpp for reason why std::hardware_destructive_interference_size is not used directly
245 |     static constexpr auto hardware_destructive_interference_size = size_type{64};
246 | 
247 |     /// Loaded and stored by the push thread; loaded by the pop thread
248 |     alignas(hardware_destructive_interference_size) CursorType pushCursor_{};
249 |     CursorRefType pushCursorRef_{pushCursor_};
250 | 
251 |     /// Exclusive to the push thread
252 |     alignas(hardware_destructive_interference_size) size_type popCursorCached_{};
253 | 
254 |     /// Loaded and stored by the pop thread; loaded by the push thread
255 |     alignas(hardware_destructive_interference_size) CursorType popCursor_{};
256 |     CursorRefType popCursorRef_{popCursor_};
257 | 
258 |     /// Exclusive to the pop thread
259 |     alignas(hardware_destructive_interference_size) size_type pushCursorCached_{};
260 | 
261 |     // Padding to avoid false sharing with adjacent objects
262 |     char padding_[hardware_destructive_interference_size - sizeof(size_type)];
263 | };
264 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | This is free and unencumbered software released into the public domain.
 2 | 
 3 | Anyone is free to copy, modify, publish, use, compile, sell, or
 4 | distribute this software, either in source code form or as a compiled
 5 | binary, for any purpose, commercial or non-commercial, and by any
 6 | means.
 7 | 
 8 | In jurisdictions that recognize copyright laws, the author or authors
 9 | of this software dedicate any and all copyright interest in the
10 | software to the public domain. We make this dedication for the benefit
11 | of the public at large and to the detriment of our heirs and
12 | successors. We intend this dedication to be an overt act of
13 | relinquishment in perpetuity of all present and future rights to this
14 | software under copyright law.
15 | 
16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 | OTHER DEALINGS IN THE SOFTWARE.
23 | 
24 | For more information, please refer to <https://unlicense.org>
25 | 


--------------------------------------------------------------------------------
/Mutex.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <cstdlib>
 4 | #include <memory>
 5 | #include <mutex>
 6 | 
 7 | 
 8 | /// Thread-safe, mutex-based FIFO
 9 | template<typename T, typename Alloc = std::allocator<T>>
10 | class Mutex : private Alloc
11 | {
12 | public:
13 |     using value_type = T;
14 |     using allocator_traits = std::allocator_traits<Alloc>;
15 |     using size_type = typename allocator_traits::size_type;
16 | 
17 |     explicit Mutex(size_type capacity, Alloc const& alloc = Alloc{})
18 |         : Alloc{alloc}
19 |         , capacity_{capacity}
20 |         , ring_{allocator_traits::allocate(*this, capacity)}
21 |     {}
22 | 
23 |     ~Mutex() {
24 |         while(not empty()) {
25 |             ring_[popCursor_ % capacity_].~T();
26 |             ++popCursor_;
27 |         }
28 |         allocator_traits::deallocate(*this, ring_, capacity_);
29 |     }
30 | 
31 |     auto capacity() const noexcept { return capacity_; }
32 |     auto size() const noexcept {
33 |         std::lock_guard<std::mutex> lock(mutex_);
34 |         return size(pushCursor_, popCursor_);
35 |     }
36 |     auto empty() const noexcept { return size() == 0; }
37 |     auto full() const noexcept { return size() == capacity(); }
38 | 
39 |     auto push(T const& value) {
40 |         std::lock_guard<std::mutex> lock(mutex_);
41 | 
42 |         if (full(pushCursor_, popCursor_)) {
43 |             return false;
44 |         }
45 |         new (&ring_[pushCursor_ % capacity_]) T(value);
46 |         ++pushCursor_;
47 |         return true;
48 |     }
49 | 
50 |     auto pop(T& value) {
51 |         std::lock_guard<std::mutex> lock(mutex_);
52 | 
53 |         if (empty(pushCursor_, popCursor_)) {
54 |             return false;
55 |         }
56 |         value = ring_[popCursor_ % capacity_];
57 |         ring_[popCursor_ % capacity_].~T();
58 |         ++popCursor_;
59 |         return true;
60 |     }
61 | 
62 | private:
63 |     static auto size (size_type pushCursor, size_type popCursor) noexcept {
64 |         return pushCursor - popCursor;
65 |     }
66 | 
67 |     auto full(size_type pushCursor, size_type popCursor) const noexcept {
68 |         return (pushCursor - popCursor) == capacity_;
69 |     }
70 |     static auto empty(size_type pushCursor, size_type popCursor) noexcept {
71 |         return pushCursor == popCursor;
72 |     }
73 | 
74 | private:
75 |     size_type capacity_;
76 |     T* ring_;
77 |     mutable std::mutex mutex_;
78 |     size_type pushCursor_{};
79 |     size_type popCursor_{};
80 | };
81 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # cppcon2023
 2 | Code for "SPSC Lock-free, Wait-free Fifo from the Ground Up" presentation at CPPCON 2023
 3 | 
 4 | I compile and run this code on Windows 10 wsl version2 configured with Ubuntu-22.04. You will need to have these installed to build it:
 5 | 
 6 |     sudo apt install git
 7 |     sudo apt install cmake
 8 |     sudo apt install g++-12
 9 |     sudo apt install libbenchmark-dev
10 |     sudo apt install libboost-all-dev
11 |     sudo apt install libgtest-dev
12 | 
13 | And these if you want to run `perf stat`:
14 | 
15 |     sudo apt install linux-tools-5.15.90.1-microsoft-standard-WSL2
16 |     sudo apt install linux-tools-standard-WSL2
17 | 


--------------------------------------------------------------------------------
/TryLock.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <cstdlib>
 4 | #include <memory>
 5 | #include <mutex>
 6 | 
 7 | 
 8 | /// Thread-safe, try-lock mutex-based FIFO
 9 | template<typename T, typename Alloc = std::allocator<T>>
10 | class TryLock : private Alloc
11 | {
12 | public:
13 |     using value_type = T;
14 |     using allocator_traits = std::allocator_traits<Alloc>;
15 |     using size_type = typename allocator_traits::size_type;
16 | 
17 |     explicit TryLock(size_type capacity, Alloc const& alloc = Alloc{})
18 |         : Alloc{alloc}
19 |         , capacity_{capacity}
20 |         , ring_{allocator_traits::allocate(*this, capacity)}
21 |     {}
22 | 
23 |     ~TryLock() {
24 |         while(not empty()) {
25 |             ring_[popCursor_ % capacity_].~T();
26 |             ++popCursor_;
27 |         }
28 |         allocator_traits::deallocate(*this, ring_, capacity_);
29 |     }
30 | 
31 |     auto capacity() const noexcept { return capacity_; }
32 |     auto size() const noexcept {
33 |         std::lock_guard<std::mutex> lock(mutex_);
34 |         return size(pushCursor_, popCursor_);
35 |     }
36 |     auto empty() const noexcept { return size() == 0; }
37 |     auto full() const noexcept { return size() == capacity(); }
38 | 
39 |     auto push(T const& value) {
40 |         if (std::unique_lock<std::mutex> lock(mutex_, std::try_to_lock); lock.owns_lock()) {
41 |             if (full(pushCursor_, popCursor_)) {
42 |                 return false;
43 |             }
44 |             new (&ring_[pushCursor_ % capacity_]) T(value);
45 |             ++pushCursor_;
46 |             return true;
47 |         }
48 |         return false;
49 |     }
50 | 
51 |     auto pop(T& value) {
52 |         if(std::unique_lock<std::mutex> lock(mutex_, std::try_to_lock); lock.owns_lock()) {
53 |             if (empty(pushCursor_, popCursor_)) {
54 |                 return false;
55 |             }
56 |             value = ring_[popCursor_ % capacity_];
57 |             ring_[popCursor_ % capacity_].~T();
58 |             ++popCursor_;
59 |             return true;
60 |         }
61 |         return false;
62 |     }
63 | 
64 | private:
65 |     static auto size (size_type pushCursor, size_type popCursor) noexcept {
66 |         return pushCursor - popCursor;
67 |     }
68 | 
69 |     auto full(size_type pushCursor, size_type popCursor) const noexcept {
70 |         return (pushCursor - popCursor) == capacity_;
71 |     }
72 |     static auto empty(size_type pushCursor, size_type popCursor) noexcept {
73 |         return pushCursor == popCursor;
74 |     }
75 | 
76 | 
77 | private:
78 |     size_type capacity_;
79 |     T* ring_;
80 |     mutable std::mutex mutex_;
81 |     size_type pushCursor_{};
82 |     size_type popCursor_{};
83 | };
84 | 


--------------------------------------------------------------------------------
/bench.cpp:
--------------------------------------------------------------------------------
  1 | #include "Fifo4.hpp"
  2 | #include "Fifo4a.hpp"
  3 | #include "Fifo4b.hpp"
  4 | #include "Fifo5.hpp"
  5 | #include "Fifo5a.hpp"
  6 | #include "Fifo5b.hpp"
  7 | #include "rigtorp.hpp"
  8 | 
  9 | #include <benchmark/benchmark.h>
 10 | 
 11 | #include <iostream>
 12 | #include <thread>
 13 | 
 14 | 
 15 | static void pinThread(int cpu) {
 16 |     if (cpu < 0) {
 17 |         return;
 18 |     }
 19 |     ::cpu_set_t cpuset;
 20 |     CPU_ZERO(&cpuset);
 21 |     CPU_SET(cpu, &cpuset);
 22 |     if (::pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset) == -1) {
 23 |         std::perror("pthread_setaffinity_rp");
 24 |         std::exit(EXIT_FAILURE);
 25 |     }
 26 | }
 27 | 
 28 | constexpr auto cpu1 = 1;
 29 | constexpr auto cpu2 = 2;
 30 | 
 31 | template<typename T>
 32 | struct isRigtorp : std::false_type {};
 33 | 
 34 | template<typename ValueT>
 35 | struct isRigtorp<rigtorp::SPSCQueue<ValueT>> : std::true_type {};
 36 | 
 37 | 
 38 | template<template<typename> class FifoT>
 39 | void BM_Fifo(benchmark::State& state) {
 40 |     using fifo_type = FifoT<std::int_fast64_t>;
 41 |     using value_type = typename fifo_type::value_type;
 42 | 
 43 |     constexpr auto fifoSize = 131072;
 44 |     fifo_type fifo(fifoSize);
 45 | 
 46 |     auto t = std::jthread([&] {
 47 |         pinThread(cpu1);
 48 |         for (auto i = value_type{};; ++i) {
 49 |             value_type val;
 50 |             if constexpr(isRigtorp<fifo_type>::value) {
 51 |                 while (!fifo.front());
 52 |                 benchmark::DoNotOptimize(val = *fifo.front());
 53 |                 fifo.pop();
 54 |             } else {
 55 |                 while (not fifo.pop(val)) {
 56 |                     ;
 57 |                 }
 58 |                 benchmark::DoNotOptimize(val);
 59 |             }
 60 |             if (val == -1) {
 61 |                 break;
 62 |             }
 63 | 
 64 |             if (val != i) {
 65 |                 throw std::runtime_error("invalid value");
 66 |             }
 67 |         }
 68 |     });
 69 | 
 70 |     auto value = value_type{};
 71 |     pinThread(cpu2);
 72 |     for (auto _ : state) {
 73 |         if constexpr(isRigtorp<fifo_type>::value) {
 74 |             while (auto again = not fifo.try_push(value)) {
 75 |                 benchmark::DoNotOptimize(again);
 76 |             }
 77 |         } else {
 78 |             while (auto again = not fifo.push(value)) {
 79 |                 benchmark::DoNotOptimize(again);
 80 |             }
 81 |         }
 82 |         ++value;
 83 | 
 84 |         while (auto again = not fifo.empty()) {
 85 |             benchmark::DoNotOptimize(again);
 86 |         }
 87 |     }
 88 |     state.counters["ops/sec"] = benchmark::Counter(double(value), benchmark::Counter::kIsRate);
 89 |     state.PauseTiming();
 90 |     if constexpr(isRigtorp<fifo_type>::value) {
 91 |         while(not fifo.try_push(-1)) {}
 92 |     } else {
 93 |         fifo.push(-1);
 94 |     }
 95 | }
 96 | 
 97 | 
 98 | BENCHMARK_TEMPLATE(BM_Fifo, Fifo4);
 99 | BENCHMARK_TEMPLATE(BM_Fifo, Fifo4a);
100 | BENCHMARK_TEMPLATE(BM_Fifo, Fifo4b);
101 | BENCHMARK_TEMPLATE(BM_Fifo, Fifo5);
102 | BENCHMARK_TEMPLATE(BM_Fifo, Fifo5a);
103 | BENCHMARK_TEMPLATE(BM_Fifo, Fifo5b);
104 | BENCHMARK_TEMPLATE(BM_Fifo, rigtorp::SPSCQueue);
105 | 
106 | BENCHMARK_MAIN();
107 | 
108 | 


--------------------------------------------------------------------------------
/bench.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <chrono>
  4 | #include <cstdint>
  5 | #include <cstdio>
  6 | #include <cstdlib>
  7 | #include <iomanip>
  8 | #include <iostream>
  9 | #include <stdexcept>
 10 | #include <thread>
 11 | #include <utility>
 12 | 
 13 | #include <pthread.h>
 14 | 
 15 | template<typename T>
 16 | inline __attribute__((always_inline)) void doNotOptimize(T const& value) {
 17 |     asm volatile("" : : "r,m" (value) : "memory");
 18 | }
 19 | 
 20 | 
 21 | static void pinThread(int cpu) {
 22 |     if (cpu < 0) {
 23 |         return;
 24 |     }
 25 |     ::cpu_set_t cpuset;
 26 |     CPU_ZERO(&cpuset);
 27 |     CPU_SET(cpu, &cpuset);
 28 |     if (::pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset) == -1) {
 29 |         std::perror("pthread_setaffinity_rp");
 30 |         std::exit(EXIT_FAILURE);
 31 |     }
 32 | }
 33 | 
 34 | template<typename T>
 35 | struct isRigtorp : std::false_type {};
 36 | 
 37 | template<typename T>
 38 | class Bench
 39 | {
 40 | public:
 41 |     using value_type = typename T::value_type;
 42 | 
 43 |     static constexpr auto fifoSize = 131072;
 44 | 
 45 |     auto operator()(long iters, int cpu1, int cpu2) {
 46 |         using namespace std::chrono_literals;
 47 | 
 48 |         auto t = std::jthread([&] {
 49 |             pinThread(cpu1);
 50 |             // pop warmup
 51 |             for (auto i = value_type{}; i < fifoSize; ++i) {
 52 |                 pop(i);
 53 |             }
 54 | 
 55 |             // pop benchmark run
 56 |             for (auto i = value_type{}; i < iters; ++i) {
 57 |                 pop(i);
 58 |             }
 59 |         });
 60 | 
 61 |         pinThread(cpu2);
 62 |         // push warmup
 63 |         for (auto i = value_type{}; i < fifoSize; ++i) {
 64 |             push(i);
 65 |         }
 66 |         waitForEmpty();
 67 | 
 68 |         // push benchmark run
 69 |         auto start = std::chrono::steady_clock::now();
 70 |         for (auto i = value_type{}; i < iters; ++i) {
 71 |             push(i);
 72 |         }
 73 |         waitForEmpty();
 74 |         auto stop = std::chrono::steady_clock::now();
 75 | 
 76 |         auto delta = stop - start;
 77 |         return (iters * 1s)/delta;
 78 |     }
 79 | 
 80 | private:
 81 |     void pop(value_type expected) {
 82 |         value_type val;
 83 |         if constexpr(isRigtorp<T>::value) {
 84 |             while (auto again = not q.front()) {
 85 |                 doNotOptimize(again);
 86 | 	    }
 87 |             val = *q.front();
 88 |             q.pop();
 89 |         } else {
 90 |             while (auto again = not q.pop(val)) {
 91 |                 doNotOptimize(again);
 92 | 	    }
 93 |         }
 94 |         if (val != expected) {
 95 |             throw std::runtime_error("invalid value");
 96 |         }
 97 |     }
 98 | 
 99 |     void push(value_type i) {
100 |         if constexpr(isRigtorp<T>::value) {
101 |             while (auto again = not q.try_push(i)) {
102 |                 doNotOptimize(again);
103 |             }
104 |         } else {
105 |             while (auto again = not q.push(i)) {
106 |                 doNotOptimize(again);
107 |             }
108 |         }
109 |     }
110 | 
111 |     void waitForEmpty() {
112 |         while (auto again = not q.empty()) {
113 |             doNotOptimize(again);
114 |         }
115 |     };
116 | 
117 | private:
118 |     T q{fifoSize};
119 | };
120 | 
121 | 
122 | // "legacy" API
123 | template<typename T>
124 | auto bench(char const* name, long iters, int cpu1, int cpu2) {
125 |     return Bench<T>{}(iters, cpu1, cpu2);
126 | }
127 | 
128 | 
129 | template<template<typename> class FifoT>
130 | void bench(char const* name, int argc, char* argv[]) {
131 |     int cpu1 = 1;
132 |     int cpu2 = 2;
133 |     if (argc == 3) {
134 |        cpu1 = std::atoi(argv[1]);
135 |        cpu2 = std::atoi(argv[2]);
136 |     }
137 | 
138 |     constexpr auto iters = 400'000'000l;
139 |     // constexpr auto iters = 100'000'000l;
140 | 
141 |     using value_type = std::int64_t;
142 | 
143 |     auto opsPerSec = bench<FifoT<value_type>>(name, iters, cpu1, cpu2);
144 |     std::cout << std::setw(7) << std::left << name << ": "
145 |         << std::setw(10) << std::right << opsPerSec << " ops/s\n";
146 | }
147 | 


--------------------------------------------------------------------------------
/bench_all.cpp:
--------------------------------------------------------------------------------
 1 | #include "Fifo2.hpp"
 2 | #include "Fifo3.hpp"
 3 | #include "Fifo3a.hpp"
 4 | #include "Fifo4.hpp"
 5 | #include "Fifo4a.hpp"
 6 | #include "Fifo4b.hpp"
 7 | #include "Fifo5.hpp"
 8 | #include "Fifo5a.hpp"
 9 | #include "Fifo5b.hpp"
10 | #include "Mutex.hpp"
11 | #include "rigtorp.hpp"
12 | #include <boost/lockfree/spsc_queue.hpp>   // boost 1.74.0
13 | 
14 | #include "bench.hpp"
15 | 
16 | #include <iostream>
17 | 
18 | 
19 | template<typename ValueT>
20 | struct isRigtorp<rigtorp::SPSCQueue<ValueT>> : std::true_type {};
21 | 
22 | 
23 | // Configuring fixed size to match Fifo5's fixed size
24 | template<typename T>
25 | using boost_spsc_queue = boost::lockfree::spsc_queue<T, boost::lockfree::fixed_sized<true>>;
26 | 
27 | 
28 | template<typename ValueT>
29 | void once(long iters, int cpu1, int cpu2) {
30 |     std::cout <<
31 |         // Bench<Fifo2<ValueT>>{}(iters, cpu1, cpu2) << "," <<
32 |         // Bench<Mutex<ValueT>>{}(iters, cpu1, cpu2) <<
33 | 
34 |         Bench<Fifo3<ValueT>>{}(iters, cpu1, cpu2) << "," << std::flush <<
35 |         Bench<Fifo3a<ValueT>>{}(iters, cpu1, cpu2) << "," << std::flush <<
36 |         Bench<Fifo4<ValueT>>{}(iters, cpu1, cpu2) << "," << std::flush <<
37 |         Bench<Fifo4a<ValueT>>{}(iters, cpu1, cpu2) << "," << std::flush <<
38 |         Bench<Fifo4b<ValueT>>{}(iters, cpu1, cpu2) << "," << std::flush <<
39 |         Bench<Fifo5<ValueT>>{}(iters, cpu1, cpu2) << "," << std::flush <<
40 |         Bench<Fifo5a<ValueT>>{}(iters, cpu1, cpu2) << "," << std::flush <<
41 |         Bench<Fifo5b<ValueT>>{}(iters, cpu1, cpu2) << "," << std::flush <<
42 |         Bench<rigtorp::SPSCQueue<ValueT>>{}(iters, cpu1, cpu2) << "," << std::flush <<
43 |         Bench<boost_spsc_queue<ValueT>>{}(iters, cpu1, cpu2) << std::flush <<
44 |         "\n";
45 | }
46 | 
47 | int main(int argc, char* argv[]) {
48 |     constexpr auto cpu1 = 1;
49 |     constexpr auto cpu2 = 2;
50 |     constexpr auto iters = 400'000'000l;
51 | 
52 |     auto reps = 10;
53 |     if (argc == 2) {
54 |        reps = std::atoi(argv[1]);
55 |     }
56 | 
57 |     using value_type = std::int64_t;
58 | 
59 |     std::cout << "Fifo3,Fifo3a,Fifo4,Fifo4a,Fifo4b,Fifo5,Fifo5a,Fifo5b,rigtorp,boost_spsc_queue" << std::endl;
60 |     // std::cout << "Fifo2,Mutex\n";
61 |     for (auto rep = 0; rep < reps; ++rep) {
62 |         once<value_type>(iters, cpu1, cpu2);
63 |     }
64 | }
65 | 


--------------------------------------------------------------------------------
/boost_lockfree.cpp:
--------------------------------------------------------------------------------
 1 | #include "bench.hpp"
 2 | #include <boost/lockfree/spsc_queue.hpp>   // boost 1.74.0; /usr/include/boost/lockfree/spsc_queue.hpp
 3 | 
 4 | // Configuring fixed size to match Fifo5's fixed size
 5 | template<typename T>
 6 | using boost_spsc_queue = boost::lockfree::spsc_queue<T, boost::lockfree::fixed_sized<true>>;
 7 | 
 8 | int main(int argc, char* argv[]) {
 9 |     bench<boost_spsc_queue>("boost::spsc_queue fixed", argc, argv);
10 | }
11 | 


--------------------------------------------------------------------------------
/fifo1.cpp:
--------------------------------------------------------------------------------
1 | #include "Fifo1.hpp"
2 | #include "bench.hpp"
3 | 
4 | int main(int argc, char* argv[]) {
5 |     bench<Fifo1>("Fifo1", argc, argv);
6 | }
7 | 


--------------------------------------------------------------------------------
/fifo2.cpp:
--------------------------------------------------------------------------------
1 | #include "Fifo2.hpp"
2 | #include "bench.hpp"
3 | 
4 | int main(int argc, char* argv[]) {
5 |     bench<Fifo2>("Fifo2", argc, argv);
6 | }
7 | 


--------------------------------------------------------------------------------
/fifo3.cpp:
--------------------------------------------------------------------------------
1 | #include "Fifo3.hpp"
2 | #include "bench.hpp"
3 | 
4 | int main(int argc, char* argv[]) {
5 |     bench<Fifo3>("Fifo3", argc, argv);
6 | }
7 | 


--------------------------------------------------------------------------------
/fifo3a.cpp:
--------------------------------------------------------------------------------
1 | #include "Fifo3a.hpp"
2 | #include "bench.hpp"
3 | 
4 | int main(int argc, char* argv[]) {
5 |     bench<Fifo3a>("Fifo3a", argc, argv);
6 | }
7 | 


--------------------------------------------------------------------------------
/fifo4.cpp:
--------------------------------------------------------------------------------
1 | #include "Fifo4.hpp"
2 | #include "bench.hpp"
3 | 
4 | int main(int argc, char* argv[]) {
5 |     bench<Fifo4>("Fifo4", argc, argv);
6 | }
7 | 


--------------------------------------------------------------------------------
/fifo4a.cpp:
--------------------------------------------------------------------------------
1 | #include "Fifo4a.hpp"
2 | #include "bench.hpp"
3 | 
4 | int main(int argc, char* argv[]) {
5 |     bench<Fifo4a>("Fifo4a", argc, argv);
6 | }
7 | 


--------------------------------------------------------------------------------
/fifo4b.cpp:
--------------------------------------------------------------------------------
1 | #include "Fifo4b.hpp"
2 | #include "bench.hpp"
3 | 
4 | int main(int argc, char* argv[]) {
5 |     bench<Fifo4b>("Fifo4b", argc, argv);
6 | }
7 | 


--------------------------------------------------------------------------------
/fifo5.cpp:
--------------------------------------------------------------------------------
1 | #include "Fifo5.hpp"
2 | #include "bench.hpp"
3 | 
4 | int main(int argc, char* argv[]) {
5 |     bench<Fifo5>("Fifo5", argc, argv);
6 | }
7 | 


--------------------------------------------------------------------------------
/fifo5a.cpp:
--------------------------------------------------------------------------------
1 | #include "Fifo5a.hpp"
2 | #include "bench.hpp"
3 | 
4 | int main(int argc, char* argv[]) {
5 |     bench<Fifo5a>("Fifo5a", argc, argv);
6 | }
7 | 


--------------------------------------------------------------------------------
/fifo5b.cpp:
--------------------------------------------------------------------------------
1 | #include "Fifo5b.hpp"
2 | #include "bench.hpp"
3 | 
4 | int main(int argc, char* argv[]) {
5 |     bench<Fifo5b>("Fifo5b", argc, argv);
6 | }
7 | 
8 | 


--------------------------------------------------------------------------------
/mutex.cpp:
--------------------------------------------------------------------------------
1 | #include "Mutex.hpp"
2 | #include "bench.hpp"
3 | 
4 | int main(int argc, char* argv[]) {
5 |     bench<Mutex>("Mutex", argc, argv);
6 | }
7 | 


--------------------------------------------------------------------------------
/rigtorp.cpp:
--------------------------------------------------------------------------------
 1 | #include "bench.hpp"
 2 | #include "rigtorp.hpp"
 3 | 
 4 | 
 5 | template<typename ValueT>
 6 | struct isRigtorp<rigtorp::SPSCQueue<ValueT>> : std::true_type {};
 7 | 
 8 | int main(int argc, char* argv[]) {
 9 |     bench<rigtorp::SPSCQueue>("rigtorp", argc, argv);
10 | }
11 | 


--------------------------------------------------------------------------------
/rigtorp.hpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright (c) 2020 Erik Rigtorp <erik@rigtorp.se>
  3 | 
  4 | Permission is hereby granted, free of charge, to any person obtaining a copy
  5 | of this software and associated documentation files (the "Software"), to deal
  6 | in the Software without restriction, including without limitation the rights
  7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  8 | copies of the Software, and to permit persons to whom the Software is
  9 | furnished to do so, subject to the following conditions:
 10 | 
 11 | The above copyright notice and this permission notice shall be included in all
 12 | copies or substantial portions of the Software.
 13 | 
 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 20 | SOFTWARE.
 21 |  */
 22 | 
 23 | // The orignal rigtorp code at https://github.com/rigtorp/SPSCQueue has
 24 | // been slightly modified to work correctly with bench 
 25 | 
 26 | #pragma once
 27 | 
 28 | #include <atomic>
 29 | #include <cassert>
 30 | #include <cstddef>
 31 | #include <memory> // std::allocator
 32 | #include <new>    // std::hardware_destructive_interference_size
 33 | #include <stdexcept>
 34 | #include <type_traits> // std::enable_if, std::is_*_constructible
 35 | 
 36 | #ifdef __has_cpp_attribute
 37 | #if __has_cpp_attribute(nodiscard)
 38 | #define RIGTORP_NODISCARD [[nodiscard]]
 39 | #endif
 40 | #endif
 41 | #ifndef RIGTORP_NODISCARD
 42 | #define RIGTORP_NODISCARD
 43 | #endif
 44 | 
 45 | namespace rigtorp {
 46 | 
 47 | template <typename T, typename Allocator = std::allocator<T>> class SPSCQueue {
 48 | #if defined(__cpp_if_constexpr) && defined(__cpp_lib_void_t)
 49 |   template <typename Alloc2, typename = void>
 50 |   struct has_allocate_at_least : std::false_type {};
 51 | 
 52 |   template <typename Alloc2>
 53 |   struct has_allocate_at_least<
 54 |       Alloc2, std::void_t<typename Alloc2::value_type,
 55 |                           decltype(std::declval<Alloc2 &>().allocate_at_least(
 56 |                               size_t{}))>> : std::true_type {};
 57 | #endif
 58 | 
 59 | public:
 60 |   // Added for bench
 61 |   using value_type = T;
 62 | 
 63 |   explicit SPSCQueue(const size_t capacity,
 64 |                      const Allocator &allocator = Allocator())
 65 |       : capacity_(capacity), allocator_(allocator) {
 66 |     // The queue needs at least one element
 67 |     if (capacity_ < 1) {
 68 |       capacity_ = 1;
 69 |     }
 70 |     capacity_++; // Needs one slack element
 71 |     // Prevent overflowing size_t
 72 |     if (capacity_ > SIZE_MAX - 2 * kPadding) {
 73 |       capacity_ = SIZE_MAX - 2 * kPadding;
 74 |     }
 75 | 
 76 | #if defined(__cpp_if_constexpr) && defined(__cpp_lib_void_t)
 77 |     if constexpr (has_allocate_at_least<Allocator>::value) {
 78 |       auto res = allocator_.allocate_at_least(capacity_ + 2 * kPadding);
 79 |       slots_ = res.ptr;
 80 |       capacity_ = res.count - 2 * kPadding;
 81 |     } else {
 82 |       slots_ = std::allocator_traits<Allocator>::allocate(
 83 |           allocator_, capacity_ + 2 * kPadding);
 84 |     }
 85 | #else
 86 |     slots_ = std::allocator_traits<Allocator>::allocate(
 87 |         allocator_, capacity_ + 2 * kPadding);
 88 | #endif
 89 | 
 90 |     static_assert(alignof(SPSCQueue<T>) == kCacheLineSize, "");
 91 |     static_assert(sizeof(SPSCQueue<T>) >= 3 * kCacheLineSize, "");
 92 |     assert(reinterpret_cast<char *>(&readIdx_) -
 93 |                reinterpret_cast<char *>(&writeIdx_) >=
 94 |            static_cast<std::ptrdiff_t>(kCacheLineSize));
 95 |   }
 96 | 
 97 |   ~SPSCQueue() {
 98 |     while (front()) {
 99 |       pop();
100 |     }
101 |     std::allocator_traits<Allocator>::deallocate(allocator_, slots_,
102 |                                                  capacity_ + 2 * kPadding);
103 |   }
104 | 
105 |   // non-copyable and non-movable
106 |   SPSCQueue(const SPSCQueue &) = delete;
107 |   SPSCQueue &operator=(const SPSCQueue &) = delete;
108 | 
109 |   template <typename... Args>
110 |   void emplace(Args &&...args) noexcept(
111 |       std::is_nothrow_constructible<T, Args &&...>::value) {
112 |     static_assert(std::is_constructible<T, Args &&...>::value,
113 |                   "T must be constructible with Args&&...");
114 |     auto const writeIdx = writeIdx_.load(std::memory_order_relaxed);
115 |     auto nextWriteIdx = writeIdx + 1;
116 |     if (nextWriteIdx == capacity_) {
117 |       nextWriteIdx = 0;
118 |     }
119 |     while (nextWriteIdx == readIdxCache_) {
120 |       readIdxCache_ = readIdx_.load(std::memory_order_acquire);
121 |     }
122 |     new (&slots_[writeIdx + kPadding]) T(std::forward<Args>(args)...);
123 |     writeIdx_.store(nextWriteIdx, std::memory_order_release);
124 |   }
125 | 
126 |   template <typename... Args>
127 |   RIGTORP_NODISCARD bool try_emplace(Args &&...args) noexcept(
128 |       std::is_nothrow_constructible<T, Args &&...>::value) {
129 |     static_assert(std::is_constructible<T, Args &&...>::value,
130 |                   "T must be constructible with Args&&...");
131 |     auto const writeIdx = writeIdx_.load(std::memory_order_relaxed);
132 |     auto nextWriteIdx = writeIdx + 1;
133 |     if (nextWriteIdx == capacity_) {
134 |       nextWriteIdx = 0;
135 |     }
136 |     if (nextWriteIdx == readIdxCache_) {
137 |       readIdxCache_ = readIdx_.load(std::memory_order_acquire);
138 |       if (nextWriteIdx == readIdxCache_) {
139 |         return false;
140 |       }
141 |     }
142 |     new (&slots_[writeIdx + kPadding]) T(std::forward<Args>(args)...);
143 |     writeIdx_.store(nextWriteIdx, std::memory_order_release);
144 |     return true;
145 |   }
146 | 
147 |   void push(const T &v) noexcept(std::is_nothrow_copy_constructible<T>::value) {
148 |     static_assert(std::is_copy_constructible<T>::value,
149 |                   "T must be copy constructible");
150 |     emplace(v);
151 |   }
152 | 
153 |   template <typename P, typename = typename std::enable_if<
154 |                             std::is_constructible<T, P &&>::value>::type>
155 |   void push(P &&v) noexcept(std::is_nothrow_constructible<T, P &&>::value) {
156 |     emplace(std::forward<P>(v));
157 |   }
158 | 
159 |   RIGTORP_NODISCARD bool
160 |   try_push(const T &v) noexcept(std::is_nothrow_copy_constructible<T>::value) {
161 |     static_assert(std::is_copy_constructible<T>::value,
162 |                   "T must be copy constructible");
163 |     return try_emplace(v);
164 |   }
165 | 
166 |   template <typename P, typename = typename std::enable_if<
167 |                             std::is_constructible<T, P &&>::value>::type>
168 |   RIGTORP_NODISCARD bool
169 |   try_push(P &&v) noexcept(std::is_nothrow_constructible<T, P &&>::value) {
170 |     return try_emplace(std::forward<P>(v));
171 |   }
172 | 
173 |   RIGTORP_NODISCARD T *front() noexcept {
174 |     auto const readIdx = readIdx_.load(std::memory_order_relaxed);
175 |     if (readIdx == writeIdxCache_) {
176 |       writeIdxCache_ = writeIdx_.load(std::memory_order_acquire);
177 |       if (writeIdxCache_ == readIdx) {
178 |         return nullptr;
179 |       }
180 |     }
181 |     return &slots_[readIdx + kPadding];
182 |   }
183 | 
184 |   void pop() noexcept {
185 |     static_assert(std::is_nothrow_destructible<T>::value,
186 |                   "T must be nothrow destructible");
187 |     auto const readIdx = readIdx_.load(std::memory_order_relaxed);
188 |     assert(writeIdx_.load(std::memory_order_acquire) != readIdx);
189 |     slots_[readIdx + kPadding].~T();
190 |     auto nextReadIdx = readIdx + 1;
191 |     if (nextReadIdx == capacity_) {
192 |       nextReadIdx = 0;
193 |     }
194 |     readIdx_.store(nextReadIdx, std::memory_order_release);
195 |   }
196 | 
197 |   RIGTORP_NODISCARD size_t size() const noexcept {
198 |     std::ptrdiff_t diff = writeIdx_.load(std::memory_order_acquire) -
199 |                           readIdx_.load(std::memory_order_acquire);
200 |     if (diff < 0) {
201 |       diff += capacity_;
202 |     }
203 |     return static_cast<size_t>(diff);
204 |   }
205 | 
206 |   RIGTORP_NODISCARD bool empty() const noexcept {
207 |     return writeIdx_.load(std::memory_order_acquire) ==
208 |            readIdx_.load(std::memory_order_acquire);
209 |   }
210 | 
211 |   RIGTORP_NODISCARD size_t capacity() const noexcept { return capacity_ - 1; }
212 | 
213 | private:
214 | #ifdef __cpp_lib_hardware_interference_size
215 |   static constexpr size_t kCacheLineSize =
216 |       std::hardware_destructive_interference_size;
217 | #else
218 |   static constexpr size_t kCacheLineSize = 64;
219 | #endif
220 | 
221 |   // Padding to avoid false sharing between slots_ and adjacent allocations
222 |   static constexpr size_t kPadding = (kCacheLineSize - 1) / sizeof(T) + 1;
223 | 
224 | private:
225 |   size_t capacity_;
226 |   T *slots_;
227 | #if defined(__has_cpp_attribute) && __has_cpp_attribute(no_unique_address)
228 |   Allocator allocator_ [[no_unique_address]];
229 | #else
230 |   Allocator allocator_;
231 | #endif
232 | 
233 |   // Align to cache line size in order to avoid false sharing
234 |   // readIdxCache_ and writeIdxCache_ is used to reduce the amount of cache
235 |   // coherency traffic
236 |   alignas(kCacheLineSize) std::atomic<size_t> writeIdx_ = {0};
237 |   alignas(kCacheLineSize) size_t readIdxCache_ = 0;
238 |   alignas(kCacheLineSize) std::atomic<size_t> readIdx_ = {0};
239 |   alignas(kCacheLineSize) size_t writeIdxCache_ = 0;
240 | 
241 |   // Padding to avoid adjacent allocations to share cache line with
242 |   // writeIdxCache_
243 |   char padding_[kCacheLineSize - sizeof(writeIdxCache_)];
244 | };
245 | } // namespace rigtorp
246 | 


--------------------------------------------------------------------------------
/run_all.sh:
--------------------------------------------------------------------------------
1 | #! /usr/bin/bash
2 | 
3 | for bench in fifo2 fifo3 fifo4 fifo4a fifo4b fifo5 fifo5a fifo5b rigtorp boost_lockfree mutex;
4 | do
5 | ./build/release/$bench 1 2
6 | done
7 | 


--------------------------------------------------------------------------------
/tryLock.cpp:
--------------------------------------------------------------------------------
1 | #include "TryLock.hpp"
2 | #include "bench.hpp"
3 | 
4 | int main(int argc, char* argv[]) {
5 |     bench<TryLock>("TryLock", argc, argv);
6 | }
7 | 


--------------------------------------------------------------------------------
/unitTests.cpp:
--------------------------------------------------------------------------------
  1 | #include "Fifo1.hpp"
  2 | #include "Fifo2.hpp"
  3 | #include "Fifo3.hpp"
  4 | #include "Fifo3a.hpp"
  5 | #include "Fifo4.hpp"
  6 | #include "Fifo4b.hpp"
  7 | #include "Fifo5.hpp"
  8 | #include "Fifo5a.hpp"
  9 | #include "Fifo5b.hpp"
 10 | 
 11 | #include <gtest/gtest.h>
 12 | 
 13 | #include <type_traits>
 14 | 
 15 | 
 16 | extern "C" {
 17 |     void __ubsan_on_report() {
 18 |           FAIL() << "Encountered an undefined behavior sanitizer error";
 19 |     }
 20 |     void __asan_on_error() {
 21 |           FAIL() << "Encountered an address sanitizer error";
 22 |     }
 23 |     void __tsan_on_report() {
 24 |           FAIL() << "Encountered a thread sanitizer error";
 25 |     }
 26 | }  // extern "C"
 27 | 
 28 | 
 29 | template<typename FifoT>
 30 | class FifoTestBase : public testing::Test {
 31 | public:
 32 |     using FifoType = FifoT;
 33 |     using value_type = typename FifoType::value_type;
 34 | 
 35 |     FifoType fifo{4};
 36 | };
 37 | 
 38 | using test_type = unsigned int;
 39 | 
 40 | 
 41 | template<typename FifoT> using FifoTest = FifoTestBase<FifoT>;
 42 | using FifoTypes = ::testing::Types<
 43 |     Fifo1<test_type>,
 44 |     Fifo2<test_type>,
 45 |     Fifo3<test_type>,
 46 |     Fifo3a<test_type>,
 47 |     Fifo4<test_type>,
 48 |     Fifo4b<test_type>,
 49 |     Fifo5<test_type>,
 50 |     Fifo5b<test_type>
 51 |     >;
 52 | TYPED_TEST_SUITE(FifoTest, FifoTypes);
 53 | 
 54 | TYPED_TEST(FifoTest, properties) {
 55 |     EXPECT_FALSE(std::is_default_constructible_v<typename TestFixture::FifoType>);
 56 |     EXPECT_TRUE((std::is_constructible_v<typename TestFixture::FifoType, unsigned long>));
 57 |     EXPECT_TRUE((std::is_constructible_v<typename TestFixture::FifoType, unsigned long, std::allocator<typename TestFixture::value_type>>));
 58 |     EXPECT_FALSE(std::is_copy_constructible_v<typename TestFixture::FifoType>);
 59 |     EXPECT_FALSE(std::is_move_constructible_v<typename TestFixture::FifoType>);
 60 |     EXPECT_FALSE(std::is_copy_assignable_v<typename TestFixture::FifoType>);
 61 |     EXPECT_FALSE(std::is_move_assignable_v<typename TestFixture::FifoType>);
 62 |     EXPECT_TRUE(std::is_destructible_v<typename TestFixture::FifoType>);
 63 | }
 64 | 
 65 | TYPED_TEST(FifoTest, initialConditions) {
 66 |     EXPECT_EQ(4u, this->fifo.capacity());
 67 |     EXPECT_EQ(0, this->fifo.size());
 68 |     EXPECT_TRUE(this->fifo.empty());
 69 |     EXPECT_FALSE(this->fifo.full());
 70 | }
 71 | 
 72 | TYPED_TEST(FifoTest, push) {
 73 |     ASSERT_EQ(4u, this->fifo.capacity());
 74 | 
 75 |     EXPECT_TRUE(this->fifo.push(42));
 76 |     EXPECT_EQ(1u, this->fifo.size());
 77 |     EXPECT_FALSE(this->fifo.empty());
 78 |     EXPECT_FALSE(this->fifo.full());
 79 | 
 80 |     EXPECT_TRUE(this->fifo.push(42));
 81 |     EXPECT_EQ(2u, this->fifo.size());
 82 |     EXPECT_FALSE(this->fifo.empty());
 83 |     EXPECT_FALSE(this->fifo.full());
 84 | 
 85 |     EXPECT_TRUE(this->fifo.push(42));
 86 |     EXPECT_EQ(3u, this->fifo.size());
 87 |     EXPECT_FALSE(this->fifo.empty());
 88 |     EXPECT_FALSE(this->fifo.full());
 89 | 
 90 |     EXPECT_TRUE(this->fifo.push(42));
 91 |     EXPECT_EQ(4u, this->fifo.size());
 92 |     EXPECT_FALSE(this->fifo.empty());
 93 |     EXPECT_TRUE(this->fifo.full());
 94 | 
 95 |     EXPECT_FALSE(this->fifo.push(42));
 96 |     EXPECT_EQ(4u, this->fifo.size());
 97 |     EXPECT_FALSE(this->fifo.empty());
 98 |     EXPECT_TRUE(this->fifo.full());
 99 | }
100 | 
101 | TYPED_TEST(FifoTest, pop) {
102 |     auto value = typename TestFixture::value_type{};
103 |     EXPECT_FALSE(this->fifo.pop(value));
104 | 
105 |     for (auto i = 0u; i < this->fifo.capacity(); ++i) {
106 |         this->fifo.push(42 + i);
107 |     }
108 | 
109 |     for (auto i = 0u; i < this->fifo.capacity(); ++i) {
110 |         EXPECT_EQ(this->fifo.capacity() - i, this->fifo.size());
111 |         auto value = typename TestFixture::value_type{};
112 |         EXPECT_TRUE(this->fifo.pop(value));
113 |         EXPECT_EQ(42 + i, value);
114 |     }
115 |     EXPECT_EQ(0, this->fifo.size());
116 |     EXPECT_TRUE(this->fifo.empty());
117 |     EXPECT_FALSE(this->fifo.pop(value));
118 | }
119 | 
120 | TYPED_TEST(FifoTest, popFullFifo) {
121 |     auto value = typename TestFixture::value_type{};
122 |     EXPECT_FALSE(this->fifo.pop(value));
123 | 
124 |     for (auto i = 0u; i < this->fifo.capacity(); ++i) {
125 |         ASSERT_EQ(i, this->fifo.size());
126 |         ASSERT_TRUE(this->fifo.push(42 + i));
127 |     }
128 |     EXPECT_EQ(this->fifo.capacity(), this->fifo.size());
129 |     EXPECT_TRUE(this->fifo.full());
130 | 
131 |     for (auto i = 0u; i < this->fifo.capacity()*4; ++i) {
132 |         EXPECT_TRUE(this->fifo.pop(value));
133 |         EXPECT_EQ(42 + i, value);
134 |         EXPECT_FALSE(this->fifo.full());
135 | 
136 |         EXPECT_TRUE(this->fifo.push(42 + 4 + i));
137 |         EXPECT_TRUE(this->fifo.full());
138 |         EXPECT_EQ(this->fifo.capacity(), this->fifo.size());
139 |     }
140 | }
141 | 
142 | TYPED_TEST(FifoTest, popEmpty) {
143 |     auto value = typename TestFixture::value_type{};
144 |     EXPECT_FALSE(this->fifo.pop(value));
145 | 
146 |     for (auto i = 0u; i < this->fifo.capacity()*4; ++i) {
147 |         EXPECT_TRUE(this->fifo.empty());
148 |         EXPECT_TRUE(this->fifo.push(42 + i));
149 |         EXPECT_TRUE(this->fifo.pop(value));
150 |         EXPECT_EQ(42 + i, value);
151 |     }
152 | 
153 |     EXPECT_TRUE(this->fifo.empty());
154 |     EXPECT_FALSE(this->fifo.pop(value));
155 | }
156 | 
157 | TYPED_TEST(FifoTest, wrap) {
158 |     auto value = typename TestFixture::value_type{};
159 |     for (auto i = 0u; i < this->fifo.capacity() * 2 + 1; ++i) {
160 |         this->fifo.push(42 + i);
161 |         EXPECT_TRUE(this->fifo.pop(value));
162 |         EXPECT_EQ(42 + i, value);
163 |     }
164 | 
165 |     for (auto i = 0u; i < 8u; ++i) {
166 |         this->fifo.push(42 + i);
167 |         EXPECT_TRUE(this->fifo.pop(value));
168 |         EXPECT_EQ(42 + i, value);
169 |     }
170 | }
171 | 
172 | 
173 | template<typename FifoT> using ProxyTest = FifoTestBase<FifoT>;
174 | using ProxyFifoTypes = ::testing::Types<
175 |     Fifo5<test_type>
176 |     >;
177 | TYPED_TEST_SUITE(ProxyTest, ProxyFifoTypes);
178 | 
179 | TYPED_TEST(ProxyTest, properties) {
180 |         EXPECT_FALSE(std::is_copy_constructible_v<typename TestFixture::FifoType::pusher_t>);
181 |         EXPECT_TRUE(std::is_move_constructible_v<typename TestFixture::FifoType::pusher_t>);
182 |         EXPECT_FALSE(std::is_copy_assignable_v<typename TestFixture::FifoType::pusher_t>);
183 |         EXPECT_TRUE(std::is_move_assignable_v<typename TestFixture::FifoType::pusher_t>);
184 | 
185 |         EXPECT_FALSE(std::is_copy_constructible_v<typename TestFixture::FifoType::popper_t>);
186 |         EXPECT_TRUE(std::is_move_constructible_v<typename TestFixture::FifoType::popper_t>);
187 |         EXPECT_FALSE(std::is_copy_assignable_v<typename TestFixture::FifoType::popper_t>);
188 |         EXPECT_TRUE(std::is_move_assignable_v<typename TestFixture::FifoType::popper_t>);
189 | }
190 | 
191 | TYPED_TEST(ProxyTest, pusher) {
192 |     for (auto i = 0u; i < this->fifo.capacity(); ++i) {
193 |         EXPECT_FALSE(this->fifo.full());
194 |         {
195 |             auto pusher = this->fifo.push();
196 |             EXPECT_TRUE(!!pusher);
197 |             pusher = i;
198 | 
199 |             EXPECT_EQ(i, *pusher.get());
200 |             EXPECT_EQ(i, *pusher);
201 |             EXPECT_EQ(i, *pusher.operator->());
202 |             // TODO EXPECT_EQ(i, static_cast<typename TestFixture::FifoType::ValueType>(pusher));
203 | 
204 |             EXPECT_EQ(i, *std::as_const(pusher).get());
205 |             EXPECT_EQ(i, *std::as_const(pusher));
206 |             EXPECT_EQ(i, *std::as_const(pusher).operator->());
207 |             // TODO EXPECT_EQ(i, static_cast<typename TestFixture::FifoType::ValueType>(std::as_const(pusher)));
208 |         }
209 |         EXPECT_FALSE(this->fifo.empty());
210 |     }
211 |     EXPECT_TRUE(this->fifo.full());
212 |     EXPECT_FALSE(!!this->fifo.push());
213 | }
214 | 
215 | TYPED_TEST(ProxyTest, pusherRelease) {
216 |     this->fifo.push() = 42;
217 |     EXPECT_FALSE(this->fifo.empty());
218 | 
219 |     {
220 |         auto pusher = this->fifo.push();
221 |         ASSERT_TRUE(!!pusher);
222 |         pusher = 24;
223 |         pusher.release();
224 |         EXPECT_FALSE(!!pusher);
225 |     }
226 |     EXPECT_EQ(42, *this->fifo.pop());
227 |     EXPECT_TRUE(this->fifo.empty());
228 | }
229 | 
230 | TYPED_TEST(ProxyTest, popperRelease) {
231 |     this->fifo.push() = 42;
232 |     EXPECT_FALSE(this->fifo.empty());
233 | 
234 |     {
235 |         auto popper = this->fifo.pop();
236 |         ASSERT_TRUE(!!popper);
237 |         EXPECT_EQ(42, *popper);
238 |         popper.release();
239 |         EXPECT_FALSE(!!popper);
240 |     }
241 |     EXPECT_FALSE(this->fifo.empty());
242 |     EXPECT_EQ(42, *this->fifo.pop());
243 | }
244 | 
245 | 
246 | struct ABC
247 | {
248 |     int a;
249 |     int b;
250 |     int c;
251 | };
252 | 
253 | // Specialize ValueSizeTraits not to copy ABC::c.
254 | template<>
255 | inline std::size_t ValueSizeTraits<ABC>::size(value_type const&) {
256 |     return sizeof(ABC::a) + sizeof(ABC::b);
257 | }
258 | 
259 | 
260 | template<typename FifoT> using ProxyMoveTest = FifoTestBase<FifoT>;
261 | using ProxyMoveFifoTypes = ::testing::Types<
262 |     Fifo5<ABC>
263 |     >;
264 | TYPED_TEST_SUITE(ProxyMoveTest, ProxyMoveFifoTypes);
265 | 
266 | TYPED_TEST(ProxyMoveTest, pusherMove) {
267 |     auto pusher = this->fifo.push();
268 | 
269 |     pusher = ABC{100, 200, 300};
270 |     EXPECT_TRUE(!!pusher);
271 |     EXPECT_EQ(100, pusher->a);
272 | 
273 |     // move ctor
274 |     auto pusher2 = std::move(pusher);
275 |     EXPECT_FALSE(!!pusher);
276 |     EXPECT_TRUE(!!pusher2);
277 |     EXPECT_EQ(100, pusher2->a);
278 | 
279 |     // move assignment
280 |     pusher = std::move(pusher2);
281 |     EXPECT_TRUE(!!pusher);
282 |     EXPECT_FALSE(!!pusher2);
283 |     EXPECT_EQ(100, pusher->a);
284 | }
285 | 
286 | TYPED_TEST(ProxyMoveTest, popperMove) {
287 |     for (auto i = 0; i < static_cast<int>(this->fifo.capacity()); ++i) {
288 |         this->fifo.push(ABC{42 + i, 43, 44});
289 |     }
290 | 
291 |     for (auto i = 0u; i < this->fifo.capacity(); ++i) {
292 |         auto popper = this->fifo.pop();
293 |         ASSERT_TRUE(!!popper);
294 |         EXPECT_EQ(42 + i, popper->a);
295 | 
296 |         // move ctor
297 |         auto popper2 = std::move(popper);
298 |         EXPECT_FALSE(!!popper);
299 |         ASSERT_TRUE(!!popper2);
300 |         EXPECT_EQ(42 + i, popper2->a);
301 | 
302 |         // move assignment
303 |         popper = std::move(popper2);
304 |         ASSERT_TRUE(!!popper);
305 |         EXPECT_FALSE(!!popper2);
306 |         EXPECT_EQ(42 + i, popper->a);
307 |     }
308 |     EXPECT_TRUE(this->fifo.empty());
309 | }
310 | 
311 | TYPED_TEST(ProxyMoveTest, pusherUsesValueSizeTraits) {
312 |     // Push and pop a value into every slot of the fifo using assign
313 |     // directly to value_type&. This bypasses use of ValueSizeTraits.
314 |     for (auto i = 0; i < static_cast<int>(this->fifo.capacity()); ++i) {
315 |         {
316 |             auto pusher = this->fifo.push();
317 |             *pusher = ABC{1, 2, 3};
318 |         }
319 |         auto popper = this->fifo.pop();
320 |         EXPECT_EQ(1, popper->a);
321 |         EXPECT_EQ(2, popper->b);
322 |         EXPECT_EQ(3, popper->c);
323 |     }
324 | 
325 |     // Now push using operator= and a different value
326 |     {
327 |         auto pusher = this->fifo.push();
328 |         pusher = ABC{100, 200, 300};
329 |     }
330 | 
331 |     auto popper = this->fifo.pop();
332 |     EXPECT_EQ(100, popper->a);
333 |     EXPECT_EQ(200, popper->b);
334 |     EXPECT_EQ(3, popper->c);
335 | 
336 | }
337 | 


--------------------------------------------------------------------------------