├── .clang-format ├── spsc.odg ├── .gitignore ├── src ├── SPSCQueueExample.cpp ├── SPSCQueueExampleHugepages.cpp ├── SPSCQueueTest.cpp └── SPSCQueueBenchmark.cpp ├── LICENSE ├── .github └── workflows │ └── ccpp.yml ├── CMakeLists.txt ├── README.md ├── include └── rigtorp │ └── SPSCQueue.h └── spsc.svg /.clang-format: -------------------------------------------------------------------------------- 1 | BasedOnStyle: LLVM -------------------------------------------------------------------------------- /spsc.odg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rigtorp/SPSCQueue/HEAD/spsc.odg -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | cmake-build-*/ 3 | *~ 4 | .idea/ 5 | .vscode/ 6 | .cache/ -------------------------------------------------------------------------------- /src/SPSCQueueExample.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | int main(int argc, char *argv[]) { 6 | (void)argc, (void)argv; 7 | 8 | using namespace rigtorp; 9 | 10 | SPSCQueue q(1); 11 | auto t = std::thread([&] { 12 | while (!q.front()) 13 | ; 14 | std::cout << *q.front() << std::endl; 15 | q.pop(); 16 | }); 17 | q.push(1); 18 | t.join(); 19 | 20 | return 0; 21 | } 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Erik Rigtorp 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /.github/workflows/ccpp.yml: -------------------------------------------------------------------------------- 1 | name: C/C++ CI 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build-ubuntu: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v1 12 | - name: Build & Test 13 | run: | 14 | cmake -E remove_directory build 15 | cmake -B build -S . -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-Werror -O2 -fsanitize=address,undefined" 16 | cmake --build build 17 | cd build 18 | ctest --output-on-failure 19 | 20 | build-windows: 21 | 22 | runs-on: windows-latest 23 | 24 | steps: 25 | - uses: actions/checkout@v1 26 | - name: Build & Test 27 | run: | 28 | cmake -E remove_directory build 29 | cmake -B build -S . 30 | cmake --build build --config Debug 31 | cd build 32 | ctest --output-on-failure 33 | 34 | build-macos: 35 | 36 | runs-on: macOS-latest 37 | 38 | steps: 39 | - uses: actions/checkout@v1 40 | - name: Build & Test 41 | run: | 42 | cmake -E remove_directory build 43 | cmake -B build -S . -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-Werror -O2 -fsanitize=address,undefined" 44 | cmake --build build 45 | cd build 46 | ctest --output-on-failure -------------------------------------------------------------------------------- /src/SPSCQueueExampleHugepages.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | template struct Allocator { 7 | using value_type = T; 8 | 9 | struct AllocationResult { 10 | T *ptr; 11 | size_t count; 12 | }; 13 | 14 | size_t roundup(size_t n) { return (((n - 1) >> 21) + 1) << 21; } 15 | 16 | AllocationResult allocate_at_least(size_t n) { 17 | size_t count = roundup(sizeof(T) * n); 18 | auto p = static_cast(mmap(nullptr, count, PROT_READ | PROT_WRITE, 19 | MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, 20 | -1, 0)); 21 | if (p == MAP_FAILED) { 22 | throw std::bad_alloc(); 23 | } 24 | return {p, count / sizeof(T)}; 25 | } 26 | 27 | void deallocate(T *p, size_t n) { munmap(p, roundup(sizeof(T) * n)); } 28 | }; 29 | 30 | int main(int argc, char *argv[]) { 31 | (void)argc, (void)argv; 32 | 33 | using namespace rigtorp; 34 | 35 | SPSCQueue> q(2); 36 | std::cout << q.capacity() << std::endl; 37 | auto t = std::thread([&] { 38 | while (!q.front()) 39 | ; 40 | std::cout << *q.front() << std::endl; 41 | q.pop(); 42 | }); 43 | q.push(1); 44 | t.join(); 45 | 46 | return 0; 47 | } 48 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.6) 2 | 3 | project(SPSCQueue VERSION 1.1 LANGUAGES CXX) 4 | 5 | add_library(${PROJECT_NAME} INTERFACE) 6 | add_library(${PROJECT_NAME}::${PROJECT_NAME} ALIAS ${PROJECT_NAME}) 7 | 8 | target_compile_features(${PROJECT_NAME} INTERFACE cxx_std_11) 9 | 10 | target_include_directories(${PROJECT_NAME} INTERFACE 11 | $ 12 | $) 13 | 14 | # Tests and examples 15 | if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) 16 | if(MSVC) 17 | add_compile_options(/permissive- /W4) 18 | else() 19 | add_compile_options(-Wall -Wextra -Wpedantic -Wno-c++17-extensions) 20 | endif() 21 | 22 | find_package(Threads REQUIRED) 23 | 24 | if (UNIX AND NOT APPLE) 25 | add_executable(SPSCQueueBenchmark src/SPSCQueueBenchmark.cpp) 26 | target_link_libraries(SPSCQueueBenchmark SPSCQueue Threads::Threads) 27 | endif() 28 | 29 | if (((CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 7.0) 30 | OR (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 9.0)) 31 | AND UNIX AND NOT APPLE) 32 | add_executable(SPSCQueueExampleHugepages src/SPSCQueueExampleHugepages.cpp) 33 | target_link_libraries(SPSCQueueExampleHugepages SPSCQueue Threads::Threads) 34 | target_compile_features(SPSCQueueExampleHugepages PRIVATE cxx_std_17) 35 | endif() 36 | 37 | add_executable(SPSCQueueExample src/SPSCQueueExample.cpp) 38 | target_link_libraries(SPSCQueueExample SPSCQueue Threads::Threads) 39 | 40 | add_executable(SPSCQueueTest src/SPSCQueueTest.cpp) 41 | target_link_libraries(SPSCQueueTest SPSCQueue Threads::Threads) 42 | 43 | enable_testing() 44 | add_test(SPSCQueueTest SPSCQueueTest) 45 | endif() 46 | 47 | # Install 48 | include(GNUInstallDirs) 49 | include(CMakePackageConfigHelpers) 50 | 51 | write_basic_package_version_file( 52 | "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake" 53 | COMPATIBILITY SameMajorVersion 54 | ) 55 | 56 | export( 57 | TARGETS ${PROJECT_NAME} 58 | NAMESPACE ${PROJECT_NAME}:: 59 | FILE "${PROJECT_NAME}Config.cmake" 60 | ) 61 | 62 | if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) 63 | install( 64 | DIRECTORY "include/" 65 | DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} 66 | ) 67 | 68 | install( 69 | TARGETS ${PROJECT_NAME} 70 | EXPORT "${PROJECT_NAME}Config" 71 | INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} 72 | ) 73 | 74 | install( 75 | EXPORT "${PROJECT_NAME}Config" 76 | NAMESPACE ${PROJECT_NAME}:: 77 | DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" 78 | ) 79 | 80 | install( 81 | FILES "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake" 82 | DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" 83 | ) 84 | endif() -------------------------------------------------------------------------------- /src/SPSCQueueTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2018 Erik Rigtorp 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | SOFTWARE. 21 | */ 22 | 23 | #undef NDEBUG 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | // TestType tracks correct usage of constructors and destructors 33 | struct TestType { 34 | static std::set constructed; 35 | TestType() noexcept { 36 | assert(constructed.count(this) == 0); 37 | constructed.insert(this); 38 | }; 39 | TestType(const TestType &other) noexcept { 40 | assert(constructed.count(this) == 0); 41 | assert(constructed.count(&other) == 1); 42 | constructed.insert(this); 43 | }; 44 | TestType(TestType &&other) noexcept { 45 | assert(constructed.count(this) == 0); 46 | assert(constructed.count(&other) == 1); 47 | constructed.insert(this); 48 | }; 49 | TestType &operator=(const TestType &other) noexcept { 50 | assert(constructed.count(this) == 1); 51 | assert(constructed.count(&other) == 1); 52 | return *this; 53 | }; 54 | TestType &operator=(TestType &&other) noexcept { 55 | assert(constructed.count(this) == 1); 56 | assert(constructed.count(&other) == 1); 57 | return *this; 58 | } 59 | ~TestType() noexcept { 60 | assert(constructed.count(this) == 1); 61 | constructed.erase(this); 62 | }; 63 | }; 64 | 65 | std::set TestType::constructed; 66 | 67 | int main(int argc, char *argv[]) { 68 | (void)argc, (void)argv; 69 | 70 | using namespace rigtorp; 71 | 72 | // Functionality test 73 | { 74 | SPSCQueue q(10); 75 | assert(q.front() == nullptr); 76 | assert(q.size() == 0); 77 | assert(q.empty() == true); 78 | assert(q.capacity() == 10); 79 | for (int i = 0; i < 10; i++) { 80 | q.emplace(); 81 | } 82 | assert(q.front() != nullptr); 83 | assert(q.size() == 10); 84 | assert(q.empty() == false); 85 | assert(TestType::constructed.size() == 10); 86 | assert(q.try_emplace() == false); 87 | q.pop(); 88 | assert(q.size() == 9); 89 | assert(TestType::constructed.size() == 9); 90 | q.pop(); 91 | assert(q.try_emplace() == true); 92 | assert(TestType::constructed.size() == 9); 93 | } 94 | assert(TestType::constructed.size() == 0); 95 | 96 | // Copyable only type 97 | { 98 | struct Test { 99 | Test() {} 100 | Test(const Test &) {} 101 | Test(Test &&) = delete; 102 | }; 103 | SPSCQueue q(16); 104 | // lvalue 105 | Test v; 106 | q.emplace(v); 107 | (void)q.try_emplace(v); 108 | q.push(v); 109 | (void)q.try_push(v); 110 | static_assert(noexcept(q.emplace(v)) == false, ""); 111 | static_assert(noexcept(q.try_emplace(v)) == false, ""); 112 | static_assert(noexcept(q.push(v)) == false, ""); 113 | static_assert(noexcept(q.try_push(v)) == false, ""); 114 | // xvalue 115 | q.push(Test()); 116 | (void)q.try_push(Test()); 117 | static_assert(noexcept(q.push(Test())) == false, ""); 118 | static_assert(noexcept(q.try_push(Test())) == false, ""); 119 | } 120 | 121 | // Copyable only type (noexcept) 122 | { 123 | struct Test { 124 | Test() noexcept {} 125 | Test(const Test &) noexcept {} 126 | Test(Test &&) = delete; 127 | }; 128 | SPSCQueue q(16); 129 | // lvalue 130 | Test v; 131 | q.emplace(v); 132 | (void)q.try_emplace(v); 133 | q.push(v); 134 | (void)q.try_push(v); 135 | static_assert(noexcept(q.emplace(v)) == true, ""); 136 | static_assert(noexcept(q.try_emplace(v)) == true, ""); 137 | static_assert(noexcept(q.push(v)) == true, ""); 138 | static_assert(noexcept(q.try_push(v)) == true, ""); 139 | // xvalue 140 | q.push(Test()); 141 | (void)q.try_push(Test()); 142 | static_assert(noexcept(q.push(Test())) == true, ""); 143 | static_assert(noexcept(q.try_push(Test())) == true, ""); 144 | } 145 | 146 | // Movable only type 147 | { 148 | SPSCQueue> q(16); 149 | // lvalue 150 | // auto v = std::unique_ptr(new int(1)); 151 | // q.emplace(v); 152 | // q.try_emplace(v); 153 | // q.push(v); 154 | // q.try_push(v); 155 | // xvalue 156 | q.emplace(std::unique_ptr(new int(1))); 157 | (void)q.try_emplace(std::unique_ptr(new int(1))); 158 | q.push(std::unique_ptr(new int(1))); 159 | (void)q.try_push(std::unique_ptr(new int(1))); 160 | auto v = std::unique_ptr(new int(1)); 161 | static_assert(noexcept(q.emplace(std::move(v))) == true, ""); 162 | static_assert(noexcept(q.try_emplace(std::move(v))) == true, ""); 163 | static_assert(noexcept(q.push(std::move(v))) == true, ""); 164 | static_assert(noexcept(q.try_push(std::move(v))) == true, ""); 165 | } 166 | 167 | // capacity < 1 168 | { 169 | SPSCQueue q(0); 170 | assert(q.capacity() == 1); 171 | } 172 | 173 | // Check that padding doesn't overflow capacity 174 | { 175 | bool throws = false; 176 | try { 177 | SPSCQueue q(SIZE_MAX - 1); 178 | } catch (...) { 179 | throws = true; 180 | } 181 | assert(throws); 182 | } 183 | 184 | // Fuzz and performance test 185 | { 186 | const size_t iter = 100000; 187 | SPSCQueue q(iter / 1000 + 1); 188 | std::atomic flag(false); 189 | std::thread producer([&] { 190 | while (!flag) 191 | ; 192 | for (size_t i = 0; i < iter; ++i) { 193 | q.emplace(i); 194 | } 195 | }); 196 | 197 | size_t sum = 0; 198 | auto start = std::chrono::system_clock::now(); 199 | flag = true; 200 | for (size_t i = 0; i < iter; ++i) { 201 | while (!q.front()) 202 | ; 203 | sum += *q.front(); 204 | q.pop(); 205 | } 206 | auto end = std::chrono::system_clock::now(); 207 | auto duration = 208 | std::chrono::duration_cast(end - start); 209 | 210 | assert(q.front() == nullptr); 211 | assert(sum == iter * (iter - 1) / 2); 212 | 213 | producer.join(); 214 | 215 | std::cout << duration.count() / iter << " ns/iter" << std::endl; 216 | } 217 | 218 | return 0; 219 | } 220 | -------------------------------------------------------------------------------- /src/SPSCQueueBenchmark.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2018 Erik Rigtorp 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | SOFTWARE. 21 | */ 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #if __has_include( ) 29 | #include 30 | #endif 31 | 32 | #if __has_include() 33 | #include 34 | #endif 35 | 36 | void pinThread(int cpu) { 37 | if (cpu < 0) { 38 | return; 39 | } 40 | cpu_set_t cpuset; 41 | CPU_ZERO(&cpuset); 42 | CPU_SET(cpu, &cpuset); 43 | if (pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset) == 44 | -1) { 45 | perror("pthread_setaffinity_no"); 46 | exit(1); 47 | } 48 | } 49 | 50 | int main(int argc, char *argv[]) { 51 | (void)argc, (void)argv; 52 | 53 | using namespace rigtorp; 54 | 55 | int cpu1 = -1; 56 | int cpu2 = -1; 57 | 58 | if (argc == 3) { 59 | cpu1 = std::stoi(argv[1]); 60 | cpu2 = std::stoi(argv[2]); 61 | } 62 | 63 | const size_t queueSize = 10000000; 64 | const int64_t iters = 10000000; 65 | 66 | std::cout << "SPSCQueue:" << std::endl; 67 | 68 | { 69 | SPSCQueue q(queueSize); 70 | auto t = std::thread([&] { 71 | pinThread(cpu1); 72 | for (int i = 0; i < iters; ++i) { 73 | while (!q.front()) 74 | ; 75 | if (*q.front() != i) { 76 | throw std::runtime_error(""); 77 | } 78 | q.pop(); 79 | } 80 | }); 81 | 82 | pinThread(cpu2); 83 | 84 | auto start = std::chrono::steady_clock::now(); 85 | for (int i = 0; i < iters; ++i) { 86 | q.emplace(i); 87 | } 88 | t.join(); 89 | auto stop = std::chrono::steady_clock::now(); 90 | std::cout << iters * 1000000 / 91 | std::chrono::duration_cast(stop - 92 | start) 93 | .count() 94 | << " ops/ms" << std::endl; 95 | } 96 | 97 | { 98 | SPSCQueue q1(queueSize), q2(queueSize); 99 | auto t = std::thread([&] { 100 | pinThread(cpu1); 101 | for (int i = 0; i < iters; ++i) { 102 | while (!q1.front()) 103 | ; 104 | q2.emplace(*q1.front()); 105 | q1.pop(); 106 | } 107 | }); 108 | 109 | pinThread(cpu2); 110 | 111 | auto start = std::chrono::steady_clock::now(); 112 | for (int i = 0; i < iters; ++i) { 113 | q1.emplace(i); 114 | while (!q2.front()) 115 | ; 116 | q2.pop(); 117 | } 118 | auto stop = std::chrono::steady_clock::now(); 119 | t.join(); 120 | std::cout << std::chrono::duration_cast(stop - 121 | start) 122 | .count() / 123 | iters 124 | << " ns RTT" << std::endl; 125 | } 126 | 127 | #if __has_include( ) 128 | std::cout << "boost::lockfree::spsc:" << std::endl; 129 | { 130 | boost::lockfree::spsc_queue q(queueSize); 131 | auto t = std::thread([&] { 132 | pinThread(cpu1); 133 | for (int i = 0; i < iters; ++i) { 134 | int val; 135 | while (q.pop(&val, 1) != 1) 136 | ; 137 | if (val != i) { 138 | throw std::runtime_error(""); 139 | } 140 | } 141 | }); 142 | 143 | pinThread(cpu2); 144 | 145 | auto start = std::chrono::steady_clock::now(); 146 | for (int i = 0; i < iters; ++i) { 147 | while (!q.push(i)) 148 | ; 149 | } 150 | t.join(); 151 | auto stop = std::chrono::steady_clock::now(); 152 | std::cout << iters * 1000000 / 153 | std::chrono::duration_cast(stop - 154 | start) 155 | .count() 156 | << " ops/ms" << std::endl; 157 | } 158 | 159 | { 160 | boost::lockfree::spsc_queue q1(queueSize), q2(queueSize); 161 | auto t = std::thread([&] { 162 | pinThread(cpu1); 163 | for (int i = 0; i < iters; ++i) { 164 | int val; 165 | while (q1.pop(&val, 1) != 1) 166 | ; 167 | while (!q2.push(val)) 168 | ; 169 | } 170 | }); 171 | 172 | pinThread(cpu2); 173 | 174 | auto start = std::chrono::steady_clock::now(); 175 | for (int i = 0; i < iters; ++i) { 176 | while (!q1.push(i)) 177 | ; 178 | int val; 179 | while (q2.pop(&val, 1) != 1) 180 | ; 181 | } 182 | auto stop = std::chrono::steady_clock::now(); 183 | t.join(); 184 | std::cout << std::chrono::duration_cast(stop - 185 | start) 186 | .count() / 187 | iters 188 | << " ns RTT" << std::endl; 189 | } 190 | #endif 191 | 192 | #if __has_include() 193 | std::cout << "folly::ProducerConsumerQueue:" << std::endl; 194 | 195 | { 196 | folly::ProducerConsumerQueue q(queueSize); 197 | auto t = std::thread([&] { 198 | pinThread(cpu1); 199 | for (int i = 0; i < iters; ++i) { 200 | int val; 201 | while (!q.read(val)) 202 | ; 203 | if (val != i) { 204 | throw std::runtime_error(""); 205 | } 206 | } 207 | }); 208 | 209 | pinThread(cpu2); 210 | 211 | auto start = std::chrono::steady_clock::now(); 212 | for (int i = 0; i < iters; ++i) { 213 | while (!q.write(i)) 214 | ; 215 | } 216 | t.join(); 217 | auto stop = std::chrono::steady_clock::now(); 218 | std::cout << iters * 1000000 / 219 | std::chrono::duration_cast(stop - 220 | start) 221 | .count() 222 | << " ops/ms" << std::endl; 223 | } 224 | 225 | { 226 | folly::ProducerConsumerQueue q1(queueSize), q2(queueSize); 227 | auto t = std::thread([&] { 228 | pinThread(cpu1); 229 | for (int i = 0; i < iters; ++i) { 230 | int val; 231 | while (!q1.read(val)) 232 | ; 233 | q2.write(val); 234 | } 235 | }); 236 | 237 | pinThread(cpu2); 238 | 239 | auto start = std::chrono::steady_clock::now(); 240 | for (int i = 0; i < iters; ++i) { 241 | while (!q1.write(i)) 242 | ; 243 | int val; 244 | while (!q2.read(val)) 245 | ; 246 | } 247 | auto stop = std::chrono::steady_clock::now(); 248 | t.join(); 249 | std::cout << std::chrono::duration_cast(stop - 250 | start) 251 | .count() / 252 | iters 253 | << " ns RTT" << std::endl; 254 | } 255 | #endif 256 | 257 | return 0; 258 | } 259 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SPSCQueue.h 2 | 3 | [![C/C++ CI](https://github.com/rigtorp/SPSCQueue/workflows/C/C++%20CI/badge.svg)](https://github.com/rigtorp/SPSCQueue/actions) 4 | [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/rigtorp/SPSCQueue/master/LICENSE) 5 | 6 | A single producer single consumer wait-free and lock-free fixed size queue 7 | written in C++11. This implementation is faster than both 8 | [*boost::lockfree::spsc*](https://www.boost.org/doc/libs/1_76_0/doc/html/boost/lockfree/spsc_queue.html) 9 | and [*folly::ProducerConsumerQueue*](https://github.com/facebook/folly/blob/master/folly/docs/ProducerConsumerQueue.md). 10 | 11 | ## Example 12 | 13 | ```cpp 14 | SPSCQueue q(1); 15 | auto t = std::thread([&] { 16 | while (!q.front()); 17 | std::cout << *q.front() << std::endl; 18 | q.pop(); 19 | }); 20 | q.push(1); 21 | t.join(); 22 | ``` 23 | 24 | See `src/SPSCQueueExample.cpp` for the full example. 25 | 26 | ## Usage 27 | 28 | - `SPSCQueue(size_t capacity);` 29 | 30 | Create a `SPSCqueue` holding items of type `T` with capacity 31 | `capacity`. Capacity needs to be at least 1. 32 | 33 | - `void emplace(Args &&... args);` 34 | 35 | Enqueue an item using inplace construction. Blocks if queue is full. 36 | 37 | - `bool try_emplace(Args &&... args);` 38 | 39 | Try to enqueue an item using inplace construction. Returns `true` on 40 | success and `false` if queue is full. 41 | 42 | - `void push(const T &v);` 43 | 44 | Enqueue an item using copy construction. Blocks if queue is full. 45 | 46 | - `template void push(P &&v);` 47 | 48 | Enqueue an item using move construction. Participates in overload 49 | resolution only if `std::is_constructible::value == true`. 50 | Blocks if queue is full. 51 | 52 | - `bool try_push(const T &v);` 53 | 54 | Try to enqueue an item using copy construction. Returns `true` on 55 | success and `false` if queue is full. 56 | 57 | - `template bool try_push(P &&v);` 58 | 59 | Try to enqueue an item using move construction. Returns `true` on 60 | success and `false` if queue is full. Participates in overload 61 | resolution only if `std::is_constructible::value == true`. 62 | 63 | - `T *front();` 64 | 65 | Return pointer to front of queue. Returns `nullptr` if queue is 66 | empty. 67 | 68 | - `void pop();` 69 | 70 | Dequeue first item of queue. You must ensure that the queue is non-empty 71 | before calling pop. This means that `front()` must have returned a 72 | non-`nullptr` before each call to `pop()`. Requires 73 | `std::is_nothrow_destructible::value == true`. 74 | 75 | - `size_t size();` 76 | 77 | Return the number of items available in the queue. 78 | 79 | - `bool empty();` 80 | 81 | Return true if queue is currently empty. 82 | 83 | Only a single writer thread can perform enqueue operations and only a 84 | single reader thread can perform dequeue operations. Any other usage 85 | is invalid. 86 | 87 | ## Huge page support 88 | 89 | In addition to supporting custom allocation through the [standard custom 90 | allocator interface](https://en.cppreference.com/w/cpp/named_req/Allocator) this 91 | library also supports standard proposal [P0401R3 Providing size feedback in the 92 | Allocator 93 | interface](http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p0401r3.html). 94 | This allows convenient use of [huge 95 | pages](https://www.kernel.org/doc/html/latest/admin-guide/mm/hugetlbpage.html) 96 | without wasting any allocated space. Using size feedback is only supported when 97 | C++17 is enabled. 98 | 99 | The library currently doesn't include a huge page allocator since the APIs for 100 | allocating huge pages are platform dependent and handling of huge page size and 101 | NUMA awareness is application specific. 102 | 103 | Below is an example huge page allocator for Linux: 104 | 105 | ```cpp 106 | #include 107 | 108 | template struct Allocator { 109 | using value_type = T; 110 | 111 | struct AllocationResult { 112 | T *ptr; 113 | size_t count; 114 | }; 115 | 116 | size_t roundup(size_t n) { return (((n - 1) >> 21) + 1) << 21; } 117 | 118 | AllocationResult allocate_at_least(size_t n) { 119 | size_t count = roundup(sizeof(T) * n); 120 | auto p = static_cast(mmap(nullptr, count, PROT_READ | PROT_WRITE, 121 | MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, 122 | -1, 0)); 123 | if (p == MAP_FAILED) { 124 | throw std::bad_alloc(); 125 | } 126 | return {p, count / sizeof(T)}; 127 | } 128 | 129 | void deallocate(T *p, size_t n) { munmap(p, roundup(sizeof(T) * n)); } 130 | }; 131 | ``` 132 | 133 | See `src/SPSCQueueExampleHugepages.cpp` for the full example on how to use huge 134 | pages on Linux. 135 | 136 | ## Implementation 137 | 138 | ![Memory layout](https://github.com/rigtorp/SPSCQueue/blob/master/spsc.svg) 139 | 140 | The underlying implementation is based on a [ring 141 | buffer](https://en.wikipedia.org/wiki/Circular_buffer). 142 | 143 | Care has been taken to make sure to avoid any issues with [false 144 | sharing](https://en.wikipedia.org/wiki/False_sharing). The head and tail indices 145 | are aligned and padded to the false sharing range (cache line size). 146 | Additionally the slots buffer is padded with the false sharing range at the 147 | beginning and end, this prevents false sharing with any adjacent allocations. 148 | 149 | This implementation has higher throughput than a typical concurrent ring buffer 150 | by locally caching the head and tail indices in the writer and reader 151 | respectively. The caching increases throughput by reducing the amount of cache 152 | coherency traffic. 153 | 154 | To understand how that works first consider a read operation in absence of 155 | caching: the head index (read index) needs to be updated and thus that cache 156 | line is loaded into the L1 cache in exclusive state. The tail (write index) 157 | needs to be read in order to check that the queue is not empty and is thus 158 | loaded into the L1 cache in shared state. Since a queue write operation needs to 159 | read the head index it's likely that a write operation requires some cache 160 | coherency traffic to bring the head index cache line back into exclusive state. 161 | In the worst case there will be one cache line transition from shared to 162 | exclusive for every read and write operation. 163 | 164 | Next consider a queue reader that caches the tail index: if the cached tail 165 | index indicates that the queue is empty, then load the tail index into the 166 | cached tail index. If the queue was non-empty multiple read operations up until 167 | the cached tail index can complete without stealing the writer's tail index 168 | cache line's exclusive state. Cache coherency traffic is therefore reduced. An 169 | analogous argument can be made for the queue write operation. 170 | 171 | This implementation allows for arbitrary non-power of two capacities, instead 172 | allocating a extra queue slot to indicate full queue. If you don't want to waste 173 | storage for a extra queue slot you should use a different implementation. 174 | 175 | References: 176 | 177 | - *Intel*. [Avoiding and Identifying False Sharing Among Threads](https://software.intel.com/en-us/articles/avoiding-and-identifying-false-sharing-among-threads). 178 | - *Wikipedia*. [Ring buffer](https://en.wikipedia.org/wiki/Circular_buffer). 179 | - *Wikipedia*. [False sharing](https://en.wikipedia.org/wiki/False_sharing). 180 | 181 | ## Testing 182 | 183 | Testing lock-free algorithms is hard. I'm using two approaches to test 184 | the implementation: 185 | 186 | - A single threaded test that the functionality works as intended, 187 | including that the item constructor and destructor is invoked 188 | correctly. 189 | - A multi-threaded fuzz test verifies that all items are enqueued and dequeued 190 | correctly under heavy contention. 191 | 192 | ## Benchmarks 193 | 194 | Throughput benchmark measures throughput between 2 threads for a queue of `int` 195 | items. 196 | 197 | Latency benchmark measures round trip time between 2 threads communicating using 198 | 2 queues of `int` items. 199 | 200 | Benchmark results for a AMD Ryzen 9 3900X 12-Core Processor, the 2 threads are 201 | running on different cores on the same chiplet: 202 | 203 | | Queue | Throughput (ops/ms) | Latency RTT (ns) | 204 | | ---------------------------- | ------------------: | ---------------: | 205 | | SPSCQueue | 362723 | 133 | 206 | | boost::lockfree::spsc | 209877 | 222 | 207 | | folly::ProducerConsumerQueue | 148818 | 147 | 208 | 209 | ## Cited by 210 | 211 | SPSCQueue have been cited by the following papers: 212 | 213 | - Peizhao Ou and Brian Demsky. 2018. Towards understanding the costs of avoiding 214 | out-of-thin-air results. Proc. ACM Program. Lang. 2, OOPSLA, Article 136 215 | (October 2018), 29 pages. DOI: 216 | 217 | ## About 218 | 219 | This project was created by [Erik Rigtorp](http://rigtorp.se) 220 | <[erik@rigtorp.se](mailto:erik@rigtorp.se)>. 221 | -------------------------------------------------------------------------------- /include/rigtorp/SPSCQueue.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2020 Erik Rigtorp 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | SOFTWARE. 21 | */ 22 | 23 | #pragma once 24 | 25 | #include 26 | #include 27 | #include 28 | #include // std::allocator 29 | #include // std::hardware_destructive_interference_size 30 | #include 31 | #include // std::enable_if, std::is_*_constructible 32 | 33 | #ifdef __has_cpp_attribute 34 | #if __has_cpp_attribute(nodiscard) 35 | #define RIGTORP_NODISCARD [[nodiscard]] 36 | #endif 37 | #endif 38 | #ifndef RIGTORP_NODISCARD 39 | #define RIGTORP_NODISCARD 40 | #endif 41 | 42 | namespace rigtorp { 43 | 44 | template > class SPSCQueue { 45 | 46 | #if defined(__cpp_if_constexpr) && defined(__cpp_lib_void_t) 47 | template 48 | struct has_allocate_at_least : std::false_type {}; 49 | 50 | template 51 | struct has_allocate_at_least< 52 | Alloc2, std::void_t().allocate_at_least( 54 | size_t{}))>> : std::true_type {}; 55 | #endif 56 | 57 | public: 58 | explicit SPSCQueue(const size_t capacity, 59 | const Allocator &allocator = Allocator()) 60 | : capacity_(capacity), allocator_(allocator) { 61 | // The queue needs at least one element 62 | if (capacity_ < 1) { 63 | capacity_ = 1; 64 | } 65 | capacity_++; // Needs one slack element 66 | // Prevent overflowing size_t 67 | if (capacity_ > SIZE_MAX - 2 * kPadding) { 68 | capacity_ = SIZE_MAX - 2 * kPadding; 69 | } 70 | 71 | #if defined(__cpp_if_constexpr) && defined(__cpp_lib_void_t) 72 | if constexpr (has_allocate_at_least::value) { 73 | auto res = allocator_.allocate_at_least(capacity_ + 2 * kPadding); 74 | slots_ = res.ptr; 75 | capacity_ = res.count - 2 * kPadding; 76 | } else { 77 | slots_ = std::allocator_traits::allocate( 78 | allocator_, capacity_ + 2 * kPadding); 79 | } 80 | #else 81 | slots_ = std::allocator_traits::allocate( 82 | allocator_, capacity_ + 2 * kPadding); 83 | #endif 84 | 85 | static_assert(alignof(SPSCQueue) == kCacheLineSize, ""); 86 | static_assert(sizeof(SPSCQueue) >= 3 * kCacheLineSize, ""); 87 | assert(reinterpret_cast(&readIdx_) - 88 | reinterpret_cast(&writeIdx_) >= 89 | static_cast(kCacheLineSize)); 90 | } 91 | 92 | ~SPSCQueue() { 93 | while (front()) { 94 | pop(); 95 | } 96 | std::allocator_traits::deallocate(allocator_, slots_, 97 | capacity_ + 2 * kPadding); 98 | } 99 | 100 | // non-copyable and non-movable 101 | SPSCQueue(const SPSCQueue &) = delete; 102 | SPSCQueue &operator=(const SPSCQueue &) = delete; 103 | 104 | template 105 | void emplace(Args &&...args) noexcept( 106 | std::is_nothrow_constructible::value) { 107 | static_assert(std::is_constructible::value, 108 | "T must be constructible with Args&&..."); 109 | auto const writeIdx = writeIdx_.load(std::memory_order_relaxed); 110 | auto nextWriteIdx = writeIdx + 1; 111 | if (nextWriteIdx == capacity_) { 112 | nextWriteIdx = 0; 113 | } 114 | while (nextWriteIdx == readIdxCache_) { 115 | readIdxCache_ = readIdx_.load(std::memory_order_acquire); 116 | } 117 | new (&slots_[writeIdx + kPadding]) T(std::forward(args)...); 118 | writeIdx_.store(nextWriteIdx, std::memory_order_release); 119 | } 120 | 121 | template 122 | RIGTORP_NODISCARD bool try_emplace(Args &&...args) noexcept( 123 | std::is_nothrow_constructible::value) { 124 | static_assert(std::is_constructible::value, 125 | "T must be constructible with Args&&..."); 126 | auto const writeIdx = writeIdx_.load(std::memory_order_relaxed); 127 | auto nextWriteIdx = writeIdx + 1; 128 | if (nextWriteIdx == capacity_) { 129 | nextWriteIdx = 0; 130 | } 131 | if (nextWriteIdx == readIdxCache_) { 132 | readIdxCache_ = readIdx_.load(std::memory_order_acquire); 133 | if (nextWriteIdx == readIdxCache_) { 134 | return false; 135 | } 136 | } 137 | new (&slots_[writeIdx + kPadding]) T(std::forward(args)...); 138 | writeIdx_.store(nextWriteIdx, std::memory_order_release); 139 | return true; 140 | } 141 | 142 | void push(const T &v) noexcept(std::is_nothrow_copy_constructible::value) { 143 | static_assert(std::is_copy_constructible::value, 144 | "T must be copy constructible"); 145 | emplace(v); 146 | } 147 | 148 | template ::value>::type> 150 | void push(P &&v) noexcept(std::is_nothrow_constructible::value) { 151 | emplace(std::forward

(v)); 152 | } 153 | 154 | RIGTORP_NODISCARD bool 155 | try_push(const T &v) noexcept(std::is_nothrow_copy_constructible::value) { 156 | static_assert(std::is_copy_constructible::value, 157 | "T must be copy constructible"); 158 | return try_emplace(v); 159 | } 160 | 161 | template ::value>::type> 163 | RIGTORP_NODISCARD bool 164 | try_push(P &&v) noexcept(std::is_nothrow_constructible::value) { 165 | return try_emplace(std::forward

(v)); 166 | } 167 | 168 | RIGTORP_NODISCARD T *front() noexcept { 169 | auto const readIdx = readIdx_.load(std::memory_order_relaxed); 170 | if (readIdx == writeIdxCache_) { 171 | writeIdxCache_ = writeIdx_.load(std::memory_order_acquire); 172 | if (writeIdxCache_ == readIdx) { 173 | return nullptr; 174 | } 175 | } 176 | return &slots_[readIdx + kPadding]; 177 | } 178 | 179 | void pop() noexcept { 180 | static_assert(std::is_nothrow_destructible::value, 181 | "T must be nothrow destructible"); 182 | auto const readIdx = readIdx_.load(std::memory_order_relaxed); 183 | assert(writeIdx_.load(std::memory_order_acquire) != readIdx && 184 | "Can only call pop() after front() has returned a non-nullptr"); 185 | slots_[readIdx + kPadding].~T(); 186 | auto nextReadIdx = readIdx + 1; 187 | if (nextReadIdx == capacity_) { 188 | nextReadIdx = 0; 189 | } 190 | readIdx_.store(nextReadIdx, std::memory_order_release); 191 | } 192 | 193 | RIGTORP_NODISCARD size_t size() const noexcept { 194 | std::ptrdiff_t diff = writeIdx_.load(std::memory_order_acquire) - 195 | readIdx_.load(std::memory_order_acquire); 196 | if (diff < 0) { 197 | diff += capacity_; 198 | } 199 | return static_cast(diff); 200 | } 201 | 202 | RIGTORP_NODISCARD bool empty() const noexcept { 203 | return writeIdx_.load(std::memory_order_acquire) == 204 | readIdx_.load(std::memory_order_acquire); 205 | } 206 | 207 | RIGTORP_NODISCARD size_t capacity() const noexcept { return capacity_ - 1; } 208 | 209 | private: 210 | #ifdef __cpp_lib_hardware_interference_size 211 | static constexpr size_t kCacheLineSize = 212 | std::hardware_destructive_interference_size; 213 | #else 214 | static constexpr size_t kCacheLineSize = 64; 215 | #endif 216 | 217 | // Padding to avoid false sharing between slots_ and adjacent allocations 218 | static constexpr size_t kPadding = (kCacheLineSize - 1) / sizeof(T) + 1; 219 | 220 | private: 221 | size_t capacity_; 222 | T *slots_; 223 | #if defined(__has_cpp_attribute) && __has_cpp_attribute(no_unique_address) 224 | Allocator allocator_ [[no_unique_address]]; 225 | #else 226 | Allocator allocator_; 227 | #endif 228 | 229 | // Align to cache line size in order to avoid false sharing 230 | // readIdxCache_ and writeIdxCache_ is used to reduce the amount of cache 231 | // coherency traffic 232 | alignas(kCacheLineSize) std::atomic writeIdx_ = {0}; 233 | alignas(kCacheLineSize) size_t readIdxCache_ = 0; 234 | alignas(kCacheLineSize) std::atomic readIdx_ = {0}; 235 | alignas(kCacheLineSize) size_t writeIdxCache_ = 0; 236 | }; 237 | } // namespace rigtorp 238 | -------------------------------------------------------------------------------- /spsc.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | Capacity 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | Slots 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | Pad 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | Head 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | Pad 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | Tail 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | Pad 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | False sharing range 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | False sharing range 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | Pad 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | Pad 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | T 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | T 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | T 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | T 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | T 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | False sharing range 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | False sharing range 334 | 335 | 336 | 337 | 338 | 339 | 340 | 341 | 342 | 343 | 344 | 345 | 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | Pad 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | 373 | 374 | 375 | 376 | 377 | Pad 378 | 379 | 380 | 381 | 382 | 383 | 384 | 385 | HeadCache 386 | 387 | 388 | 389 | 390 | 391 | 392 | 393 | 394 | 395 | 396 | 397 | 398 | 399 | 400 | 401 | 402 | 403 | 404 | 405 | 406 | 407 | Pad 408 | 409 | 410 | 411 | 412 | 413 | 414 | 415 | TailCache 416 | 417 | 418 | 419 | 420 | 421 | 422 | 423 | 424 | 425 | 426 | 427 | 428 | 429 | 430 | 431 | 432 | 433 | 434 | 435 | 436 | 437 | Pad 438 | 439 | 440 | 441 | 442 | 443 | 444 | 445 | 446 | 447 | 448 | False sharing range 449 | 450 | 451 | 452 | 453 | 454 | 455 | 456 | 457 | 458 | 459 | False sharing range 460 | 461 | 462 | 463 | 464 | 465 | 466 | 467 | --------------------------------------------------------------------------------