├── .bazelignore ├── .bazelrc ├── .gitignore ├── BUILD.bazel ├── COPYRIGHT ├── LICENSE ├── MODULE.bazel ├── MODULE.bazel.lock ├── README.md ├── WORKSPACE ├── bzl ├── BUILD.bazel ├── local_rules │ ├── BUILD.bazel │ └── pybind.bzl └── third_party │ ├── BUILD.bazel │ └── pybind11.BUILD ├── client ├── BUILD.bazel ├── client.cc ├── client.h ├── client_channel.h ├── client_test.cc ├── options.h └── python │ ├── BUILD.bazel │ ├── client.cc │ └── client_test.py ├── common ├── BUILD.bazel ├── channel.cc └── channel.h ├── docs └── subspace.pdf ├── manual_tests ├── BUILD.bazel ├── perf_subspace.cc ├── perf_subspace_pub.cc ├── perf_subspace_sub.cc ├── perf_tcp_recv.cc ├── perf_tcp_send.cc ├── perf_test.sh ├── pub.cc └── sub.cc ├── proto ├── BUILD.bazel └── subspace.proto └── server ├── BUILD.bazel ├── client_handler.cc ├── client_handler.h ├── main.cc ├── server.cc ├── server.h ├── server_channel.cc └── server_channel.h /.bazelignore: -------------------------------------------------------------------------------- 1 | bazel-ipc 2 | -------------------------------------------------------------------------------- /.bazelrc: -------------------------------------------------------------------------------- 1 | build:asan --strip=never 2 | build:asan --copt -fsanitize=address 3 | build:asan --copt -DADDRESS_SANITIZER 4 | build:asan --copt -g 5 | build:asan --copt -fno-omit-frame-pointer 6 | build:asan --linkopt -fsanitize=address 7 | 8 | # For all builds, use C++17 9 | build --cxxopt="-std=c++17" 10 | build --cxxopt='-Wno-sign-compare' 11 | 12 | # For Apple Silicon 13 | build:apple_silicon --cpu=darwin_arm64 14 | 15 | # Common flags for Clang 16 | build:clang --action_env=BAZEL_COMPILER=clang 17 | build:clang --action_env=CC=clang --action_env=CXX=clang++ 18 | #build:clang --linkopt=-fuse-ld=lld 19 | 20 | # Clang with libc++ 21 | build:libc++ --config=clang 22 | build:libc++ --action_env=CXXFLAGS=-stdlib=libc++ 23 | build:libc++ --action_env=LDFLAGS=-stdlib=libc++ 24 | build:libc++ --action_env=BAZEL_CXXOPTS=-stdlib=libc++ 25 | build:libc++ --action_env=BAZEL_LINKLIBS=-l%:libc++.a:-l%:libc++abi.a 26 | build:libc++ --action_env=BAZEL_LINKOPTS=-lm:-pthread 27 | build:libc++ --define force_libcpp=enabled 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bazel-* 2 | .vscode/* 3 | -------------------------------------------------------------------------------- /BUILD.bazel: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | config_setting( 4 | name = "macos_arm64", 5 | values = {"cpu": "darwin_arm64"}, 6 | ) 7 | 8 | config_setting( 9 | name = "macos_default", 10 | values = {"cpu": "darwin"}, 11 | ) 12 | 13 | config_setting( 14 | name = "macos_x86_64", 15 | values = {"cpu": "darwin_x86_64"}, 16 | ) 17 | 18 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Copyright 2023 David Allison 2 | All Rights Reserved 3 | 4 | This is licensed under the Apache 2 license (see the LICENSE file). 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /MODULE.bazel: -------------------------------------------------------------------------------- 1 | module( 2 | name = "subspace", 3 | ) 4 | 5 | http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 6 | 7 | bazel_dep(name = "bazel_skylib", version = "1.7.1") 8 | bazel_dep(name = "platforms", version = "0.0.10") 9 | bazel_dep(name = "abseil-cpp", version = "20240722.0.bcr.1", repo_name = "com_google_absl") 10 | bazel_dep(name = "googletest", version = "1.15.2", repo_name = "com_google_googletest") 11 | bazel_dep(name = "protobuf", version = "29.0", repo_name = "com_google_protobuf") 12 | 13 | bazel_dep(name = "rules_cc", version = "0.0.16") 14 | bazel_dep(name = "rules_pkg", version = "1.0.1") 15 | bazel_dep(name = "zlib", version = "1.3.1.bcr.3") 16 | 17 | # Toolbelt 18 | http_archive( 19 | name = "toolbelt", 20 | integrity = "sha256-vlRb0hMtip/JDPWbK6YIXnds8HjrMAoc/iZjQ/BX9Vc=", 21 | strip_prefix = "cpp_toolbelt-1.1.7", 22 | urls = ["https://github.com/dallison/cpp_toolbelt/archive/refs/tags/1.1.7.tar.gz"], 23 | ) 24 | # For local debugging of toolbelt coroutine library. 25 | #bazel_dep(name = "toolbelt") 26 | #local_path_override( 27 | # module_name = "toolbelt", 28 | # path = "../cpp_toolbelt", 29 | #) 30 | 31 | # Coroutines 32 | http_archive( 33 | name = "coroutines", 34 | integrity = "sha256-cJ3a89VebabjRgLjHNsEsjIQE+hi+5vdtuAh4RfTXCI=", 35 | strip_prefix = "co-1.3.7", 36 | urls = ["https://github.com/dallison/co/archive/refs/tags/1.3.7.tar.gz"], 37 | ) 38 | # For local debugging of co coroutine library. 39 | #bazel_dep(name = "coroutines") 40 | #local_path_override( 41 | # module_name = "coroutines", 42 | # path = "../co", 43 | #) 44 | 45 | bazel_dep(name = "rules_python", version = "0.40.0") 46 | 47 | python = use_extension("@rules_python//python/extensions:python.bzl", "python") 48 | python.toolchain( 49 | configure_coverage_tool = True, 50 | is_default = True, 51 | python_version = "3.11.1", 52 | ) 53 | 54 | bazel_dep(name = "pybind11_bazel", version = "2.13.6") 55 | -------------------------------------------------------------------------------- /MODULE.bazel.lock: -------------------------------------------------------------------------------- 1 | { 2 | "lockFileVersion": 11, 3 | "registryFileHashes": { 4 | "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", 5 | "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", 6 | "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", 7 | "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0", 8 | "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", 9 | "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", 10 | "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", 11 | "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", 12 | "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", 13 | "https://bcr.bazel.build/modules/abseil-cpp/20240722.0.bcr.1/MODULE.bazel": "c0aa5eaefff1121b40208397f229604c717bd2fdf214ff67586d627118e17720", 14 | "https://bcr.bazel.build/modules/abseil-cpp/20240722.0.bcr.1/source.json": "e067fdd217bacbe74c88a975434be5df0b44315a247be180f0e20f891715210c", 15 | "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", 16 | "https://bcr.bazel.build/modules/apple_support/1.15.1/source.json": "517f2b77430084c541bc9be2db63fdcbb7102938c5f64c17ee60ffda2e5cf07b", 17 | "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef", 18 | "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", 19 | "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", 20 | "https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d", 21 | "https://bcr.bazel.build/modules/bazel_features/1.17.0/MODULE.bazel": "039de32d21b816b47bd42c778e0454217e9c9caac4a3cf8e15c7231ee3ddee4d", 22 | "https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a", 23 | "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58", 24 | "https://bcr.bazel.build/modules/bazel_features/1.19.0/source.json": "d7bf14517c1b25b9d9c580b0f8795fceeae08a7590f507b76aace528e941375d", 25 | "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", 26 | "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", 27 | "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", 28 | "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", 29 | "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", 30 | "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", 31 | "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", 32 | "https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d", 33 | "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", 34 | "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", 35 | "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", 36 | "https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d", 37 | "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", 38 | "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953", 39 | "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", 40 | "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", 41 | "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", 42 | "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", 43 | "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", 44 | "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", 45 | "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", 46 | "https://bcr.bazel.build/modules/googletest/1.15.2/source.json": "dbdda654dcb3a0d7a8bc5d0ac5fc7e150b58c2a986025ae5bc634bb2cb61f470", 47 | "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", 48 | "https://bcr.bazel.build/modules/jsoncpp/1.9.5/source.json": "4108ee5085dd2885a341c7fab149429db457b3169b86eb081fa245eadf69169d", 49 | "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", 50 | "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", 51 | "https://bcr.bazel.build/modules/platforms/0.0.10/source.json": "f22828ff4cf021a6b577f1bf6341cb9dcd7965092a439f64fc1bb3b7a5ae4bd5", 52 | "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", 53 | "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37", 54 | "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615", 55 | "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814", 56 | "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", 57 | "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", 58 | "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", 59 | "https://bcr.bazel.build/modules/protobuf/23.1/MODULE.bazel": "88b393b3eb4101d18129e5db51847cd40a5517a53e81216144a8c32dfeeca52a", 60 | "https://bcr.bazel.build/modules/protobuf/24.4/MODULE.bazel": "7bc7ce5f2abf36b3b7b7c8218d3acdebb9426aeb35c2257c96445756f970eb12", 61 | "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c", 62 | "https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d", 63 | "https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df", 64 | "https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e", 65 | "https://bcr.bazel.build/modules/protobuf/29.0/source.json": "b857f93c796750eef95f0d61ee378f3420d00ee1dd38627b27193aa482f4f981", 66 | "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", 67 | "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", 68 | "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", 69 | "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", 70 | "https://bcr.bazel.build/modules/pybind11_bazel/2.13.6/MODULE.bazel": "2d746fda559464b253b2b2e6073cb51643a2ac79009ca02100ebbc44b4548656", 71 | "https://bcr.bazel.build/modules/pybind11_bazel/2.13.6/source.json": "6aa0703de8efb20cc897bbdbeb928582ee7beaf278bcd001ac253e1605bddfae", 72 | "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206", 73 | "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", 74 | "https://bcr.bazel.build/modules/re2/2024-07-02/source.json": "547d0111a9d4f362db32196fef805abbf3676e8d6afbe44d395d87816c1130ca", 75 | "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", 76 | "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", 77 | "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", 78 | "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", 79 | "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", 80 | "https://bcr.bazel.build/modules/rules_cc/0.0.14/MODULE.bazel": "5e343a3aac88b8d7af3b1b6d2093b55c347b8eefc2e7d1442f7a02dc8fea48ac", 81 | "https://bcr.bazel.build/modules/rules_cc/0.0.15/MODULE.bazel": "6704c35f7b4a72502ee81f61bf88706b54f06b3cbe5558ac17e2e14666cd5dcc", 82 | "https://bcr.bazel.build/modules/rules_cc/0.0.16/MODULE.bazel": "7661303b8fc1b4d7f532e54e9d6565771fea666fbdf839e0a86affcd02defe87", 83 | "https://bcr.bazel.build/modules/rules_cc/0.0.16/source.json": "227e83737046aa4f50015da48e98e0d8ab42fd0ec74d8d653b6cc9f9a357f200", 84 | "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", 85 | "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f", 86 | "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", 87 | "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", 88 | "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", 89 | "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", 90 | "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/source.json": "c8b1e2c717646f1702290959a3302a178fb639d987ab61d548105019f11e527e", 91 | "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", 92 | "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", 93 | "https://bcr.bazel.build/modules/rules_java/6.0.0/MODULE.bazel": "8a43b7df601a7ec1af61d79345c17b31ea1fedc6711fd4abfd013ea612978e39", 94 | "https://bcr.bazel.build/modules/rules_java/6.4.0/MODULE.bazel": "e986a9fe25aeaa84ac17ca093ef13a4637f6107375f64667a15999f77db6c8f6", 95 | "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31", 96 | "https://bcr.bazel.build/modules/rules_java/7.1.0/MODULE.bazel": "30d9135a2b6561c761bd67bd4990da591e6bdc128790ce3e7afd6a3558b2fb64", 97 | "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a", 98 | "https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6", 99 | "https://bcr.bazel.build/modules/rules_java/7.12.2/source.json": "b0890f9cda8ff1b8e691a3ac6037b5c14b7fd4134765a3946b89f31ea02e5884", 100 | "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab", 101 | "https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2", 102 | "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", 103 | "https://bcr.bazel.build/modules/rules_java/7.6.5/MODULE.bazel": "481164be5e02e4cab6e77a36927683263be56b7e36fef918b458d7a8a1ebadb1", 104 | "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", 105 | "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", 106 | "https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036", 107 | "https://bcr.bazel.build/modules/rules_jvm_external/5.3/MODULE.bazel": "bf93870767689637164657731849fb887ad086739bd5d360d90007a581d5527d", 108 | "https://bcr.bazel.build/modules/rules_jvm_external/6.1/MODULE.bazel": "75b5fec090dbd46cf9b7d8ea08cf84a0472d92ba3585b476f44c326eda8059c4", 109 | "https://bcr.bazel.build/modules/rules_jvm_external/6.3/MODULE.bazel": "c998e060b85f71e00de5ec552019347c8bca255062c990ac02d051bb80a38df0", 110 | "https://bcr.bazel.build/modules/rules_jvm_external/6.3/source.json": "6f5f5a5a4419ae4e37c35a5bb0a6ae657ed40b7abc5a5189111b47fcebe43197", 111 | "https://bcr.bazel.build/modules/rules_kotlin/1.9.0/MODULE.bazel": "ef85697305025e5a61f395d4eaede272a5393cee479ace6686dba707de804d59", 112 | "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/MODULE.bazel": "d269a01a18ee74d0335450b10f62c9ed81f2321d7958a2934e44272fe82dcef3", 113 | "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/source.json": "2faa4794364282db7c06600b7e5e34867a564ae91bda7cae7c29c64e9466b7d5", 114 | "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", 115 | "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", 116 | "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", 117 | "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", 118 | "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", 119 | "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", 120 | "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", 121 | "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", 122 | "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", 123 | "https://bcr.bazel.build/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "1e5b502e2e1a9e825eef74476a5a1ee524a92297085015a052510b09a1a09483", 124 | "https://bcr.bazel.build/modules/rules_proto/6.0.2/MODULE.bazel": "ce916b775a62b90b61888052a416ccdda405212b6aaeb39522f7dc53431a5e73", 125 | "https://bcr.bazel.build/modules/rules_proto/7.0.2/MODULE.bazel": "bf81793bd6d2ad89a37a40693e56c61b0ee30f7a7fdbaf3eabbf5f39de47dea2", 126 | "https://bcr.bazel.build/modules/rules_proto/7.0.2/source.json": "1e5e7260ae32ef4f2b52fd1d0de8d03b606a44c91b694d2f1afb1d3b28a48ce1", 127 | "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", 128 | "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7", 129 | "https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300", 130 | "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382", 131 | "https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed", 132 | "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58", 133 | "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", 134 | "https://bcr.bazel.build/modules/rules_python/0.34.0/MODULE.bazel": "1d623d026e075b78c9fde483a889cda7996f5da4f36dffb24c246ab30f06513a", 135 | "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", 136 | "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", 137 | "https://bcr.bazel.build/modules/rules_python/0.40.0/source.json": "939d4bd2e3110f27bfb360292986bb79fd8dcefb874358ccd6cdaa7bda029320", 138 | "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", 139 | "https://bcr.bazel.build/modules/rules_shell/0.2.0/source.json": "7f27af3c28037d9701487c4744b5448d26537cc66cdef0d8df7ae85411f8de95", 140 | "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", 141 | "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", 142 | "https://bcr.bazel.build/modules/stardoc/0.5.6/MODULE.bazel": "c43dabc564990eeab55e25ed61c07a1aadafe9ece96a4efabb3f8bf9063b71ef", 143 | "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", 144 | "https://bcr.bazel.build/modules/stardoc/0.7.1/MODULE.bazel": "3548faea4ee5dda5580f9af150e79d0f6aea934fc60c1cc50f4efdd9420759e7", 145 | "https://bcr.bazel.build/modules/stardoc/0.7.1/source.json": "b6500ffcd7b48cd72c29bb67bcac781e12701cc0d6d55d266a652583cfcdab01", 146 | "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", 147 | "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", 148 | "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", 149 | "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", 150 | "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79", 151 | "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d", 152 | "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" 153 | }, 154 | "selectedYankedVersions": {}, 155 | "moduleExtensions": { 156 | "@@apple_support~//crosstool:setup.bzl%apple_cc_configure_extension": { 157 | "general": { 158 | "bzlTransitiveDigest": "ltCGFbl/LQQZXn/LEMXfKX7pGwyqNiOCHcmiQW0tmjM=", 159 | "usagesDigest": "2Jj0sTGzjx2KfYRjWYbL6DZ1bi8HL2roIAGfOViiul8=", 160 | "recordedFileInputs": {}, 161 | "recordedDirentsInputs": {}, 162 | "envVariables": {}, 163 | "generatedRepoSpecs": { 164 | "local_config_apple_cc_toolchains": { 165 | "bzlFile": "@@apple_support~//crosstool:setup.bzl", 166 | "ruleClassName": "_apple_cc_autoconf_toolchains", 167 | "attributes": {} 168 | }, 169 | "local_config_apple_cc": { 170 | "bzlFile": "@@apple_support~//crosstool:setup.bzl", 171 | "ruleClassName": "_apple_cc_autoconf", 172 | "attributes": {} 173 | } 174 | }, 175 | "recordedRepoMappingEntries": [ 176 | [ 177 | "apple_support~", 178 | "bazel_tools", 179 | "bazel_tools" 180 | ] 181 | ] 182 | } 183 | }, 184 | "@@platforms//host:extension.bzl%host_platform": { 185 | "general": { 186 | "bzlTransitiveDigest": "xelQcPZH8+tmuOHVjL9vDxMnnQNMlwj0SlvgoqBkm4U=", 187 | "usagesDigest": "hgylFkgWSg0ulUwWZzEM1aIftlUnbmw2ynWLdEfHnZc=", 188 | "recordedFileInputs": {}, 189 | "recordedDirentsInputs": {}, 190 | "envVariables": {}, 191 | "generatedRepoSpecs": { 192 | "host_platform": { 193 | "bzlFile": "@@platforms//host:extension.bzl", 194 | "ruleClassName": "host_platform_repo", 195 | "attributes": {} 196 | } 197 | }, 198 | "recordedRepoMappingEntries": [] 199 | } 200 | }, 201 | "@@pybind11_bazel~//:internal_configure.bzl%internal_configure_extension": { 202 | "general": { 203 | "bzlTransitiveDigest": "Jsid+Er+k0JcuPNjK83nTid7q9/fJQeD+ntFHFVkKQg=", 204 | "usagesDigest": "CrO575Nyn72y6/xY81m7gZFNCtDmWWrrC58+CWZQVsQ=", 205 | "recordedFileInputs": {}, 206 | "recordedDirentsInputs": {}, 207 | "envVariables": {}, 208 | "generatedRepoSpecs": { 209 | "pybind11": { 210 | "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", 211 | "ruleClassName": "http_archive", 212 | "attributes": { 213 | "build_file": "@@pybind11_bazel~//:pybind11-BUILD.bazel", 214 | "strip_prefix": "pybind11-2.13.6", 215 | "url": "https://github.com/pybind/pybind11/archive/refs/tags/v2.13.6.tar.gz", 216 | "integrity": "sha256-4Iy4f0dz2pf6e18DXeh2OrxlbYfVdz5i9toFh9Hw7CA=" 217 | } 218 | } 219 | }, 220 | "recordedRepoMappingEntries": [ 221 | [ 222 | "pybind11_bazel~", 223 | "bazel_tools", 224 | "bazel_tools" 225 | ] 226 | ] 227 | } 228 | }, 229 | "@@rules_kotlin~//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { 230 | "general": { 231 | "bzlTransitiveDigest": "fus14IFJ/1LGWWGKPH/U18VnJCoMjfDt1ckahqCnM0A=", 232 | "usagesDigest": "aJF6fLy82rR95Ff5CZPAqxNoFgOMLMN5ImfBS0nhnkg=", 233 | "recordedFileInputs": {}, 234 | "recordedDirentsInputs": {}, 235 | "envVariables": {}, 236 | "generatedRepoSpecs": { 237 | "com_github_jetbrains_kotlin_git": { 238 | "bzlFile": "@@rules_kotlin~//src/main/starlark/core/repositories:compiler.bzl", 239 | "ruleClassName": "kotlin_compiler_git_repository", 240 | "attributes": { 241 | "urls": [ 242 | "https://github.com/JetBrains/kotlin/releases/download/v1.9.23/kotlin-compiler-1.9.23.zip" 243 | ], 244 | "sha256": "93137d3aab9afa9b27cb06a824c2324195c6b6f6179d8a8653f440f5bd58be88" 245 | } 246 | }, 247 | "com_github_jetbrains_kotlin": { 248 | "bzlFile": "@@rules_kotlin~//src/main/starlark/core/repositories:compiler.bzl", 249 | "ruleClassName": "kotlin_capabilities_repository", 250 | "attributes": { 251 | "git_repository_name": "com_github_jetbrains_kotlin_git", 252 | "compiler_version": "1.9.23" 253 | } 254 | }, 255 | "com_github_google_ksp": { 256 | "bzlFile": "@@rules_kotlin~//src/main/starlark/core/repositories:ksp.bzl", 257 | "ruleClassName": "ksp_compiler_plugin_repository", 258 | "attributes": { 259 | "urls": [ 260 | "https://github.com/google/ksp/releases/download/1.9.23-1.0.20/artifacts.zip" 261 | ], 262 | "sha256": "ee0618755913ef7fd6511288a232e8fad24838b9af6ea73972a76e81053c8c2d", 263 | "strip_version": "1.9.23-1.0.20" 264 | } 265 | }, 266 | "com_github_pinterest_ktlint": { 267 | "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", 268 | "ruleClassName": "http_file", 269 | "attributes": { 270 | "sha256": "01b2e0ef893383a50dbeb13970fe7fa3be36ca3e83259e01649945b09d736985", 271 | "urls": [ 272 | "https://github.com/pinterest/ktlint/releases/download/1.3.0/ktlint" 273 | ], 274 | "executable": true 275 | } 276 | }, 277 | "rules_android": { 278 | "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", 279 | "ruleClassName": "http_archive", 280 | "attributes": { 281 | "sha256": "cd06d15dd8bb59926e4d65f9003bfc20f9da4b2519985c27e190cddc8b7a7806", 282 | "strip_prefix": "rules_android-0.1.1", 283 | "urls": [ 284 | "https://github.com/bazelbuild/rules_android/archive/v0.1.1.zip" 285 | ] 286 | } 287 | } 288 | }, 289 | "recordedRepoMappingEntries": [ 290 | [ 291 | "rules_kotlin~", 292 | "bazel_tools", 293 | "bazel_tools" 294 | ] 295 | ] 296 | } 297 | } 298 | } 299 | } 300 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Subspace IPC 2 | Next Generation, sub-microsecond latency shared memory IPC. 3 | 4 | This is a shared-memory based pub/sub Interprocess Communication system that can be used 5 | in robotics and other applications. Why *subspace*? If your messages are transported 6 | between processes on the same computer, they travel through extremely low latency 7 | and high bandwidth shared memory buffers, kind of like they are going 8 | faster than light (not really, of course). If they go between computers, they are 9 | transported over the network at sub-light speed. 10 | 11 | It has the following features: 12 | 13 | 1. Single threaded coroutine based server process written in C++17 14 | 1. Coroutine-aware client library, in C++17. 15 | 1. Publish/subscribe methodology with multiple publisher and multiple subscribers per channel. 16 | 1. No communication with server for message transfer. 17 | 1. Message type agnostic transmission – bring your own serialization. 18 | 2. Channel types, meaningful to user, not system. 19 | 1. Single lock POSIX shared memory channels 20 | 1. Both unreliable and reliable communications between publishers and subscribers. 21 | 1. Ability to read the next or newest message in a channel. 22 | 1. File-descriptor-based event triggers. 23 | 1. Automatic UDP discovery and TCP bridging of channels between servers. 24 | 1. Shared and weak pointers for message references. 25 | 1. Ports to MacOS and Linux, ARM64 and x86_64. 26 | 1. Builds using Bazel and uses Abseil and Protocol Buffers from Google. 27 | 1. Uses my C++ coroutine library (https://github.com/dallison/cocpp) 28 | 29 | See the file docs/subspace.pdf for full documentation. 30 | 31 | # Building 32 | This uses Google's Bazel to build. You will need to download Bazel to build it. 33 | The build also needs some external libraries, but Bazel takes care of downloading them. 34 | The *.bazelrc* file contains some configuration options. 35 | 36 | ## To build on Mac Apple Silicon 37 | ``` 38 | bazel build --config=apple_silicon ... 39 | ``` 40 | 41 | ## To build on Linux 42 | Subspace really wants to be built using *clang*. Depending on how your OS is configured, you 43 | might need to tell bazel what compiler to use. 44 | 45 | ``` 46 | CC=clang bazel build ... 47 | ``` 48 | 49 | It does build with *g++* but you will get some compiler warnings about different signed comparisons 50 | that clang doesn't care about. 51 | 52 | ### Example: Ubuntu 20.04 53 | Build a minimal set of binaries: 54 | 55 | ``` 56 | CC=clang bazel build //server:subspace_server //manual_tests:{pub,sub} 57 | ``` 58 | 59 | Then run each in a separate terminal: 60 | 61 | * `./bazel-bin/server/subspace_server` 62 | * `./bazel-bin/manual_tests/sub` 63 | * `./bazel-bin/manual_tests/pub` 64 | 65 | # Bazel WORKSPACE 66 | Add this to your Bazel WORKSPACE file to get access to this library without downloading it manually. 67 | 68 | ``` 69 | http_archive( 70 | name = "subspace", 71 | urls = ["https://github.com/dallison/subspace/archive/refs/tags/A.B.C.tar.gz"], 72 | strip_prefix = "subspace-A.B.C", 73 | ) 74 | 75 | ``` 76 | 77 | You can also add a sha256 field to ensure a canonical build if you like. Bazel 78 | will tell you what to put in for the hash when you first build it. 79 | 80 | -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | # This file marks the root of the Bazel workspace. 2 | # See MODULE.bazel for external dependencies setup. 3 | 4 | workspace(name = "subspace") 5 | -------------------------------------------------------------------------------- /bzl/BUILD.bazel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dallison/subspace/4975a941f77278f94c52ce86b39f312d9a647c47/bzl/BUILD.bazel -------------------------------------------------------------------------------- /bzl/local_rules/BUILD.bazel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dallison/subspace/4975a941f77278f94c52ce86b39f312d9a647c47/bzl/local_rules/BUILD.bazel -------------------------------------------------------------------------------- /bzl/local_rules/pybind.bzl: -------------------------------------------------------------------------------- 1 | """Rule(s) for creation of pybind python bindings of C/C++ code.""" 2 | 3 | load("@rules_cc//cc:defs.bzl", "cc_binary") 4 | load("@rules_python//python:defs.bzl", "py_library") 5 | 6 | PYBIND_DEPS = [ 7 | "@pybind11", 8 | ] 9 | 10 | def pybind_module( 11 | name, 12 | srcs = [], 13 | deps = [], 14 | data = [], 15 | **kwargs): 16 | """Build a pybind target. 17 | 18 | This creates a cc_binary with the proper link options and a corresponding 19 | py_library. 20 | 21 | Args: 22 | name: pybind target name 23 | srcs: source files. Defaults to []. 24 | deps: dependencies. Defaults to []. 25 | data: data targets. Defaults to []. 26 | **kwargs: additional keyword arguments to 27 | pass to cc_library AND py_library. 28 | """ 29 | cc_output_name = name + ".so" 30 | cc_binary( 31 | name = name + ".so", 32 | srcs = srcs, 33 | linkshared = 1, 34 | deps = deps + PYBIND_DEPS, 35 | **kwargs 36 | ) 37 | 38 | py_library( 39 | name = name, 40 | srcs = [], 41 | data = [cc_output_name] + data, 42 | **kwargs 43 | ) 44 | -------------------------------------------------------------------------------- /bzl/third_party/BUILD.bazel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dallison/subspace/4975a941f77278f94c52ce86b39f312d9a647c47/bzl/third_party/BUILD.bazel -------------------------------------------------------------------------------- /bzl/third_party/pybind11.BUILD: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 The Pybind Development Team. All rights reserved. 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions are met: 5 | # 6 | # 1. Redistributions of source code must retain the above copyright notice, this 7 | # list of conditions and the following disclaimer. 8 | # 9 | # 2. Redistributions in binary form must reproduce the above copyright notice, 10 | # this list of conditions and the following disclaimer in the documentation 11 | # and/or other materials provided with the distribution. 12 | # 13 | # 3. Neither the name of the copyright holder nor the names of its contributors 14 | # may be used to endorse or promote products derived from this software 15 | # without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18 | # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 21 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 25 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | # 28 | # You are under no obligation whatsoever to provide any bug fixes, patches, or 29 | # upgrades to the features, functionality or performance of the source code 30 | # ("Enhancements") to anyone; however, if you choose to make your Enhancements 31 | # available either publicly, or directly to the author of this software, without 32 | # imposing a separate written license agreement for such Enhancements, then you 33 | # hereby grant the following license: a non-exclusive, royalty-free perpetual 34 | # license to install, use, modify, prepare derivative works, incorporate into 35 | # other computer software, distribute, and sublicense such enhancements or 36 | # derivative works thereof, in binary and source code form. 37 | 38 | load("@rules_cc//cc:defs.bzl", "cc_library") 39 | 40 | package(default_visibility = ["//visibility:public"]) 41 | 42 | licenses(["notice"]) 43 | 44 | exports_files(["LICENSE"]) 45 | 46 | OPTIONS = [ 47 | "-fexceptions", 48 | # Useless warnings 49 | "-Xclang-only=-Wno-undefined-inline", 50 | "-Xclang-only=-Wno-pragma-once-outside-header", 51 | "-Xgcc-only=-Wno-error", # no way to just disable the pragma-once warning in gcc 52 | ] 53 | 54 | INCLUDES = [ 55 | "include/pybind11/*.h", 56 | "include/pybind11/detail/*.h", 57 | ] 58 | 59 | EXCLUDES = [ 60 | # Deprecated file that just emits a warning 61 | "include/pybind11/common.h", 62 | ] 63 | 64 | cc_library( 65 | name = "pybind11", 66 | hdrs = glob( 67 | INCLUDES, 68 | exclude = EXCLUDES, 69 | ), 70 | copts = OPTIONS, 71 | includes = ["include"], 72 | deps = ["@python_default//:python_headers"], 73 | ) 74 | 75 | cc_library( 76 | name = "pybind11_embed", 77 | hdrs = glob( 78 | INCLUDES, 79 | exclude = EXCLUDES, 80 | ), 81 | copts = OPTIONS, 82 | includes = ["include"], 83 | deps = ["@python_default//:python_embed"], 84 | ) 85 | -------------------------------------------------------------------------------- /client/BUILD.bazel: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | cc_library( 4 | name = "subspace_client", 5 | srcs = [ 6 | "client.cc", 7 | ], 8 | hdrs = [ 9 | "client.h", 10 | "client_channel.h", 11 | "options.h", 12 | ], 13 | deps = [ 14 | "//common:subspace_common", 15 | "@com_google_absl//absl/container:flat_hash_map", 16 | "@com_google_absl//absl/container:flat_hash_set", 17 | "@com_google_absl//absl/status", 18 | "@com_google_absl//absl/status:statusor", 19 | "@com_google_absl//absl/strings:str_format", 20 | "@coroutines//:co", 21 | ], 22 | ) 23 | 24 | cc_test( 25 | name = "client_test", 26 | size = "small", 27 | srcs = ["client_test.cc"], 28 | data = [ 29 | "//server:subspace_server", 30 | ], 31 | deps = [ 32 | ":subspace_client", 33 | "//server", 34 | "@com_google_absl//absl/flags:flag", 35 | "@com_google_absl//absl/flags:parse", 36 | "@com_google_absl//absl/hash:hash_testing", 37 | "@com_google_absl//absl/status", 38 | "@com_google_absl//absl/status:statusor", 39 | "@com_google_googletest//:gtest", 40 | "@coroutines//:co", 41 | ], 42 | ) 43 | -------------------------------------------------------------------------------- /client/client_channel.h: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Allison 2 | // All Rights Reserved 3 | // See LICENSE file for licensing information. 4 | 5 | #ifndef __CLIENT_CLIENT_CHANNEL_H 6 | #define __CLIENT_CLIENT_CHANNEL_H 7 | 8 | #include "client/options.h" 9 | #include "common/channel.h" 10 | #include "coroutine.h" 11 | #include "proto/subspace.pb.h" 12 | #include "toolbelt/fd.h" 13 | #include "toolbelt/sockets.h" 14 | #include "toolbelt/triggerfd.h" 15 | #include 16 | 17 | // Notification strategy 18 | // --------------------- 19 | // Each subscriber and reliable publisher has a TriggerFd, which is 20 | // either implemented as a pipe or an eventfd. The idea is that the 21 | // program can wait for the event to be triggered for a new 22 | // message or the ability to send another message. 23 | // 24 | // For a subscriber, the event is triggered when there are messages 25 | // for the subscriber to read. Once triggered, the subscriber 26 | // needs to read all the available messages before going back into 27 | // wait mode. 28 | // 29 | // For a reliable publisher, the publisher should wait for its 30 | // event if it fails to get a buffer to send a message. The 31 | // subscribers will trigger the event when they have read all 32 | // the aviailable messages. 33 | // 34 | // The naive approach to get this working is for the publisher 35 | // to trigger the subscribers events for every message it sends, but 36 | // this would mean a write to a pipe every time a message is 37 | // sent. That's a system call and takes some time. 38 | // Instead we take the approach that the publisher only triggers when 39 | // it publishes a message immediately after a message that has been 40 | // seen by a subscriber. In other words, if a publisher is publishing 41 | // a bunch of messages at once, only the first one will result in 42 | // the subscribers being triggered. All the other messages are 43 | // buffered ahead for the subscriber to pick up when it starts 44 | // reading. 45 | // 46 | // Practically, this means that the publisher will notify the 47 | // subscriber if the last message in the active list before 48 | // it adds a new one has the kMessageSeen flag set (has been 49 | // seen by a subscriber). If the publisher is publishing slowly 50 | // it will trigger the subscribers for every message. 51 | // 52 | // For reliable publisher triggers, the naive design would trigger 53 | // the publisher every time it reads a message. The better 54 | // approach is for the subscriber to only trigger 55 | // the event when it has finished reading all the messages that 56 | // are available. When the publisher receives the event it can then 57 | // fill up all the slots that have been read before it runs out 58 | // of slots again (if it's going fast). 59 | // 60 | // Using this strategy, we limit the number of event triggers and 61 | // thus improve the latency of the system by eliminating unnecessary 62 | // system calls to write to a pipe or eventfd. 63 | // 64 | // As of time of writing, the latency for a message on the same 65 | // computer using shared memory is about 0.25 microseconds on a Mac M1. 66 | 67 | namespace subspace { 68 | 69 | class ClientImpl; 70 | 71 | namespace details { 72 | 73 | // This is a channel as seen by a client. It's going to be either 74 | // a publisher or a subscriber, as defined as the subclasses. 75 | class ClientChannel : public Channel { 76 | public: 77 | ClientChannel(const std::string &name, int num_slots, int channel_id, 78 | std::string type) 79 | : Channel(name, num_slots, channel_id, std::move(type)) {} 80 | virtual ~ClientChannel() = default; 81 | MessageSlot *CurrentSlot() const { return slot_; } 82 | const ChannelCounters &GetCounters() const { 83 | return GetScb()->counters[GetChannelId()]; 84 | } 85 | 86 | protected: 87 | virtual bool IsSubscriber() const { return false; } 88 | virtual bool IsPublisher() const { return false; } 89 | 90 | void SetSlot(MessageSlot *slot) { slot_ = slot; } 91 | void *GetCurrentBufferAddress() { return GetBufferAddress(slot_); } 92 | 93 | void SetMessageSize(int64_t message_size) { 94 | slot_->message_size = message_size; 95 | } 96 | 97 | protected: 98 | MessageSlot *slot_ = nullptr; // Current slot. 99 | }; 100 | 101 | // This is a publisher. It maps in the channel's memory and allows 102 | // messages to be published. 103 | class PublisherImpl : public ClientChannel { 104 | public: 105 | PublisherImpl(const std::string &name, int num_slots, int channel_id, 106 | int publisher_id, std::string type, 107 | const PublisherOptions &options) 108 | : ClientChannel(name, num_slots, channel_id, std::move(type)), 109 | publisher_id_(publisher_id), options_(options) {} 110 | 111 | bool IsReliable() const { return options_.IsReliable(); } 112 | bool IsLocal() const { return options_.IsLocal(); } 113 | bool IsFixedSize() const { return options_.IsFixedSize(); } 114 | 115 | private: 116 | friend class ::subspace::ClientImpl; 117 | 118 | bool IsPublisher() const override { return true; } 119 | 120 | PublishedMessage 121 | ActivateSlotAndGetAnother(bool reliable, bool is_activation, bool omit_prefix, 122 | bool *notify, 123 | std::function reload) { 124 | return Channel::ActivateSlotAndGetAnother(slot_, reliable, is_activation, 125 | publisher_id_, omit_prefix, 126 | notify, std::move(reload)); 127 | } 128 | void ClearSubscribers() { subscribers_.clear(); } 129 | void AddSubscriber(toolbelt::FileDescriptor fd) { 130 | subscribers_.emplace_back(toolbelt::FileDescriptor(), std::move(fd)); 131 | } 132 | size_t NumSubscribers() { return subscribers_.size(); } 133 | 134 | void SetTriggerFd(toolbelt::FileDescriptor fd) { 135 | trigger_.SetTriggerFd(std::move(fd)); 136 | } 137 | void SetPollFd(toolbelt::FileDescriptor fd) { 138 | trigger_.SetPollFd(std::move(fd)); 139 | } 140 | 141 | toolbelt::FileDescriptor &GetPollFd() { return trigger_.GetPollFd(); } 142 | 143 | void TriggerSubscribers() { 144 | for (auto &fd : subscribers_) { 145 | fd.Trigger(); 146 | } 147 | } 148 | int GetPublisherId() const { return publisher_id_; } 149 | 150 | void ClearPollFd() { trigger_.Clear(); } 151 | 152 | toolbelt::TriggerFd trigger_; 153 | int publisher_id_; 154 | std::vector subscribers_; 155 | PublisherOptions options_; 156 | }; 157 | 158 | // A subscriber reads messages from a channel. It maps the channel 159 | // shared memory. 160 | class SubscriberImpl : public ClientChannel { 161 | public: 162 | SubscriberImpl(const std::string &name, int num_slots, int channel_id, 163 | int subscriber_id, std::string type, 164 | const SubscriberOptions &options) 165 | : ClientChannel(name, num_slots, channel_id, std::move(type)), 166 | subscriber_id_(subscriber_id), options_(options) { 167 | shared_ptr_refs_ = 168 | std::make_unique[]>(options.MaxSharedPtrs()); 169 | for (int i = 0; i < options.MaxSharedPtrs(); ++i) { 170 | shared_ptr_refs_[i] = 0; 171 | } 172 | } 173 | 174 | std::shared_ptr shared_from_this() { 175 | return std::static_pointer_cast( 176 | Channel::shared_from_this()); 177 | } 178 | 179 | int64_t CurrentOrdinal() const { 180 | return CurrentSlot() == nullptr ? -1 : CurrentSlot()->ordinal; 181 | } 182 | int64_t Timestamp() const { 183 | return CurrentSlot() == nullptr ? 0 : Prefix(CurrentSlot())->timestamp; 184 | } 185 | bool IsReliable() const { return options_.IsReliable(); } 186 | 187 | int32_t SlotSize() const { return Channel::SlotSize(CurrentSlot()); } 188 | 189 | // We need another shared ptr. Find a free reference and return 190 | // an index. We have already checked that there is room so this 191 | // can never fail. 192 | size_t AllocateSharedPtr() { 193 | for (size_t i = 0; i < size_t(MaxSharedPtrs()); ++i) { 194 | int zero = 0; 195 | if (shared_ptr_refs_[i].compare_exchange_strong(zero, 1)) { 196 | num_shared_ptrs_++; 197 | return i; 198 | } 199 | } 200 | // Can never get here as the limits have been checked. 201 | abort(); 202 | } 203 | 204 | void IncDecSharedPtrRefCount(int inc, size_t index) { 205 | for (;;) { 206 | int curr = shared_ptr_refs_[index]; 207 | int next = curr + inc; 208 | if (shared_ptr_refs_[index].compare_exchange_strong(curr, next)) { 209 | if (next == 0) { 210 | num_shared_ptrs_--; 211 | } 212 | return; 213 | } 214 | } 215 | } 216 | int NumSharedPtrs() const { return num_shared_ptrs_; } 217 | int MaxSharedPtrs() const { return options_.MaxSharedPtrs(); } 218 | bool CheckSharedPtrCount() const { 219 | return num_shared_ptrs_ < options_.MaxSharedPtrs(); 220 | } 221 | 222 | int GetSharedPtrRefCount(size_t index) const { 223 | return shared_ptr_refs_[index]; 224 | } 225 | 226 | bool LockForShared(MessageSlot *slot, int64_t ordinal) { 227 | return LockForSharedInternal(slot, ordinal, IsReliable()); 228 | } 229 | 230 | private: 231 | friend class ::subspace::ClientImpl; 232 | 233 | bool IsSubscriber() const override { return true; } 234 | 235 | void ClearPublishers() { reliable_publishers_.clear(); } 236 | void AddPublisher(toolbelt::FileDescriptor fd) { 237 | reliable_publishers_.emplace_back(toolbelt::FileDescriptor(), 238 | std::move(fd)); 239 | } 240 | size_t NumReliablePublishers() { return reliable_publishers_.size(); } 241 | 242 | void SetTriggerFd(toolbelt::FileDescriptor fd) { 243 | trigger_.SetTriggerFd(std::move(fd)); 244 | } 245 | void SetPollFd(toolbelt::FileDescriptor fd) { 246 | trigger_.SetPollFd(std::move(fd)); 247 | } 248 | int GetSubscriberId() const { return subscriber_id_; } 249 | void TriggerReliablePublishers() { 250 | for (auto &fd : reliable_publishers_) { 251 | fd.Trigger(); 252 | } 253 | } 254 | void Trigger() { trigger_.Trigger(); } 255 | 256 | MessageSlot *NextSlot(std::function reload) { 257 | return Channel::NextSlot(CurrentSlot(), IsReliable(), subscriber_id_, 258 | std::move(reload)); 259 | } 260 | 261 | MessageSlot *LastSlot(std::function reload) { 262 | return Channel::LastSlot(CurrentSlot(), IsReliable(), subscriber_id_, 263 | std::move(reload)); 264 | } 265 | 266 | toolbelt::FileDescriptor &GetPollFd() { return trigger_.GetPollFd(); } 267 | void ClearPollFd() { trigger_.Clear(); } 268 | 269 | MessageSlot *FindMessage(uint64_t timestamp) { 270 | MessageSlot *slot = 271 | FindActiveSlotByTimestamp(CurrentSlot(), timestamp, IsReliable(), 272 | GetSubscriberId(), search_buffer_, nullptr); 273 | if (slot != nullptr) { 274 | SetSlot(slot); 275 | } 276 | return slot; 277 | } 278 | 279 | int subscriber_id_; 280 | toolbelt::TriggerFd trigger_; 281 | std::vector reliable_publishers_; 282 | SubscriberOptions options_; 283 | std::atomic num_shared_ptrs_ = 0; 284 | 285 | // Each shared_ptr has an index into this array that holds a reference 286 | // counter. Each copy of a shared_ptr increments the reference counter 287 | // and when it goes to zero, we know that there are no more copies of 288 | // the shared_ptr. 289 | std::unique_ptr[]> shared_ptr_refs_; 290 | 291 | // It is rare that subscribers need to search for messges by timestamp. This 292 | // will keep the memory allocation to the first search on a subscriber. Most 293 | // subscribers won't use this. 294 | std::vector search_buffer_; 295 | }; 296 | } // namespace details 297 | } // namespace subspace 298 | 299 | #endif // __CLIENT_CLIENT_CHANNEL_H 300 | -------------------------------------------------------------------------------- /client/options.h: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Allison 2 | // All Rights Reserved 3 | // See LICENSE file for licensing information. 4 | 5 | #ifndef __CLIENT_OPTIONS_H 6 | #define __CLIENT_OPTIONS_H 7 | 8 | namespace subspace { 9 | 10 | // You can use the options in two ways depending on your 11 | // coding guidelines. You can either use the Google/Java-style 12 | // chained setters method: 13 | // Options().SetA(1).SetB(2).SetC(3) 14 | // 15 | // or you can use the newer designated initializer style: 16 | // {.a = 1, .b = 2, .c = 3}. 17 | // The designated initializer allows you to omit the call to 18 | // the constructor and pass the initializer-list directly. 19 | // 20 | // When reading the option values, you can use the getter function 21 | // or you can access the struct member directly. 22 | // 23 | // Your choice. 24 | 25 | // Options when creating a publisher. 26 | struct PublisherOptions { 27 | // A public publisher's messages will be seen outside of the 28 | // publishing computer. 29 | PublisherOptions &SetLocal(bool v) { 30 | local = v; 31 | return *this; 32 | } 33 | // A reliable publisher's messages will never be missed by 34 | // a reliable subscriber. 35 | PublisherOptions &SetReliable(bool v) { 36 | reliable = v; 37 | return *this; 38 | } 39 | // Set the type of the message to be published. The type is 40 | // not meaningful to the subspace system. It's up to the 41 | // user to figure out what it means. The same type must 42 | // be used by all other subscribers and publishers. 43 | // By default there is no type. 44 | PublisherOptions &SetType(std::string t) { 45 | type = std::move(t); 46 | return *this; 47 | } 48 | 49 | // Set the option to allow the channel's slots to be resized 50 | // if necessary. 51 | PublisherOptions &SetFixedSize(bool v) { 52 | fixed_size = v; 53 | return *this; 54 | } 55 | 56 | bool IsLocal() const { return local; } 57 | bool IsReliable() const { return reliable; } 58 | bool IsFixedSize() const { return fixed_size; } 59 | const std::string &Type() const { return type; } 60 | 61 | PublisherOptions &SetBridge(bool v) { 62 | bridge = v; 63 | return *this; 64 | } 65 | 66 | bool IsBridge() const { return bridge; } 67 | 68 | bool local = false; 69 | bool reliable = false; 70 | bool bridge = false; 71 | bool fixed_size = false; 72 | std::string type; 73 | }; 74 | 75 | struct SubscriberOptions { 76 | // A reliable subscriber will never miss a message from a reliable 77 | // publisher. 78 | SubscriberOptions &SetReliable(bool v) { 79 | reliable = v; 80 | return *this; 81 | } 82 | // Set the type of the message on the channel. The type is 83 | // not meaningful to the subspace system. It's up to the 84 | // user to figure out what it means. The same type must 85 | // be used by all other subscribers and publishers. 86 | // By default there is no type. 87 | SubscriberOptions &SetType(std::string t) { 88 | type = std::move(t); 89 | return *this; 90 | } 91 | 92 | SubscriberOptions &SetMaxSharedPtrs(int n) { 93 | max_shared_ptrs = n; 94 | return *this; 95 | } 96 | 97 | bool IsReliable() const { return reliable; } 98 | const std::string &Type() const { return type; } 99 | int MaxSharedPtrs() const { return max_shared_ptrs; } 100 | 101 | SubscriberOptions &SetBridge(bool v) { 102 | bridge = v; 103 | return *this; 104 | } 105 | bool IsBridge() const { return bridge; } 106 | 107 | bool reliable = false; 108 | bool bridge = false; 109 | std::string type; 110 | int max_shared_ptrs = 0; 111 | }; 112 | 113 | } // namespace subspace 114 | 115 | #endif // __CLIENT_OPTIONS_H 116 | -------------------------------------------------------------------------------- /client/python/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") 2 | 3 | pybind_extension( 4 | name = "subspace", 5 | srcs = ["client.cc"], 6 | visibility = ["//visibility:public"], 7 | deps = ["//client:subspace_client"], 8 | ) 9 | 10 | py_test( 11 | name = "client_test", 12 | srcs = ["client_test.py",], 13 | data = [ 14 | "//server:subspace_server", 15 | ], 16 | deps = [ 17 | ":subspace", 18 | "@rules_python//python/runfiles", 19 | ], 20 | ) 21 | 22 | -------------------------------------------------------------------------------- /client/python/client.cc: -------------------------------------------------------------------------------- 1 | 2 | #include "client/client.h" 3 | 4 | #include "pybind11/pybind11.h" 5 | #include 6 | #include 7 | 8 | namespace subspace { 9 | namespace python { 10 | 11 | namespace py = pybind11; 12 | 13 | PYBIND11_MODULE(subspace, m) { 14 | 15 | m.doc() = "This is a python module to pass messages over the Subspace inter-process communication protocol."; 16 | 17 | py::class_ publisher_class(m, "Publisher", "The Publisher class is the main interface for sending messages."); 18 | 19 | publisher_class.def( 20 | "publish_message", 21 | [](Publisher* self, py::bytes message_data) { 22 | auto msg_view = static_cast(message_data); 23 | absl::StatusOr dest_buffer = self->GetMessageBuffer(msg_view.size()); 24 | if (!dest_buffer.ok()) { 25 | throw std::runtime_error(dest_buffer.status().ToString()); 26 | } 27 | std::memcpy(*dest_buffer, msg_view.data(), msg_view.size()); 28 | absl::StatusOr send_result = self->PublishMessage(msg_view.size()); 29 | if (!send_result.ok()) { 30 | throw std::runtime_error(send_result.status().ToString()); 31 | } 32 | }, 33 | "Publish the message in the publisher's buffer."); 34 | 35 | publisher_class.def( 36 | "wait", 37 | [](Publisher* self) { 38 | absl::Status result = self->Wait(); 39 | if (!result.ok()) { 40 | throw std::runtime_error(result.ToString()); 41 | } 42 | }, 43 | "Wait until a reliable publisher can try again to send a message."); 44 | 45 | publisher_class.def("type", &Publisher::Type); 46 | publisher_class.def("is_reliable", &Publisher::IsReliable); 47 | publisher_class.def("is_local", &Publisher::IsLocal); 48 | publisher_class.def("is_fixed_size", &Publisher::IsFixedSize); 49 | publisher_class.def("slot_size", &Publisher::SlotSize); 50 | 51 | 52 | py::class_ subscriber_class(m, "Subscriber", "The Subscriber class is the main interface for receiving messages."); 53 | 54 | subscriber_class.def( 55 | "read_message", 56 | [](Subscriber* self, bool skip_to_newest) { 57 | absl::StatusOr read_result = self->ReadMessage(skip_to_newest ? ReadMode::kReadNewest : ReadMode::kReadNext); 58 | if (!read_result.ok()) { 59 | throw std::runtime_error(read_result.status().ToString()); 60 | } 61 | return py::bytes(reinterpret_cast(read_result->buffer), read_result->length); 62 | }, 63 | R"doc("Read a message from a subscriber. If there are no available messages, 64 | the returned bytes will have zero length. Setting the 'skip_to_newest' argument 65 | to True, causes the read to skip ahead to the newest available message, otherwise, 66 | it reads the next available message (oldest message not read yet).)doc", 67 | py::arg("skip_to_newest") = false, 68 | py::return_value_policy::copy); 69 | 70 | subscriber_class.def( 71 | "wait", 72 | [](Subscriber* self) { 73 | absl::Status result = self->Wait(); 74 | if (!result.ok()) { 75 | throw std::runtime_error(result.ToString()); 76 | } 77 | }, 78 | "Wait until there's a message available to be read by the subscriber."); 79 | 80 | subscriber_class.def("type", &Subscriber::Type); 81 | subscriber_class.def("is_reliable", &Subscriber::IsReliable); 82 | subscriber_class.def("slot_size", &Subscriber::SlotSize); 83 | 84 | 85 | py::class_ client_class(m, "Client", 86 | R"doc(This is an Subspace client. 87 | It must be initialized by calling Init() before 88 | it can be used. The Init() function connects it to an Subspace server that 89 | is listening on the same Unix Domain Socket.)doc"); 90 | 91 | client_class.def(py::init<>()); 92 | 93 | client_class.def( 94 | "init", 95 | [](Client* self, const std::string& server_socket, const std::string& client_name) { 96 | absl::Status result = self->Init(server_socket, client_name); 97 | if (!result.ok()) { 98 | throw std::runtime_error(result.ToString()); 99 | } 100 | }, 101 | "Initialize the client by connecting to the server.", 102 | py::arg("server_socket") = std::string("/tmp/subspace"), 103 | py::arg("client_name") = std::string("")); 104 | 105 | 106 | client_class.def( 107 | "create_publisher", 108 | [](Client* self, const std::string &channel_name, int slot_size, int num_slots, 109 | bool local, bool reliable, bool fixed_size, const std::string& type) -> Publisher { 110 | absl::StatusOr result = self->CreatePublisher(channel_name, slot_size, num_slots, 111 | PublisherOptions().SetLocal(local).SetReliable(reliable).SetFixedSize(fixed_size).SetType(type)); 112 | if (!result.ok()) { 113 | throw std::runtime_error(result.status().ToString()); 114 | } 115 | return std::move(*result); 116 | }, 117 | R"doc(Create a publisher for the given channel. If the channel doesn't exit 118 | it will be created with num_slots slots, each of which is slot_size 119 | bytes long.)doc", 120 | py::arg("channel_name"), py::arg("slot_size"), py::arg("num_slots"), 121 | py::arg("local") = false, py::arg("reliable") = false, 122 | py::arg("fixed_size") = false, py::arg("type") = std::string(""), 123 | py::return_value_policy::move); 124 | 125 | 126 | client_class.def( 127 | "create_subscriber", 128 | [](Client* self, const std::string &channel_name, 129 | bool reliable, const std::string& type) -> Subscriber { 130 | absl::StatusOr result = self->CreateSubscriber(channel_name, 131 | SubscriberOptions().SetReliable(reliable).SetType(type)); 132 | if (!result.ok()) { 133 | throw std::runtime_error(result.status().ToString()); 134 | } 135 | return std::move(*result); 136 | }, 137 | R"doc(Create a subscriber for the given channel. This can be done before there 138 | are any publishers on the channel.)doc", 139 | py::arg("channel_name"), py::arg("reliable") = false, py::arg("type") = std::string(""), 140 | py::return_value_policy::move); 141 | 142 | } 143 | 144 | } // namespace python 145 | } // namespace subspace 146 | 147 | 148 | -------------------------------------------------------------------------------- /client/python/client_test.py: -------------------------------------------------------------------------------- 1 | 2 | from rules_python.python.runfiles import runfiles 3 | 4 | import client.python.subspace as subspace 5 | import os 6 | import subprocess 7 | import tempfile 8 | import time 9 | import unittest 10 | 11 | class TestSubspaceClient(unittest.TestCase): 12 | def setUp(self): 13 | (socket_fd, self.socket_name) = tempfile.mkstemp(dir="/tmp", prefix="subspace") 14 | socket_file = os.fdopen(socket_fd, "w") 15 | socket_file.close() 16 | 17 | r = runfiles.Create() 18 | 19 | self.server_proc = subprocess.Popen([r.Rlocation("__main__/server/subspace_server"), "--local", "--socket=" + self.socket_name]) 20 | 21 | # Wait for client to be able to init. 22 | waiting_client = subspace.Client() 23 | while True: 24 | try: 25 | waiting_client.init(server_socket=self.socket_name, client_name="waiting_client") 26 | break 27 | except: 28 | time.sleep(0.0001) 29 | waiting_client = None 30 | 31 | def tearDown(self): 32 | self.server_proc.terminate() 33 | self.server_proc.wait() 34 | os.remove(self.socket_name) 35 | 36 | def test_one_message(self): 37 | client = subspace.Client() 38 | client.init(server_socket=self.socket_name, client_name="one_message_client") 39 | 40 | pub = client.create_publisher( 41 | channel_name="dave0", slot_size=256, num_slots=16, type="foobar") 42 | 43 | sub = client.create_subscriber( 44 | channel_name="dave0", type="foobar") 45 | 46 | pub.publish_message(b'Hello world!') 47 | 48 | sub.wait() 49 | 50 | # Check we get the right message: 51 | received_message = sub.read_message().decode('utf-8') 52 | self.assertEqual(received_message, "Hello world!") 53 | 54 | # Note, accessors of subscriber can only be used after first read_message. 55 | self.assertEqual(sub.type(), pub.type()) 56 | self.assertEqual(sub.is_reliable(), pub.is_reliable()) 57 | self.assertEqual(sub.slot_size(), pub.slot_size()) 58 | 59 | # Check that no other message exists: 60 | received_message = sub.read_message().decode('utf-8') 61 | self.assertEqual(len(received_message), 0) 62 | 63 | pub.publish_message(b'Hello again!') 64 | 65 | sub.wait() 66 | 67 | # New message appeared: 68 | received_message = sub.read_message().decode('utf-8') 69 | self.assertEqual(received_message, "Hello again!") 70 | 71 | # Check that no other message exists: 72 | received_message = sub.read_message().decode('utf-8') 73 | self.assertEqual(len(received_message), 0) 74 | 75 | # Must destroy sub / pub before client goes away. 76 | # Why, oh why, doesn't Python have RAII! 77 | pub = None 78 | sub = None 79 | 80 | def test_two_message_skip_one(self): 81 | client = subspace.Client() 82 | client.init(server_socket=self.socket_name, client_name="one_message_client") 83 | 84 | pub = client.create_publisher( 85 | channel_name="dave0", slot_size=256, num_slots=16, type="foobar") 86 | 87 | sub = client.create_subscriber( 88 | channel_name="dave0", type="foobar") 89 | 90 | pub.publish_message(b'Hello world!') 91 | pub.publish_message(b'Hello again!') 92 | 93 | sub.wait() 94 | 95 | # Check we get the right message: 96 | received_message = sub.read_message(skip_to_newest=True).decode('utf-8') 97 | self.assertEqual(received_message, "Hello again!") 98 | 99 | # Check that no other message exists: 100 | received_message = sub.read_message().decode('utf-8') 101 | self.assertEqual(len(received_message), 0) 102 | 103 | # Must destroy sub / pub before client goes away. 104 | # Why, oh why, doesn't Python have RAII! 105 | pub = None 106 | sub = None 107 | 108 | 109 | if __name__ == '__main__': 110 | unittest.main() 111 | 112 | -------------------------------------------------------------------------------- /common/BUILD.bazel: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | cc_library( 4 | name = "subspace_common", 5 | srcs = [ 6 | "channel.cc", 7 | ], 8 | hdrs = [ 9 | "channel.h", 10 | ], 11 | linkopts = select({ 12 | "//:macos_x86_64": [], 13 | "//:macos_arm64": [], 14 | "//:macos_default": [], 15 | "//conditions:default": ["-lrt"], 16 | }), 17 | deps = [ 18 | "//proto:subspace_cc_proto", 19 | "@com_google_absl//absl/container:flat_hash_map", 20 | "@com_google_absl//absl/status", 21 | "@com_google_absl//absl/status:statusor", 22 | "@com_google_absl//absl/strings", 23 | "@com_google_absl//absl/strings:str_format", 24 | "@com_google_protobuf//:protobuf", 25 | "@coroutines//:co", 26 | "@toolbelt//toolbelt", 27 | ], 28 | ) 29 | -------------------------------------------------------------------------------- /common/channel.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Allison 2 | // All Rights Reserved 3 | // See LICENSE file for licensing information. 4 | 5 | #include "common/channel.h" 6 | #include "absl/strings/str_format.h" 7 | #include "toolbelt/clock.h" 8 | #include "toolbelt/hexdump.h" 9 | #include "toolbelt/mutex.h" 10 | #include 11 | #include 12 | #if defined(__APPLE__) 13 | #include 14 | #endif 15 | #include "absl/container/flat_hash_map.h" 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | namespace subspace { 22 | 23 | // Set this to 1 to print the memory mapping and unmapping calls. 24 | #define SHOW_MMAPS 0 25 | 26 | // Set this to 1 to debug calls to map and unmap memory. This 27 | // is only valid when NDEBUG is not defined (in debug mode). 28 | #define DEBUG_MMAPS 0 29 | 30 | #if !NDEBUG && DEBUG_MMAPS 31 | // NOTE: due to C++'s undefined initialization and destruction order 32 | // these can't be actual instances. They will be allocated on the 33 | // first call to MapMemory and won't be deleted. 34 | static absl::flat_hash_map *mapped_regions; 35 | static std::mutex *region_lock; 36 | #endif 37 | 38 | static void *MapMemory(int fd, size_t size, int prot, const char *purpose) { 39 | void *p = mmap(NULL, size, prot, MAP_SHARED, fd, 0); 40 | #if SHOW_MMAPS 41 | printf("%d: mapping %s with size %zd: %p -> %p\n", getpid(), purpose, size, p, 42 | reinterpret_cast(p) + size); 43 | #endif 44 | #if !NDEBUG && DEBUG_MMAPS 45 | if (region_lock == nullptr) { 46 | region_lock = new std::mutex; 47 | mapped_regions = new absl::flat_hash_map; 48 | } 49 | std::unique_lock l(*region_lock); 50 | if (mapped_regions->find(p) != mapped_regions->end()) { 51 | fprintf(stderr, "Attempting to remap region at %p with size %zd\n", p, 52 | size); 53 | } 54 | (*mapped_regions)[p] = size; 55 | #endif 56 | return p; 57 | } 58 | 59 | static void UnmapMemory(void *p, size_t size, const char *purpose) { 60 | #if SHOW_MMAPS 61 | printf("%d: unmapping %s with size %zd: %p -> %p\n", getpid(), purpose, size, 62 | p, reinterpret_cast(p) + size); 63 | #endif 64 | #if !NDEBUG && DEBUG_MMAPS 65 | assert(region_lock != nullptr); 66 | std::unique_lock l(*region_lock); 67 | 68 | auto it = mapped_regions->find(p); 69 | if (it == mapped_regions->end()) { 70 | fprintf(stderr, 71 | "Attempting to unmap unknown region %s at %p with size %zd\n", 72 | purpose, p, size); 73 | return; 74 | } else if (it->second != size) { 75 | fprintf(stderr, 76 | "Attempting to unmap region %s at %p with wrong size %zd/%zd\n", 77 | purpose, p, it->second, size); 78 | return; 79 | } 80 | #endif 81 | munmap(p, size); 82 | #if !NDEBUG && DEBUG_MMAPS 83 | 84 | mapped_regions->erase(p); 85 | #endif 86 | } 87 | 88 | template 89 | static void UnmapBuffers(BufferSetIter first, BufferSetIter last, 90 | int num_slots) { 91 | // Unmap any previously mapped buffers. 92 | for (; first < last; ++first) { 93 | int64_t buffers_size = 94 | sizeof(BufferHeader) + 95 | num_slots * (Aligned<32>(first->slot_size) + sizeof(MessagePrefix)); 96 | if (buffers_size > 0 && first->buffer != nullptr) { 97 | UnmapMemory(first->buffer, buffers_size, "buffers"); 98 | first->buffer = nullptr; 99 | first->slot_size = 0; 100 | } 101 | } 102 | } 103 | 104 | static absl::StatusOr CreateSharedMemory(int id, const char *suffix, 105 | int64_t size, bool map, 106 | toolbelt::FileDescriptor &fd) { 107 | char shm_file[NAME_MAX]; // Unique file in file system. 108 | char *shm_name; // Name passed to shm_* (starts with /) 109 | int tmpfd; 110 | #if defined(__linux__) 111 | // On Linux we have actual files in /dev/shm so we can create a unique file. 112 | snprintf(shm_file, sizeof(shm_file), "/dev/shm/%d.%s.XXXXXX", id, suffix); 113 | tmpfd = mkstemp(shm_file); 114 | shm_name = shm_file + 8; // After /dev/shm 115 | #else 116 | // On other systems (BSD, MacOS, etc), we need to use a file in /tmp. 117 | // This is just used to ensure uniqueness. 118 | snprintf(shm_file, sizeof(shm_file), "/tmp/%d.%s.XXXXXX", id, suffix); 119 | tmpfd = mkstemp(shm_file); 120 | shm_name = shm_file + 4; // After /tmp 121 | #endif 122 | // Remove any existing shared memory. 123 | shm_unlink(shm_name); 124 | 125 | // Open the shared memory file. 126 | int shm_fd = shm_open(shm_name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); 127 | if (shm_fd == -1) { 128 | return absl::InternalError(absl::StrFormat( 129 | "Failed to open shared memory %s: %s", shm_name, strerror(errno))); 130 | } 131 | 132 | // Make it the appropriate size. 133 | int e = ftruncate(shm_fd, size); 134 | if (e == -1) { 135 | shm_unlink(shm_name); 136 | return absl::InternalError( 137 | absl::StrFormat("Failed to set length of shared memory %s: %s", 138 | shm_name, strerror(errno))); 139 | } 140 | 141 | // Map it into memory if asked 142 | void *p = nullptr; 143 | if (map) { 144 | p = MapMemory(shm_fd, size, PROT_READ | PROT_WRITE, suffix); 145 | if (p == MAP_FAILED) { 146 | shm_unlink(shm_name); 147 | return absl::InternalError(absl::StrFormat( 148 | "Failed to map shared memory %s: %s", shm_name, strerror(errno))); 149 | } 150 | } 151 | 152 | // Don't need the file now. It stays open and available to be mapped in 153 | // using the file descriptor. 154 | shm_unlink(shm_name); 155 | fd.SetFd(shm_fd); 156 | (void)close(tmpfd); 157 | return p; 158 | } 159 | 160 | static void InitMutex(pthread_mutex_t &mutex) { 161 | pthread_mutexattr_t attr; 162 | pthread_mutexattr_init(&attr); 163 | pthread_mutexattr_setpshared(&attr, 1); 164 | #ifdef __linux__ 165 | pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST); 166 | #endif 167 | 168 | pthread_mutex_init(&mutex, &attr); 169 | pthread_mutexattr_destroy(&attr); 170 | } 171 | 172 | absl::StatusOr 173 | CreateSystemControlBlock(toolbelt::FileDescriptor &fd) { 174 | absl::StatusOr s = CreateSharedMemory( 175 | 0, "scb", sizeof(SystemControlBlock), /*map=*/true, fd); 176 | if (!s.ok()) { 177 | return s.status(); 178 | } 179 | SystemControlBlock *scb = reinterpret_cast(*s); 180 | memset(&scb->counters, 0, sizeof(scb->counters)); 181 | return scb; 182 | } 183 | 184 | Channel::Channel(const std::string &name, int num_slots, int channel_id, 185 | std::string type) 186 | : name_(name), num_slots_(num_slots), channel_id_(channel_id), 187 | type_(std::move(type)) {} 188 | 189 | absl::StatusOr 190 | Channel::Allocate(const toolbelt::FileDescriptor &scb_fd, int slot_size, 191 | int num_slots) { 192 | // Unmap existing memory. 193 | Unmap(); 194 | 195 | // If the channel is being remapped (a subscriber that existed 196 | // before the first publisher), num_slots_ will be zero and we 197 | // set it here now that we know it. If num_slots_ was already 198 | // set we need to make sure that the value passed here is 199 | // the same as the current value. 200 | if (num_slots_ != 0) { 201 | assert(num_slots_ == num_slots); 202 | } else { 203 | num_slots_ = num_slots; 204 | } 205 | 206 | // We are allocating a channel, so we only have one buffer. 207 | buffers_.clear(); 208 | 209 | // Map SCB into process memory. 210 | scb_ = reinterpret_cast(MapMemory( 211 | scb_fd.Fd(), sizeof(SystemControlBlock), PROT_READ | PROT_WRITE, "SCB")); 212 | if (scb_ == MAP_FAILED) { 213 | return absl::InternalError(absl::StrFormat( 214 | "Failed to map SystemControlBlock: %s", strerror(errno))); 215 | } 216 | 217 | SharedMemoryFds fds; 218 | 219 | // One buffer. The fd will be set when the buffers are allocated in 220 | // shared memmory. 221 | fds.buffers.emplace_back(slot_size); 222 | 223 | // Create CCB in shared memory and map into process memory. 224 | int64_t ccb_size = 225 | sizeof(ChannelControlBlock) + sizeof(MessageSlot) * num_slots_; 226 | absl::StatusOr p = 227 | CreateSharedMemory(channel_id_, "ccb", ccb_size, /*map=*/true, fds.ccb); 228 | if (!p.ok()) { 229 | UnmapMemory(scb_, sizeof(SystemControlBlock), "SCB"); 230 | return p.status(); 231 | } 232 | ccb_ = reinterpret_cast(*p); 233 | 234 | // Create a single buffer but don't map it in. There is no need to 235 | // map in the buffers in the server since they will never be used. 236 | int64_t buffers_size = 237 | sizeof(BufferHeader) + 238 | num_slots_ * (Aligned<32>(slot_size) + sizeof(MessagePrefix)); 239 | if (buffers_size == 0) { 240 | buffers_size = 256; 241 | } 242 | p = CreateSharedMemory(channel_id_, "buffers0", buffers_size, 243 | /*map=*/false, fds.buffers[0].fd); 244 | if (!p.ok()) { 245 | UnmapMemory(scb_, sizeof(SystemControlBlock), "SCB"); 246 | UnmapMemory(ccb_, ccb_size, "CCB"); 247 | return p.status(); 248 | } 249 | buffers_.emplace_back(slot_size, reinterpret_cast(*p)); 250 | ccb_->num_buffers = 1; 251 | 252 | // Initialize the CCB. 253 | InitMutex(ccb_->lock); 254 | 255 | // Build CCB data. 256 | // Copy possibly truncated channel name into CCB for ease 257 | // of debugging (you can see it in all processes). 258 | strncpy(ccb_->channel_name, name_.c_str(), kMaxChannelName - 1); 259 | ccb_->num_slots = num_slots_; 260 | ccb_->next_ordinal = 1; 261 | 262 | ListInit(&ccb_->active_list); 263 | ListInit(&ccb_->busy_list); 264 | ListInit(&ccb_->free_list); 265 | 266 | // Initialize all slots and insert into the free list. 267 | for (int32_t i = 0; i < num_slots_; i++) { 268 | MessageSlot *slot = &ccb_->slots[i]; 269 | ListElementInit(&slot->element); 270 | slot->id = i; 271 | slot->ref_count = 0; 272 | slot->reliable_ref_count = 0; 273 | slot->buffer_index = -1; // No buffer in the free list. 274 | slot->owners.Init(); 275 | ListInsertAtEnd(&ccb_->free_list, &slot->element); 276 | } 277 | 278 | if (debug_) { 279 | printf("Channel allocated: scb: %p, ccb: %p, buffers: %p\n", scb_, ccb_, 280 | buffers_[0].buffer); 281 | Dump(); 282 | } 283 | return fds; 284 | } 285 | 286 | void Channel::PrintList(const SlotList *list) const { 287 | void *p = FromCCBOffset(list->first); 288 | while (p != FromCCBOffset(0)) { 289 | MessageSlot *slot = reinterpret_cast(p); 290 | printf("%d(%d/%d)@%d ", slot->id, slot->ref_count, slot->reliable_ref_count, 291 | slot->buffer_index); 292 | p = FromCCBOffset(slot->element.next); 293 | } 294 | printf("\n"); 295 | } 296 | 297 | void Channel::PrintLists() const { 298 | printf("Free list: "); 299 | PrintList(&ccb_->free_list); 300 | printf("Active list: "); 301 | PrintList(&ccb_->active_list); 302 | printf("Busy list: "); 303 | PrintList(&ccb_->busy_list); 304 | } 305 | 306 | absl::Status Channel::Map(SharedMemoryFds fds, 307 | const toolbelt::FileDescriptor &scb_fd) { 308 | scb_ = reinterpret_cast(MapMemory( 309 | scb_fd.Fd(), sizeof(SystemControlBlock), PROT_READ | PROT_WRITE, "SCB")); 310 | if (scb_ == MAP_FAILED) { 311 | return absl::InternalError(absl::StrFormat( 312 | "Failed to map SystemControlBlock: %s", strerror(errno))); 313 | } 314 | 315 | int64_t ccb_size = 316 | sizeof(ChannelControlBlock) + sizeof(MessageSlot) * num_slots_; 317 | ccb_ = reinterpret_cast( 318 | MapMemory(fds.ccb.Fd(), ccb_size, PROT_READ | PROT_WRITE, "CCB")); 319 | if (ccb_ == MAP_FAILED) { 320 | UnmapMemory(scb_, sizeof(SystemControlBlock), "SCB"); 321 | return absl::InternalError(absl::StrFormat( 322 | "Failed to map ChannelControlBlock: %s", strerror(errno))); 323 | } 324 | int index = 0; 325 | for (const auto &buffer : fds.buffers) { 326 | int64_t buffers_size = 327 | sizeof(BufferHeader) + 328 | num_slots_ * (Aligned<32>(buffer.slot_size) + sizeof(MessagePrefix)); 329 | if (buffers_size != 0) { 330 | char *mem = reinterpret_cast( 331 | MapMemory(fds.buffers[index].fd.Fd(), buffers_size, 332 | PROT_READ | PROT_WRITE, "buffers")); 333 | 334 | if (mem == MAP_FAILED) { 335 | UnmapMemory(scb_, sizeof(SystemControlBlock), "SCB"); 336 | UnmapMemory(ccb_, ccb_size, "CCB"); 337 | // Unmap any previously mapped buffers. 338 | UnmapBuffers(buffers_.begin(), buffers_.begin() + index, num_slots_); 339 | return absl::InternalError(absl::StrFormat( 340 | "Failed to map channel buffers: %s", strerror(errno))); 341 | } 342 | buffers_.emplace_back(buffer.slot_size, mem); 343 | index++; 344 | } 345 | } 346 | 347 | if (debug_) { 348 | printf("Channel mapped: scb: %p, ccb: %p\n", scb_, ccb_); 349 | Dump(); 350 | } 351 | return absl::OkStatus(); 352 | } 353 | 354 | void Channel::Unmap() { 355 | if (scb_ == nullptr) { 356 | // Not yet mapped. 357 | return; 358 | } 359 | UnmapMemory(scb_, sizeof(SystemControlBlock), "SCB"); 360 | 361 | for (auto &buffer : buffers_) { 362 | int64_t buffers_size = 363 | sizeof(BufferHeader) + 364 | num_slots_ * (Aligned<32>(buffer.slot_size) + sizeof(MessagePrefix)); 365 | if (buffers_size > 0 && buffer.buffer != nullptr) { 366 | UnmapMemory(buffer.buffer, buffers_size, "buffers"); 367 | } 368 | } 369 | buffers_.clear(); 370 | 371 | int64_t ccb_size = 372 | sizeof(ChannelControlBlock) + sizeof(MessageSlot) * num_slots_; 373 | UnmapMemory(ccb_, ccb_size, "CCB"); 374 | } 375 | 376 | // Called on server to extend the allocated buffers. 377 | absl::StatusOr 378 | Channel::ExtendBuffers(int32_t new_slot_size) { 379 | ChannelLock lock(&ccb_->lock); 380 | 381 | int64_t buffers_size = 382 | sizeof(BufferHeader) + 383 | num_slots_ * (Aligned<32>(new_slot_size) + sizeof(MessagePrefix)); 384 | 385 | char buffer_name[32]; 386 | snprintf(buffer_name, sizeof(buffer_name), "buffers%d\n", 387 | ccb_->num_buffers - 1); 388 | toolbelt::FileDescriptor fd; 389 | // Create the shared memory for the buffer but don't map it in. This is 390 | // in the server and it is not used here. The result of a successful 391 | // creation will be nullptr. 392 | absl::StatusOr p = CreateSharedMemory( 393 | channel_id_, buffer_name, buffers_size, /*map=*/false, fd); 394 | 395 | if (!p.ok()) { 396 | return absl::InternalError( 397 | absl::StrFormat("Failed to map memory for extension: %s", 398 | p.status().ToString().c_str())); 399 | } 400 | buffers_.emplace_back(new_slot_size, reinterpret_cast(*p)); 401 | ccb_->num_buffers++; 402 | return fd; 403 | } 404 | 405 | absl::Status Channel::MapNewBuffers(std::vector buffers) { 406 | size_t start = buffers_.size(); 407 | if (debug_) { 408 | printf("Mapping new buffers starting at %zd\n", start); 409 | } 410 | for (size_t i = start; i < buffers.size(); i++) { 411 | const SlotBuffer &buffer = buffers[i]; 412 | 413 | int64_t buffers_size = 414 | sizeof(BufferHeader) + 415 | num_slots_ * (Aligned<32>(buffer.slot_size) + sizeof(MessagePrefix)); 416 | if (buffers_size != 0) { 417 | char *mem = reinterpret_cast(MapMemory( 418 | buffer.fd.Fd(), buffers_size, PROT_READ | PROT_WRITE, "new buffers")); 419 | 420 | if (mem == MAP_FAILED) { 421 | // Unmap any newly mapped buffers. 422 | UnmapBuffers(buffers_.begin() + start, buffers_.end(), num_slots_); 423 | return absl::InternalError(absl::StrFormat( 424 | "Failed to map new channel buffers: %s", strerror(errno))); 425 | } 426 | buffers_.emplace_back(buffer.slot_size, mem); 427 | } 428 | } 429 | return absl::OkStatus(); 430 | } 431 | 432 | void Channel::UnmapUnusedBuffers() { 433 | for (size_t i = 0; i + 1 < buffers_.size(); i++) { 434 | if (buffers_[i].buffer == nullptr) { 435 | continue; 436 | } 437 | BufferHeader *hdr = reinterpret_cast(buffers_[i].buffer + 438 | sizeof(BufferHeader)) - 439 | 1; 440 | if (hdr->refs == 0) { 441 | int64_t buffers_size = sizeof(BufferHeader) + 442 | num_slots_ * (Aligned<32>(buffers_[i].slot_size) + 443 | sizeof(MessagePrefix)); 444 | if (buffers_size > 0) { 445 | if (debug_) { 446 | printf("%p: Unmapping unused buffers at index %zd\n", this, i); 447 | } 448 | UnmapMemory(buffers_[i].buffer, buffers_size, "buffers"); 449 | buffers_[i].buffer = nullptr; 450 | buffers_[i].slot_size = 0; 451 | } 452 | } 453 | } 454 | } 455 | 456 | void Channel::Dump() const { 457 | printf("SCB:\n"); 458 | toolbelt::Hexdump(scb_, 64); 459 | 460 | printf("CCB:\n"); 461 | int64_t ccb_size = 462 | sizeof(ChannelControlBlock) + sizeof(MessageSlot) * num_slots_; 463 | toolbelt::Hexdump(ccb_, ccb_size); 464 | PrintLists(); 465 | printf("Buffers:\n"); 466 | int index = 0; 467 | for (auto &buffer : buffers_) { 468 | printf(" (%d) %d: %p\n", index++, buffer.slot_size, buffer.buffer); 469 | } 470 | } 471 | 472 | void Channel::ClaimPublisherSlot(MessageSlot *slot, int owner, SlotList &list) { 473 | ListRemove(&list, &slot->element); 474 | AddToBusyList(slot); 475 | slot->owners.Set(owner); 476 | SetSlotToBiggestBuffer(slot); 477 | } 478 | 479 | void Channel::DecrementBufferRefs(int buffer_index) { 480 | BufferHeader *hdr = 481 | reinterpret_cast(buffers_[buffer_index].buffer + 482 | sizeof(BufferHeader)) - 483 | 1; 484 | assert(hdr->refs > 0); 485 | hdr->refs--; 486 | if (debug_) { 487 | printf("Decremented buffers refs for buffer %d to %d\n", buffer_index, 488 | hdr->refs); 489 | } 490 | } 491 | 492 | void Channel::IncrementBufferRefs(int buffer_index) { 493 | BufferHeader *hdr = 494 | reinterpret_cast(buffers_[buffer_index].buffer + 495 | sizeof(BufferHeader)) - 496 | 1; 497 | hdr->refs++; 498 | if (debug_) { 499 | printf("Incremented buffers refs for buffer %d to %d\n", buffer_index, 500 | hdr->refs); 501 | } 502 | } 503 | 504 | void Channel::SetSlotToBiggestBuffer(MessageSlot *slot) { 505 | if (slot == nullptr) { 506 | return; 507 | } 508 | if (slot->buffer_index != -1) { 509 | // If the slot has a buffer (it's not in the free list), decrement the 510 | // refs for the buffer. 511 | DecrementBufferRefs(slot->buffer_index); 512 | } 513 | slot->buffer_index = buffers_.size() - 1; // Use biggest buffer. 514 | IncrementBufferRefs(slot->buffer_index); 515 | } 516 | 517 | MessageSlot *Channel::FindFreeSlotLocked(bool reliable, int owner) { 518 | // Check if there is a free slot and if so, take it. 519 | if (ccb_->free_list.first != 0) { 520 | MessageSlot *slot = 521 | reinterpret_cast(FromCCBOffset(ccb_->free_list.first)); 522 | ClaimPublisherSlot(slot, owner, ccb_->free_list); 523 | return slot; 524 | } 525 | 526 | // No free slot, search for first slot with no references in the 527 | // active list. 528 | // If reliable is set, don't go past a slot with a reliable_ref_count 529 | // or an activation message that hasn't been seen by a subscriber. 530 | void *p = FromCCBOffset(ccb_->active_list.first); 531 | while (p != FromCCBOffset(0)) { 532 | MessageSlot *slot = reinterpret_cast(p); 533 | if (reliable && slot->reliable_ref_count != 0) { 534 | // Don't go past slot with reliable reference. 535 | return nullptr; 536 | } 537 | MessagePrefix *prefix = Prefix(slot); 538 | if (reliable && (prefix->flags & kMessageSeen) == 0) { 539 | // An message that hasn't been seen. 540 | return nullptr; 541 | } 542 | if (slot->ref_count == 0) { 543 | prefix->flags = 0; 544 | ClaimPublisherSlot(slot, owner, ccb_->active_list); 545 | return slot; 546 | } 547 | p = FromCCBOffset(slot->element.next); 548 | } 549 | return nullptr; 550 | } 551 | 552 | MessageSlot *Channel::FindFreeSlot(bool reliable, int owner, 553 | std::function reload) { 554 | ChannelLock lock(&ccb_->lock, std::move(reload)); 555 | return FindFreeSlotLocked(reliable, owner); 556 | } 557 | 558 | void Channel::GetStatsCounters(int64_t &total_bytes, int64_t &total_messages) { 559 | ChannelLock lock(&ccb_->lock); 560 | total_bytes = ccb_->total_bytes; 561 | total_messages = ccb_->total_messages; 562 | } 563 | 564 | Channel::PublishedMessage Channel::ActivateSlotAndGetAnother( 565 | MessageSlot *slot, bool reliable, bool is_activation, int owner, 566 | bool omit_prefix, bool *notify, std::function reload) { 567 | ChannelLock lock(&ccb_->lock, std::move(reload)); 568 | 569 | // Move slot from busy list to active list. 570 | ListRemove(&ccb_->busy_list, &slot->element); 571 | slot->owners.Clear(owner); 572 | AddToActiveList(slot); 573 | 574 | // If the previously last element in the active list has been seen by a 575 | // subscriber we need to notify the subscribers that we've added a new 576 | // message. If hasn't been seen, we've already notified the subscribers when 577 | // we added the slot to the active list. 578 | MessageSlot *prev = 579 | reinterpret_cast(FromCCBOffset(slot->element.prev)); 580 | if (notify != nullptr) { 581 | if (prev == FromCCBOffset(0) || (Prefix(prev)->flags & kMessageSeen) != 0) { 582 | *notify = true; 583 | } 584 | } 585 | void *buffer = GetBufferAddress(slot); 586 | MessagePrefix *prefix = reinterpret_cast(buffer) - 1; 587 | 588 | // Copy message parameters into message prefix in buffer. 589 | if (omit_prefix) { 590 | slot->ordinal = prefix->ordinal; // Copy ordinal from prefix. 591 | } else { 592 | slot->ordinal = ccb_->next_ordinal++; 593 | prefix->message_size = slot->message_size; 594 | prefix->ordinal = slot->ordinal; 595 | prefix->timestamp = toolbelt::Now(); 596 | prefix->flags = 0; 597 | if (is_activation) { 598 | prefix->flags |= kMessageActivate; 599 | } 600 | } 601 | 602 | // Update counters. 603 | ccb_->total_messages++; 604 | ccb_->total_bytes += slot->message_size; 605 | 606 | // A reliable publisher doesn't allocate a slot until it is asked for. 607 | if (reliable) { 608 | return {nullptr, prefix->ordinal, prefix->timestamp}; 609 | } 610 | // Find a new slot. 611 | return {FindFreeSlotLocked(reliable, owner), prefix->ordinal, 612 | prefix->timestamp}; 613 | } 614 | 615 | inline void IncDecRefCount(MessageSlot *slot, bool reliable, int inc) { 616 | slot->ref_count += inc; 617 | if (reliable) { 618 | slot->reliable_ref_count += inc; 619 | } 620 | } 621 | 622 | void Channel::CleanupSlots(int owner, bool reliable) { 623 | ChannelLock lock(&ccb_->lock); 624 | // Clean up active list. Remove references for any slot owned by the 625 | // owner. 626 | void *p = FromCCBOffset(ccb_->active_list.first); 627 | while (p != FromCCBOffset(0)) { 628 | MessageSlot *slot = reinterpret_cast(p); 629 | if (slot->owners.IsSet(owner)) { 630 | slot->owners.Clear(owner); 631 | IncDecRefCount(slot, reliable, -1); 632 | } 633 | 634 | p = FromCCBOffset(slot->element.next); 635 | } 636 | 637 | // Remove any publishers from the busy list. 638 | p = FromCCBOffset(ccb_->busy_list.first); 639 | while (p != FromCCBOffset(0)) { 640 | MessageSlot *slot = reinterpret_cast(p); 641 | p = FromCCBOffset(slot->element.next); 642 | 643 | if (slot->owners.IsSet(owner)) { 644 | slot->buffer_index = -1; 645 | slot->owners.Clear(owner); 646 | // Move the slot to the free list. 647 | ListRemove(&ccb_->busy_list, &slot->element); 648 | ListInsertAtEnd(&ccb_->free_list, &slot->element); 649 | } 650 | } 651 | } 652 | 653 | MessageSlot *Channel::NextSlot(MessageSlot *slot, bool reliable, int owner, 654 | std::function reload) { 655 | ChannelLock lock(&ccb_->lock, std::move(reload)); 656 | if (slot == nullptr) { 657 | // No current slot, first in list. 658 | if (ccb_->active_list.first == 0) { 659 | return nullptr; 660 | } 661 | // Take first slot in active list. 662 | slot = 663 | reinterpret_cast(FromCCBOffset(ccb_->active_list.first)); 664 | slot->owners.Set(owner); 665 | IncDecRefCount(slot, reliable, +1); 666 | Prefix(slot)->flags |= kMessageSeen; 667 | return slot; 668 | } 669 | if (slot->element.next == 0) { 670 | // No more active slots. 671 | // We need to hold onto the slot because when we call NextSlot again 672 | // for the next batch of messages this slot needs still to allocated 673 | // to the subsciber. 674 | return nullptr; 675 | } 676 | // Going to move to another slot. Decrement refs on current slot. 677 | IncDecRefCount(slot, reliable, -1); 678 | slot->owners.Clear(owner); 679 | 680 | slot = reinterpret_cast(FromCCBOffset(slot->element.next)); 681 | IncDecRefCount(slot, reliable, +1); 682 | Prefix(slot)->flags |= kMessageSeen; 683 | slot->owners.Set(owner); 684 | return slot; 685 | } 686 | 687 | MessageSlot *Channel::LastSlot(MessageSlot *slot, bool reliable, int owner, 688 | std::function reload) { 689 | ChannelLock lock(&ccb_->lock, std::move(reload)); 690 | if (ccb_->active_list.last == 0) { 691 | return nullptr; 692 | } 693 | if (slot != nullptr) { 694 | IncDecRefCount(slot, reliable, -1); 695 | slot->owners.Clear(owner); 696 | } 697 | slot = reinterpret_cast(FromCCBOffset(ccb_->active_list.last)); 698 | IncDecRefCount(slot, reliable, +1); 699 | Prefix(slot)->flags |= kMessageSeen; 700 | slot->owners.Set(owner); 701 | return slot; 702 | } 703 | 704 | MessageSlot * 705 | Channel::FindActiveSlotByTimestamp(MessageSlot *old_slot, uint64_t timestamp, 706 | bool reliable, int owner, 707 | std::vector &buffer, 708 | std::function reload) { 709 | ChannelLock lock(&ccb_->lock, std::move(reload)); 710 | 711 | // Copy pointers to active list slots into search buffer. They are already 712 | // in timestamp order. 713 | buffer.clear(); 714 | buffer.reserve(NumSlots()); 715 | void *p = FromCCBOffset(ccb_->active_list.first); 716 | while (p != FromCCBOffset(0)) { 717 | MessageSlot *slot = reinterpret_cast(p); 718 | buffer.push_back(slot); 719 | p = FromCCBOffset(slot->element.next); 720 | } 721 | // Apparently, lower_bound will return the first in the range if 722 | // the value is less than the whole range. That's unexpected. 723 | if (buffer.empty() || timestamp < Prefix(buffer.front())->timestamp) { 724 | return nullptr; 725 | } 726 | 727 | // Binary search the search buffer. 728 | auto it = std::lower_bound( 729 | buffer.begin(), buffer.end(), timestamp, 730 | [this](MessageSlot *s, uint64_t t) { return Prefix(s)->timestamp < t; }); 731 | if (it == buffer.end()) { 732 | // Not found, nothing changes. 733 | return nullptr; 734 | } 735 | if (old_slot != nullptr) { 736 | IncDecRefCount(old_slot, reliable, -1); 737 | old_slot->owners.Clear(owner); 738 | } 739 | MessageSlot *new_slot = *it; 740 | IncDecRefCount(new_slot, reliable, +1); 741 | Prefix(new_slot)->flags |= kMessageSeen; 742 | new_slot->owners.Set(owner); 743 | return new_slot; 744 | } 745 | 746 | bool Channel::LockForSharedInternal(MessageSlot *slot, int64_t ordinal, 747 | bool reliable) { 748 | ChannelLock lock(&ccb_->lock); 749 | if (slot->ordinal != ordinal) { 750 | return false; 751 | } 752 | IncDecRefCount(slot, reliable, +1); 753 | return true; 754 | } 755 | 756 | } // namespace subspace 757 | -------------------------------------------------------------------------------- /common/channel.h: -------------------------------------------------------------------------------- 1 | 2 | // Copyright 2023 David Allison 3 | // All Rights Reserved 4 | // See LICENSE file for licensing information. 5 | 6 | #ifndef __COMMON_CHANNEL_H 7 | #define __COMMON_CHANNEL_H 8 | 9 | #include 10 | 11 | #include "absl/status/status.h" 12 | #include "absl/status/statusor.h" 13 | #include "toolbelt/bitset.h" 14 | #include "toolbelt/fd.h" 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | namespace subspace { 21 | 22 | // Max message size for comms with server. 23 | static constexpr size_t kMaxMessage = 4096; 24 | 25 | // This is stored immediately before the channel buffer in shared 26 | // memory. It is transferred intact across the TCP bridges. 27 | // 32 bytes long. 28 | // 29 | // Since this is used primarily for channel bridging, we 30 | // include 4 bytes of padding at offset 0 so that that the 31 | // Socket::SendMessage function has somewhere to put the 32 | // length of the message and avoid 2 sends to the socket. 33 | // 34 | // Note that this precludes us mapping the subscriber's 35 | // channel in read-only memory since the bridge will need 36 | // to write to the padding address when it calls 37 | // Socket::SendMessage. 38 | // 39 | // On the receiving end of the bridge, the padding is 40 | // not received and will not be written to. 41 | struct MessagePrefix { 42 | int32_t padding; // Padding for Socket::SendMessage. 43 | int32_t message_size; 44 | int64_t ordinal; 45 | uint64_t timestamp; 46 | int64_t flags; 47 | }; 48 | 49 | // Flag for flags field in MessagePrefix. 50 | constexpr int kMessageActivate = 1; // This is a reliable activation message. 51 | constexpr int kMessageBridged = 2; // This message came from the bridge. 52 | constexpr int kMessageSeen = 4; // Message has been seen. 53 | 54 | // We need a max channels number because the size of things in 55 | // shared memory needs to be fixed. 56 | constexpr int kMaxChannels = 1024; 57 | 58 | // Maximum number of owners for a lot. One per subscriber reference 59 | // and publisher reference. Best if it's a multiple of 64 because 60 | // it's used as the size in a toolbelt::BitSet. 61 | constexpr int kMaxSlotOwners = 1024; 62 | 63 | // Max length of a channel name in shared memory. A name longer 64 | // this this will be truncated but the full name will be available 65 | // in process memory. 66 | constexpr size_t kMaxChannelName = 64; 67 | 68 | // This is a global (to a server) structure in shared memory that holds 69 | // counts for the number of updates to publishers and subscribers on 70 | // that server. The server updates these counts when a publisher or 71 | // subscriber is created or deleted. The purpose is to allow a client 72 | // to check if it needs to update its local information about the 73 | // channel by contacting the server. Things such as trigger file 74 | // descriptors are distributed by the server to clients. 75 | // 76 | // This is in shared memory, but it is only ever written by the 77 | // server so there is no lock required to access it in the clients. 78 | struct ChannelCounters { 79 | uint16_t num_pub_updates; // Number of updates to publishers. 80 | uint16_t num_sub_updates; // Number of updates to subscribers. 81 | uint16_t num_pubs; // Current number of publishers. 82 | uint16_t num_reliable_pubs; // Current number of reliable publishers. 83 | uint16_t num_subs; // Current number of subscribers. 84 | uint16_t num_reliable_subs; // Current number of reliable subscribers. 85 | }; 86 | 87 | // ChannelLock locks a channel and also handles reload races where 88 | // another client has changed the channel's buffers while we 89 | // were waiting for the lock. If the 'reload' function returns 90 | // true, the buffers have been changed and we have mapped 91 | // them in. We all reload in a loop because we need to unlock 92 | // the channel while we talk to the server and another client could 93 | // get in and change the buffers while we are not looking. 94 | class ChannelLock { 95 | public: 96 | ChannelLock(pthread_mutex_t *lock, 97 | std::function reload = nullptr) 98 | : lock_(lock), reload_(std::move(reload)) { 99 | Lock(); 100 | if (reload_ != nullptr) { 101 | // This will look to see if a reload is needed and if so, 102 | // unlock the channel, talk to the server and map in the 103 | // new buffers. The lock is reacquired before it returns. 104 | while (reload_(this)) { 105 | // Nothing to do, just try reloading again. 106 | } 107 | } 108 | } 109 | 110 | ~ChannelLock() { Unlock(); } 111 | 112 | void Lock() { pthread_mutex_lock(lock_); } 113 | void Unlock() { pthread_mutex_unlock(lock_); } 114 | 115 | private: 116 | pthread_mutex_t *lock_; 117 | std::function reload_; 118 | }; 119 | 120 | struct SystemControlBlock { 121 | ChannelCounters counters[kMaxChannels]; 122 | }; 123 | 124 | // Message slots are held in a double linked list, each element of 125 | // which is a SlotListElement (embedded at offset 0 in the MessageSlot 126 | // struct in shared memory). The linked lists do not use pointers 127 | // because this is in shared memory mapped at different virtual 128 | // addresses in each client. Instead they use an offset from the 129 | // start of the ChannelControlBlock (CCB) as a pointer. 130 | struct SlotListElement { 131 | int32_t prev; 132 | int32_t next; 133 | }; 134 | 135 | // Double linked list header in shared memory. 136 | struct SlotList { 137 | int32_t first; 138 | int32_t last; 139 | }; 140 | 141 | // This is the meta data for a slot. It is always in a linked list. 142 | struct MessageSlot { 143 | SlotListElement element; 144 | int32_t id; // Unique ID for slot (0...num_slots-1). 145 | int16_t ref_count; // Number of subscribers referring to this slot. 146 | int16_t reliable_ref_count; // Number of reliable subscriber references. 147 | int64_t ordinal; // Message ordinal held currently in slot. 148 | int64_t message_size; // Size of message held in slot. 149 | int32_t buffer_index; // Index of buffer. 150 | toolbelt::BitSet owners; // One bit per publisher/subscriber. 151 | }; 152 | 153 | // This is located just before the prefix of the first slot's buffer. It 154 | // is 64 bits long to align the prefix to 64 bits. 155 | struct BufferHeader { 156 | int32_t refs; // Number of references to this buffer. 157 | int32_t padding; // Align to 64 bits. 158 | }; 159 | 160 | // The control data for a channel. This memory is 161 | // allocated by the server and mapped into the process 162 | // for all publishers and subscribers. Each mapped CCB is mapped 163 | // at a virtual address chosen by the OS. 164 | // 165 | // This is in shared memory so no pointers are possible. 166 | struct ChannelControlBlock { // a.k.a CCB 167 | char channel_name[kMaxChannelName]; // So that you can see the name in a 168 | // debugger or hexdump. 169 | int num_slots; 170 | int64_t next_ordinal; // Next ordinal to use. 171 | int buffer_index; // Which buffer in buffers array to use. 172 | int num_buffers; // Size of buffers array in shared memory. 173 | 174 | // Statistics counters. 175 | int64_t total_bytes; 176 | int64_t total_messages; 177 | 178 | // Slot lists. 179 | // Active list: slots with active messages in them. 180 | // Busy list: slots allocated to publishers 181 | // Free list: slots not allocated. 182 | SlotList active_list; 183 | SlotList busy_list; 184 | SlotList free_list; 185 | 186 | pthread_mutex_t lock; // Lock for this channel only. 187 | 188 | // Variable number of MessageSlot structs (num_slots long). 189 | MessageSlot slots[0]; 190 | }; 191 | 192 | absl::StatusOr 193 | CreateSystemControlBlock(toolbelt::FileDescriptor &fd); 194 | 195 | struct SlotBuffer { 196 | SlotBuffer(int32_t slot_sz) : slot_size(slot_sz) {} 197 | SlotBuffer(int32_t slot_sz, toolbelt::FileDescriptor f) 198 | : slot_size(slot_sz), fd(std::move(f)) {} 199 | int32_t slot_size; 200 | toolbelt::FileDescriptor fd; 201 | }; 202 | 203 | // This holds the shared memory file descriptors for a channel. 204 | // ccb: Channel Control Block 205 | // buffers: message buffer memory. 206 | struct SharedMemoryFds { 207 | SharedMemoryFds() = default; 208 | SharedMemoryFds(toolbelt::FileDescriptor ccb_fd, std::vector bufs) 209 | : ccb(std::move(ccb_fd)), buffers(std::move(bufs)) {} 210 | SharedMemoryFds(const SharedMemoryFds &) = delete; 211 | 212 | SharedMemoryFds(SharedMemoryFds &&c) { 213 | ccb = std::move(c.ccb); 214 | buffers = std::move(c.buffers); 215 | } 216 | SharedMemoryFds &operator=(const SharedMemoryFds &) = delete; 217 | 218 | SharedMemoryFds &operator=(SharedMemoryFds &&c) { 219 | ccb = std::move(c.ccb); 220 | buffers = std::move(c.buffers); 221 | return *this; 222 | } 223 | 224 | toolbelt::FileDescriptor ccb; // Channel Control Block. 225 | std::vector buffers; // Message Buffers. 226 | }; 227 | 228 | // Aligned to given power of 2. 229 | template int64_t Aligned(int64_t v) { 230 | return (v + (alignment - 1)) & ~(alignment - 1); 231 | } 232 | 233 | struct BufferSet { 234 | BufferSet() = default; 235 | BufferSet(int32_t slot_sz, char *buf) : slot_size(slot_sz), buffer(buf) {} 236 | int32_t slot_size = 0; 237 | char *buffer = nullptr; 238 | }; 239 | 240 | // This is the representation of a channel as seen by a publisher 241 | // or subscriber. There is one of these objects per publisher 242 | // and per subscriber. The object is created by the client 243 | // after communicating with the server for it to allocate 244 | // the shared memory, or get the file descriptors of existing 245 | // shared memory. 246 | // 247 | // The server allocates the shared memory for a channel and 248 | // keeps the file descriptors for the POSIX shared memory, 249 | // which it distributes to the clients upon request. Clients 250 | // use mmap to map the shared memory into their address 251 | // space. If there are multiple publishers or subscribers in the 252 | // same process, each of them maps in the shared memory. No attempt 253 | // is made to share the Channel objects. 254 | class Channel : public std::enable_shared_from_this { 255 | public: 256 | struct PublishedMessage { 257 | MessageSlot *new_slot; 258 | int64_t ordinal; 259 | uint64_t timestamp; 260 | }; 261 | 262 | Channel(const std::string &name, int num_slots, int channel_id, 263 | std::string type); 264 | ~Channel() { Unmap(); } 265 | 266 | // Allocate the shared memory for a channel. The num_slots_ 267 | // and slot_size_ member variables will either be 0 (for a subscriber 268 | // to channel with no publishers), or will contain the channel 269 | // size parameters. Unless there's an error, it returns the 270 | // file descriptors for the allocated CCB and buffers. The 271 | // SCB has already been allocated and will be mapped in for 272 | // this channel. This is only used in the server. 273 | absl::StatusOr 274 | Allocate(const toolbelt::FileDescriptor &scb_fd, int slot_size, 275 | int num_slots); 276 | 277 | // Client-side channel mapping. The SharedMemoryFds contains the 278 | // file descriptors for the CCB and buffers. The num_slots_ 279 | // member variable contains either 0 or the 280 | // channel size parameters. 281 | absl::Status Map(SharedMemoryFds fds, const toolbelt::FileDescriptor &scb_fd); 282 | void Unmap(); 283 | 284 | const std::string &Name() const { return name_; } 285 | 286 | // For debug, prints the contents of the three linked lists in 287 | // shared memory, 288 | void PrintLists() const; 289 | 290 | // A placeholder is a channel created for a subscriber where there are 291 | // no publishers and thus the shared memory is not yet valid. 292 | bool IsPlaceholder() const { return NumSlots() == 0; } 293 | 294 | // What is the address of the message buffer (after the MessagePrefix) 295 | // for the slot given a slot id. 296 | void *GetBufferAddress(int slot_id) const { 297 | return Buffer(slot_id) + 298 | (sizeof(MessagePrefix) + Aligned<32>(SlotSize(slot_id))) * slot_id + 299 | sizeof(MessagePrefix); 300 | } 301 | 302 | // Gets the address for the message buffer given a slot pointer. 303 | void *GetBufferAddress(MessageSlot *slot) const { 304 | if (slot == nullptr) { 305 | return nullptr; 306 | } 307 | return Buffer(slot->id) + 308 | (sizeof(MessagePrefix) + Aligned<32>(SlotSize(slot->id))) * 309 | slot->id + 310 | sizeof(MessagePrefix); 311 | } 312 | 313 | // Find a free slot to use. This will come from the free list if there 314 | // is a free slot. Otherwise it will search for the first unreferenced slot 315 | // in the active list. If reliable is true, the search will not go past 316 | // a slot with a reliable_ref_count > 0. The owner is the ID of the 317 | // subscriber or publisher, allocated by the server. 318 | // 319 | // This locks the CCB. 320 | MessageSlot *FindFreeSlot(bool reliable, int owner, 321 | std::function reload); 322 | 323 | // A publisher is done with its busy slot (it now contains a message). The 324 | // slot is moved from the busy list to the end of the active list and other 325 | // slot is obtained, unless reliable is set to true. The new slot 326 | // is returned and this will be nullptr if reliable is true (reliable 327 | // publishers get a new slot later). The function fills in the 328 | // MessagePrefix. 329 | // 330 | // The args are: 331 | // reliable: this is for a reliable publisher, so don't allocate a new slot 332 | // is_activation: this is an activation message for a reliable publisher 333 | // owner: the ID of the publisher 334 | // omit_prefix: don't fill in the MessagePrefix (used when a message 335 | // is read from the bridge) 336 | // notify: set to true if we should notify the subscribers. 337 | // 338 | // Locks the CCB. 339 | PublishedMessage 340 | ActivateSlotAndGetAnother(MessageSlot *slot, bool reliable, 341 | bool is_activation, int owner, bool omit_prefix, 342 | bool *notify, 343 | std::function reload); 344 | 345 | // A subscriber wants to find a slot with a message in it. There are 346 | // two ways to get this: 347 | // NextSlot: gets the next slot in the active list 348 | // LastSlot: gets the last slot in the active list. 349 | // Can return nullptr if there is no slot. 350 | // If reliable is true, the reliable_ref_count in the MessageSlot will 351 | // be manipulated. The owner is the subscriber ID. 352 | // 353 | // Locks the CCB. 354 | MessageSlot *NextSlot(MessageSlot *slot, bool reliable, int owner, 355 | std::function reload); 356 | MessageSlot *LastSlot(MessageSlot *slot, bool reliable, int owner, 357 | std::function reload); 358 | 359 | // Get a pointer to the MessagePrefix for a given slot. 360 | MessagePrefix *Prefix(MessageSlot *slot) const { 361 | MessagePrefix *p = reinterpret_cast( 362 | Buffer(slot->id) + 363 | (sizeof(MessagePrefix) + Aligned<32>(SlotSize(slot->id))) * slot->id); 364 | return p; 365 | } 366 | 367 | absl::StatusOr ExtendBuffers(int32_t new_slot_size); 368 | 369 | void Dump() const; 370 | 371 | // NOTE: these functions access the CCB without locking. They only access 372 | // the buffer_index member of the MessageSlot and that is only updated by the 373 | // the publisher when it calls ClaimSlot and inserts the slot into the active 374 | // list. 375 | 376 | // Get the size associated with the given slot id. 377 | int SlotSize(int slot_id) const { 378 | return buffers_.empty() 379 | ? 0 380 | : buffers_[ccb_->slots[slot_id].buffer_index].slot_size; 381 | } 382 | 383 | int SlotSize(MessageSlot *slot) const { 384 | if (slot == nullptr) { 385 | return 0; 386 | } 387 | return buffers_.empty() 388 | ? 0 389 | : buffers_[ccb_->slots[slot->id].buffer_index].slot_size; 390 | } 391 | // Get the biggest slot size for the channel. 392 | int SlotSize() const { 393 | return buffers_.empty() ? 0 : buffers_.back().slot_size; 394 | } 395 | 396 | // Get the number of slots in the channel (can't be changed) 397 | int NumSlots() const { return num_slots_; } 398 | void SetNumSlots(int n) { num_slots_ = n; } 399 | 400 | // Get the buffer associated with the given slot id. The first buffer 401 | // starts immediately after the buffer header. 402 | char *Buffer(int slot_id) const { 403 | int index = ccb_->slots[slot_id].buffer_index; 404 | if (index < 0 || index >= buffers_.size()) { 405 | std::cerr << "Invalid buffer index for slot " << slot_id << ": " << index 406 | << std::endl; 407 | abort(); 408 | } 409 | return buffers_.empty() ? nullptr 410 | : (buffers_[index].buffer + sizeof(BufferHeader)); 411 | } 412 | void CleanupSlots(int owner, bool reliable); 413 | void UnmapUnusedBuffers(); 414 | 415 | int GetChannelId() const { return channel_id_; } 416 | 417 | int NumUpdates() const { return num_updates_; } 418 | void SetNumUpdates(int num_updates) { 419 | num_updates_ = static_cast(num_updates); 420 | } 421 | 422 | SystemControlBlock *GetScb() const { return scb_; } 423 | ChannelControlBlock *GetCcb() const { return ccb_; } 424 | 425 | // Gets the statistics counters. Locks the CCB. 426 | void GetStatsCounters(int64_t &total_bytes, int64_t &total_messages); 427 | 428 | bool LockForSharedInternal(MessageSlot *slot, int64_t ordinal, bool reliable); 429 | void SetDebug(bool v) { debug_ = v; } 430 | 431 | // Search the active list for a message with the given timestamp. If found, 432 | // move the owner to the slot found. Return nullptr if nothing found in which 433 | // no slot ownership changes are done. This uses the memory inside buffer 434 | // to perform a fast search of the slots. The caller keeps onership of the 435 | // buffer, but this function will modify it. This is to avoid memory 436 | // allocation for every search or buffer allocation for every subscriber when 437 | // searches are rare. 438 | MessageSlot * 439 | FindActiveSlotByTimestamp(MessageSlot *old_slot, uint64_t timestamp, 440 | bool reliable, int owner, 441 | std::vector &buffer, 442 | std::function reload); 443 | 444 | void SetType(std::string type) { type_ = std::move(type); } 445 | const std::string Type() const { return type_; } 446 | 447 | bool BuffersChanged() const { 448 | return ccb_->num_buffers != static_cast(buffers_.size()); 449 | } 450 | 451 | absl::Status MapNewBuffers(std::vector buffers); 452 | 453 | void SetSlotToBiggestBuffer(MessageSlot *slot); 454 | 455 | const std::vector &GetBuffers() const { return buffers_; } 456 | 457 | private: 458 | int32_t ToCCBOffset(void *addr) const { 459 | return (int32_t)(reinterpret_cast(addr) - 460 | reinterpret_cast(ccb_)); 461 | } 462 | 463 | void *FromCCBOffset(int32_t offset) const { 464 | return reinterpret_cast(ccb_) + offset; 465 | } 466 | 467 | void ListInsertAtEnd(SlotList *list, SlotListElement *e) { 468 | int32_t offset = ToCCBOffset(e); 469 | if (list->last == 0) { 470 | list->first = list->last = offset; 471 | } else { 472 | SlotListElement *last = 473 | reinterpret_cast(FromCCBOffset(list->last)); 474 | last->next = offset; 475 | e->prev = list->last; 476 | list->last = offset; 477 | } 478 | } 479 | 480 | static void ListInit(SlotList *list) { list->first = list->last = 0; } 481 | 482 | static void ListElementInit(SlotListElement *e) { e->prev = e->next = 0; } 483 | 484 | void ListRemove(SlotList *list, SlotListElement *e) { 485 | if (e->prev == 0) { 486 | list->first = e->next; 487 | } else { 488 | SlotListElement *prev = 489 | reinterpret_cast(FromCCBOffset(e->prev)); 490 | prev->next = e->next; 491 | } 492 | if (e->next == 0) { 493 | list->last = e->prev; 494 | } else { 495 | SlotListElement *next = 496 | reinterpret_cast(FromCCBOffset(e->next)); 497 | next->prev = e->prev; 498 | } 499 | e->prev = e->next = 0; 500 | } 501 | 502 | void PrintList(const SlotList *list) const; 503 | 504 | void AddToBusyList(MessageSlot *slot) { 505 | ListInsertAtEnd(&ccb_->busy_list, &slot->element); 506 | } 507 | void AddToActiveList(MessageSlot *slot) { 508 | ListInsertAtEnd(&ccb_->active_list, &slot->element); 509 | } 510 | MessageSlot *FindFreeSlotLocked(bool reliable, int owner); 511 | 512 | void ClaimPublisherSlot(MessageSlot *slot, int owner, SlotList &list); 513 | 514 | void DecrementBufferRefs(int buffer_index); 515 | void IncrementBufferRefs(int buffer_index); 516 | 517 | std::string name_; 518 | int num_slots_; 519 | 520 | int channel_id_; // ID allocated from server. 521 | std::string type_; 522 | 523 | uint16_t num_updates_ = 0; 524 | 525 | SystemControlBlock *scb_ = nullptr; 526 | ChannelControlBlock *ccb_ = nullptr; 527 | std::vector buffers_; 528 | bool debug_ = false; 529 | }; 530 | 531 | } // namespace subspace 532 | #endif /* __COMMON_CHANNEL_H */ 533 | -------------------------------------------------------------------------------- /docs/subspace.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dallison/subspace/4975a941f77278f94c52ce86b39f312d9a647c47/docs/subspace.pdf -------------------------------------------------------------------------------- /manual_tests/BUILD.bazel: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | cc_binary( 4 | name = "pub", 5 | srcs = [ 6 | "pub.cc", 7 | ], 8 | deps = [ 9 | "//client:subspace_client", 10 | "@com_google_absl//absl/status", 11 | "@com_google_absl//absl/status:statusor", 12 | "@com_google_absl//absl/strings:str_format", 13 | "@com_google_absl//absl/flags:flag", 14 | "@com_google_absl//absl/flags:parse", 15 | 16 | ], 17 | linkopts = select({ 18 | "//:macos_x86_64": [], 19 | "//:macos_arm64": [], 20 | "//:macos_default": [], 21 | "//conditions:default": [ "-lrt" ], 22 | }), 23 | ) 24 | 25 | 26 | 27 | cc_binary( 28 | name = "sub", 29 | srcs = [ 30 | "sub.cc", 31 | ], 32 | deps = [ 33 | "//client:subspace_client", 34 | "@com_google_absl//absl/status", 35 | "@com_google_absl//absl/status:statusor", 36 | "@com_google_absl//absl/strings:str_format", 37 | "@com_google_absl//absl/flags:flag", 38 | "@com_google_absl//absl/flags:parse", 39 | ], 40 | linkopts = select({ 41 | "//:macos_x86_64": [], 42 | "//:macos_arm64": [], 43 | "//:macos_default": [], 44 | "//conditions:default": [ "-lrt" ], 45 | }), 46 | ) 47 | 48 | 49 | cc_binary( 50 | name = "perf_subspace_sub", 51 | srcs = [ 52 | "perf_subspace_sub.cc", 53 | ], 54 | deps = [ 55 | "//client:subspace_client", 56 | "//common:subspace_common", 57 | "@com_google_absl//absl/status", 58 | "@com_google_absl//absl/status:statusor", 59 | "@com_google_absl//absl/strings:str_format", 60 | "@com_google_absl//absl/flags:flag", 61 | "@com_google_absl//absl/flags:parse", 62 | ], 63 | linkopts = select({ 64 | "//:macos_x86_64": [], 65 | "//:macos_arm64": [], 66 | "//:macos_default": [], 67 | "//conditions:default": [ "-lrt" ], 68 | }), 69 | ) 70 | 71 | cc_binary( 72 | name = "perf_subspace_pub", 73 | srcs = [ 74 | "perf_subspace_pub.cc", 75 | ], 76 | deps = [ 77 | "//client:subspace_client", 78 | "//common:subspace_common", 79 | "@com_google_absl//absl/status", 80 | "@com_google_absl//absl/status:statusor", 81 | "@com_google_absl//absl/strings:str_format", 82 | "@com_google_absl//absl/flags:flag", 83 | "@com_google_absl//absl/flags:parse", 84 | ], 85 | linkopts = select({ 86 | "//:macos_x86_64": [], 87 | "//:macos_arm64": [], 88 | "//:macos_default": [], 89 | "//conditions:default": [ "-lrt" ], 90 | }), 91 | ) 92 | 93 | cc_binary( 94 | name = "perf_tcp_recv", 95 | srcs = [ 96 | "perf_tcp_recv.cc", 97 | ], 98 | deps = [ 99 | "//common:subspace_common", 100 | "@com_google_absl//absl/status", 101 | "@com_google_absl//absl/status:statusor", 102 | "@com_google_absl//absl/strings:str_format", 103 | "@com_google_absl//absl/flags:flag", 104 | "@com_google_absl//absl/flags:parse", 105 | ], 106 | linkopts = select({ 107 | "//:macos_x86_64": [], 108 | "//:macos_arm64": [], 109 | "//:macos_default": [], 110 | "//conditions:default": [ "-lrt" ], 111 | }), 112 | ) 113 | 114 | cc_binary( 115 | name = "perf_tcp_send", 116 | srcs = [ 117 | "perf_tcp_send.cc", 118 | ], 119 | deps = [ 120 | "//common:subspace_common", 121 | "@com_google_absl//absl/status", 122 | "@com_google_absl//absl/status:statusor", 123 | "@com_google_absl//absl/strings:str_format", 124 | "@com_google_absl//absl/flags:flag", 125 | "@com_google_absl//absl/flags:parse", 126 | ], 127 | linkopts = select({ 128 | "//:macos_x86_64": [], 129 | "//:macos_arm64": [], 130 | "//:macos_default": [], 131 | "//conditions:default": [ "-lrt" ], 132 | }), 133 | ) 134 | 135 | cc_binary( 136 | name = "perf_subspace", 137 | srcs = [ 138 | "perf_subspace.cc", 139 | ], 140 | deps = [ 141 | "//client:subspace_client", 142 | "@com_google_absl//absl/status", 143 | "@com_google_absl//absl/status:statusor", 144 | "@com_google_absl//absl/strings:str_format", 145 | "@com_google_absl//absl/flags:flag", 146 | "@com_google_absl//absl/flags:parse", 147 | "@coroutines//:co", 148 | ], 149 | linkopts = select({ 150 | "//:macos_x86_64": [], 151 | "//:macos_arm64": [], 152 | "//:macos_default": [], 153 | "//conditions:default": [ "-lrt" ], 154 | }), 155 | ) 156 | -------------------------------------------------------------------------------- /manual_tests/perf_subspace.cc: -------------------------------------------------------------------------------- 1 | // This is a single process reliable pub/sub performance test. 2 | #include "absl/flags/flag.h" 3 | #include "absl/flags/parse.h" 4 | #include "client/client.h" 5 | #include "toolbelt/clock.h" 6 | #include "toolbelt/hexdump.h" 7 | #include "coroutine.h" 8 | #include 9 | #include 10 | 11 | ABSL_FLAG(std::string, socket, "/tmp/subspace", 12 | "Name of Unix socket to listen on"); 13 | ABSL_FLAG(int, num_msgs, 1000, "Number of messages to measure"); 14 | ABSL_FLAG(int, slot_size, 4096, "Size of a slot"); 15 | ABSL_FLAG(bool, csv, false, "CSV output"); 16 | ABSL_FLAG(int, num_slots, 500, "Number of slots in channel"); 17 | 18 | void PubCoroutine(co::Coroutine *c) { 19 | subspace::Client client(c); 20 | absl::Status init_status = client.Init(absl::GetFlag(FLAGS_socket)); 21 | if (!init_status.ok()) { 22 | fprintf(stderr, "Can't connect to Subspace server: %s\n", 23 | init_status.ToString().c_str()); 24 | exit(1); 25 | } 26 | int num_slots = absl::GetFlag(FLAGS_num_slots); 27 | int slot_size = absl::GetFlag(FLAGS_slot_size); 28 | 29 | absl::StatusOr pub = client.CreatePublisher( 30 | "test", slot_size, num_slots, 31 | subspace::PublisherOptions().SetLocal(true).SetReliable(true)); 32 | if (!pub.ok()) { 33 | fprintf(stderr, "Can't create publisher: %s\n", 34 | pub.status().ToString().c_str()); 35 | exit(1); 36 | } 37 | int num_msgs = absl::GetFlag(FLAGS_num_msgs); 38 | 39 | for (int i = 0; i < num_msgs; i++) { 40 | for (;;) { 41 | absl::StatusOr buffer = pub->GetMessageBuffer(); 42 | if (!buffer.ok()) { 43 | fprintf(stderr, "Can't get publisher buffer: %s\n", 44 | buffer.status().ToString().c_str()); 45 | exit(1); 46 | } 47 | if (*buffer == nullptr) { 48 | // Wait for publisher trigger. 49 | absl::Status wait_status = pub->Wait(); 50 | if (!wait_status.ok()) { 51 | fprintf(stderr, "Can't wait for publisher: %s", wait_status.ToString().c_str()); 52 | exit(1); 53 | } 54 | continue; 55 | } 56 | break; 57 | } 58 | // printf("publishing %d\n", i); 59 | absl::StatusOr status = pub->PublishMessage(slot_size); 60 | if (!status.ok()) { 61 | fprintf(stderr, "Can't publish message: %s\n", status.status().ToString().c_str()); 62 | exit(1); 63 | } 64 | } 65 | } 66 | 67 | void SubCoroutine(co::Coroutine *c) { 68 | subspace::Client client(c); 69 | absl::Status init_status = client.Init(absl::GetFlag(FLAGS_socket)); 70 | if (!init_status.ok()) { 71 | fprintf(stderr, "Can't connect to Subspace server: %s\n", 72 | init_status.ToString().c_str()); 73 | exit(1); 74 | } 75 | int slot_size = absl::GetFlag(FLAGS_slot_size); 76 | 77 | absl::StatusOr sub = client.CreateSubscriber( 78 | "test", subspace::SubscriberOptions().SetReliable(true)); 79 | if (!sub.ok()) { 80 | fprintf(stderr, "Can't create subscriber: %s\n", 81 | sub.status().ToString().c_str()); 82 | exit(1); 83 | } 84 | 85 | sub->RegisterDroppedMessageCallback( 86 | [](subspace::Subscriber *s, int64_t n) { 87 | printf("Dropped %" PRId64 " messages\n", n); 88 | }); 89 | 90 | int num_msgs = absl::GetFlag(FLAGS_num_msgs); 91 | uint64_t start = 0; 92 | int64_t total_bytes = 0; 93 | uint64_t total_wait = 0; 94 | int i = 0; 95 | while (i < num_msgs) { 96 | uint64_t wait_start = 0; 97 | if (start != 0) { 98 | wait_start = toolbelt::Now(); 99 | } 100 | if (absl::Status wait_status = sub->Wait(); !wait_status.ok()) { 101 | fprintf(stderr, "Can't wait for subscriber: %s\n", wait_status.ToString().c_str()); 102 | exit(1); 103 | } 104 | if (wait_start != 0) { 105 | total_wait += toolbelt::Now() - wait_start; 106 | } 107 | for (;;) { 108 | absl::StatusOr msg = sub->ReadMessage(); 109 | if (!msg.ok()) { 110 | fprintf(stderr, "Can't read message: %s\n", 111 | msg.status().ToString().c_str()); 112 | exit(1); 113 | } 114 | if (msg->length == 0) { 115 | break; 116 | } 117 | if (start == 0) { 118 | start = toolbelt::Now(); 119 | } 120 | // printf("got %d\n", i); 121 | total_bytes += msg->length; 122 | i++; 123 | } 124 | } 125 | uint64_t end = toolbelt::Now(); 126 | double period = (end - start - total_wait) / 1e9; 127 | double msg_rate = num_msgs / period; 128 | double byte_rate = total_bytes / period; 129 | double latency = (period * 1e6) / num_msgs; 130 | if (absl::GetFlag(FLAGS_csv)) { 131 | printf("%d,%d,%g,%g,%g,%g\n", slot_size, num_msgs, period, msg_rate, 132 | latency, byte_rate); 133 | } else { 134 | printf("Subspace: %d bytes, %d messages received in %gs, %g msgs/sec, " 135 | "latency: %gus. %g bytes/sec\n", 136 | slot_size, num_msgs, period, msg_rate, latency, byte_rate); 137 | } 138 | } 139 | 140 | int main(int argc, char **argv) { 141 | absl::ParseCommandLine(argc, argv); 142 | signal(SIGPIPE, SIG_IGN); 143 | 144 | co::CoroutineScheduler scheduler; 145 | co::Coroutine pub(scheduler, PubCoroutine); 146 | co::Coroutine sub(scheduler, SubCoroutine); 147 | 148 | scheduler.Run(); 149 | } 150 | -------------------------------------------------------------------------------- /manual_tests/perf_subspace_pub.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Allison 2 | // All Rights Reserved 3 | // See LICENSE file for licensing information. 4 | 5 | #include "absl/flags/flag.h" 6 | #include "absl/flags/parse.h" 7 | #include "client/client.h" 8 | #include 9 | #include 10 | 11 | ABSL_FLAG(std::string, socket, "/tmp/subspace", 12 | "Name of Unix socket to listen on"); 13 | ABSL_FLAG(int, num_msgs, 1000, "Number of messages to send"); 14 | ABSL_FLAG(bool, reliable, true, "Use reliable transport"); 15 | ABSL_FLAG(int, num_slots, 500, "Number of slots in channel"); 16 | ABSL_FLAG(int, slot_size, 4096, "Size of a slot"); 17 | 18 | int main(int argc, char **argv) { 19 | absl::ParseCommandLine(argc, argv); 20 | signal(SIGPIPE, SIG_IGN); 21 | 22 | subspace::Client client; 23 | absl::Status init_status = client.Init(absl::GetFlag(FLAGS_socket)); 24 | if (!init_status.ok()) { 25 | fprintf(stderr, "Can't connect to Subspace server: %s\n", init_status.ToString().c_str()); 26 | exit(1); 27 | } 28 | bool reliable = absl::GetFlag(FLAGS_reliable); 29 | int num_slots = absl::GetFlag(FLAGS_num_slots); 30 | int slot_size = absl::GetFlag(FLAGS_slot_size); 31 | 32 | absl::StatusOr pub = client.CreatePublisher( 33 | "test", slot_size, num_slots, 34 | subspace::PublisherOptions().SetLocal(true).SetReliable(reliable)); 35 | if (!pub.ok()) { 36 | fprintf(stderr, "Can't create publisher: %s\n", 37 | pub.status().ToString().c_str()); 38 | exit(1); 39 | } 40 | int num_msgs = absl::GetFlag(FLAGS_num_msgs); 41 | 42 | for (int i = 0; i < num_msgs; i++) { 43 | for (;;) { 44 | absl::StatusOr buffer = pub->GetMessageBuffer(); 45 | if (!buffer.ok()) { 46 | fprintf(stderr, "Can't get publisher buffer: %s\n", 47 | buffer.status().ToString().c_str()); 48 | exit(1); 49 | } 50 | if (*buffer == nullptr) { 51 | // Wait for publisher trigger. 52 | absl::Status wait_status = pub->Wait(); 53 | if (!wait_status.ok()) { 54 | fprintf(stderr, "Can't wait for publisher: %s", wait_status.ToString().c_str()); 55 | exit(1); 56 | } 57 | continue; 58 | } 59 | break; 60 | } 61 | //printf("publishing %d\n", i); 62 | absl::StatusOr status = pub->PublishMessage(slot_size); 63 | if (!status.ok()) { 64 | fprintf(stderr, "Can't publish message: %s\n", status.status().ToString().c_str()); 65 | exit(1); 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /manual_tests/perf_subspace_sub.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Allison 2 | // All Rights Reserved 3 | // See LICENSE file for licensing information. 4 | 5 | #include "absl/flags/flag.h" 6 | #include "absl/flags/parse.h" 7 | #include "client/client.h" 8 | #include "toolbelt/clock.h" 9 | #include 10 | #include 11 | 12 | ABSL_FLAG(std::string, socket, "/tmp/subspace", 13 | "Name of Unix socket to listen on"); 14 | ABSL_FLAG(bool, reliable, true, "Use reliable transport"); 15 | ABSL_FLAG(int, num_msgs, 1000, "Number of messages to measure"); 16 | ABSL_FLAG(int, slot_size, 4096, "Size of a slot"); 17 | ABSL_FLAG(bool, csv, false, "CSV output"); 18 | 19 | int main(int argc, char **argv) { 20 | absl::ParseCommandLine(argc, argv); 21 | signal(SIGPIPE, SIG_IGN); 22 | 23 | subspace::Client client; 24 | absl::Status init_status = client.Init(absl::GetFlag(FLAGS_socket)); 25 | if (!init_status.ok()) { 26 | fprintf(stderr, "Can't connect to Subspace server: %s\n", 27 | init_status.ToString().c_str()); 28 | exit(1); 29 | } 30 | bool reliable = absl::GetFlag(FLAGS_reliable); 31 | int slot_size = absl::GetFlag(FLAGS_slot_size); 32 | 33 | absl::StatusOr sub = client.CreateSubscriber( 34 | "test", subspace::SubscriberOptions().SetReliable(reliable)); 35 | if (!sub.ok()) { 36 | fprintf(stderr, "Can't create subscriber: %s\n", 37 | sub.status().ToString().c_str()); 38 | exit(1); 39 | } 40 | 41 | sub->RegisterDroppedMessageCallback( 42 | [](subspace::Subscriber *s, int64_t n) { 43 | printf("Dropped %" PRId64 " messages\n", n); 44 | }); 45 | 46 | int num_msgs = absl::GetFlag(FLAGS_num_msgs); 47 | uint64_t start = 0; 48 | int64_t total_bytes = 0; 49 | uint64_t total_wait = 0; 50 | 51 | int i = 0; 52 | while (i < num_msgs) { 53 | uint64_t wait_start = 0; 54 | if (start != 0) { 55 | wait_start = toolbelt::Now(); 56 | } 57 | if (absl::Status wait_status = sub->Wait(); !wait_status.ok()) { 58 | fprintf(stderr, "Can't wait for subscriber: %s\n", wait_status.ToString().c_str()); 59 | exit(1); 60 | } 61 | if (wait_start != 0) { 62 | total_wait += toolbelt::Now() - wait_start; 63 | } 64 | for (;;) { 65 | absl::StatusOr msg = sub->ReadMessage(); 66 | if (!msg.ok()) { 67 | fprintf(stderr, "Can't read message: %s\n", 68 | msg.status().ToString().c_str()); 69 | exit(1); 70 | } 71 | if (msg->length == 0) { 72 | break; 73 | } 74 | if (start == 0) { 75 | start = toolbelt::Now(); 76 | } 77 | // printf("%d\n", i); 78 | total_bytes += msg->length; 79 | i++; 80 | } 81 | } 82 | uint64_t end = toolbelt::Now(); 83 | double period = (end - start - total_wait) / 1e9; 84 | double msg_rate = num_msgs / period; 85 | double byte_rate = total_bytes / period; 86 | double latency = (period * 1e6) / num_msgs; 87 | if (absl::GetFlag(FLAGS_csv)) { 88 | printf("%d,%d,%g,%g,%g,%g\n", slot_size, num_msgs, period, msg_rate, 89 | latency, byte_rate); 90 | } else { 91 | printf("Subspace: %d bytes, %d messages received in %gs, %g msgs/sec, " 92 | "latency: %gus. %g bytes/sec\n", 93 | slot_size, num_msgs, period, msg_rate, latency, byte_rate); 94 | } 95 | } -------------------------------------------------------------------------------- /manual_tests/perf_tcp_recv.cc: -------------------------------------------------------------------------------- 1 | #include "toolbelt/sockets.h" 2 | 3 | #include "absl/flags/flag.h" 4 | #include "absl/flags/parse.h" 5 | #include "toolbelt/clock.h" 6 | #include 7 | #include 8 | 9 | ABSL_FLAG(int, num_msgs, 1000, "Number of messages to measure"); 10 | ABSL_FLAG(int, msg_size, 1000, "Size of each message"); 11 | ABSL_FLAG(std::string, hostname, "localhost", "Hostname"); 12 | ABSL_FLAG(bool, csv, false, "CSV output"); 13 | 14 | int main(int argc, char **argv) { 15 | absl::ParseCommandLine(argc, argv); 16 | signal(SIGPIPE, SIG_IGN); 17 | 18 | toolbelt::TCPSocket socket; 19 | std::string hostname = absl::GetFlag(FLAGS_hostname); 20 | 21 | if (absl::Status s = 22 | socket.Bind(toolbelt::InetAddress(hostname.c_str(), 6522), true); 23 | !s.ok()) { 24 | fprintf(stderr, "Failed to bind to localhost: %s\n", s.ToString().c_str()); 25 | exit(1); 26 | } 27 | if (absl::Status status = socket.SetReuseAddr(); !status.ok()) { 28 | fprintf(stderr, "%s", status.ToString().c_str()); 29 | exit(1); 30 | } 31 | absl::StatusOr s = socket.Accept(); 32 | if (!s.ok()) { 33 | fprintf(stderr, "Failed to accept: %s\n", s.status().ToString().c_str()); 34 | exit(1); 35 | } 36 | int msg_size = absl::GetFlag(FLAGS_msg_size); 37 | int num_msgs = absl::GetFlag(FLAGS_num_msgs); 38 | void *memory = malloc(msg_size); 39 | 40 | uint64_t start = 0; 41 | int64_t total_bytes = 0; 42 | 43 | for (int i = 0; i < num_msgs; i++) { 44 | absl::StatusOr n = 45 | s->Receive(reinterpret_cast(memory), msg_size); 46 | if (!n.ok()) { 47 | fprintf(stderr, "Failed to read: %s\n", n.status().ToString().c_str()); 48 | exit(1); 49 | } 50 | if (start == 0) { 51 | start = toolbelt::Now(); 52 | } 53 | total_bytes += *n; 54 | } 55 | free(memory); 56 | 57 | uint64_t end = toolbelt::Now(); 58 | double period = (end - start) / 1e9; 59 | double msg_rate = num_msgs / period; 60 | double byte_rate = total_bytes / period; 61 | double latency = (period * 1e6) / num_msgs; 62 | if (absl::GetFlag(FLAGS_csv)) { 63 | printf("%d,%d,%g,%g,%g,%g\n", msg_size, num_msgs, period, msg_rate, latency, 64 | byte_rate); 65 | } else { 66 | printf("TCP: %d bytes, %d messages received in %gs, %g msgs/sec, latency: " 67 | "%gus. %g bytes/sec\n", 68 | msg_size, num_msgs, period, msg_rate, latency, byte_rate); 69 | } 70 | } -------------------------------------------------------------------------------- /manual_tests/perf_tcp_send.cc: -------------------------------------------------------------------------------- 1 | #include "toolbelt/sockets.h" 2 | 3 | #include "absl/flags/flag.h" 4 | #include "absl/flags/parse.h" 5 | #include "toolbelt/clock.h" 6 | #include 7 | #include 8 | 9 | ABSL_FLAG(int, num_msgs, 1000, "Number of messages to send"); 10 | ABSL_FLAG(int, msg_size, 1000, "Size of each message"); 11 | ABSL_FLAG(std::string, hostname, "localhost", "Hostname"); 12 | 13 | int main(int argc, char **argv) { 14 | absl::ParseCommandLine(argc, argv); 15 | signal(SIGPIPE, SIG_IGN); 16 | 17 | toolbelt::TCPSocket socket; 18 | std::string hostname = absl::GetFlag(FLAGS_hostname); 19 | 20 | absl::Status status = socket.Connect(toolbelt::InetAddress(hostname.c_str(), 6522)); 21 | if (!status.ok()) { 22 | fprintf(stderr, "Failed to connect: %s\n", status.ToString().c_str()); 23 | exit(1); 24 | } 25 | int msg_size = absl::GetFlag(FLAGS_msg_size); 26 | int num_msgs = absl::GetFlag(FLAGS_num_msgs); 27 | void *memory = malloc(msg_size); 28 | 29 | for (int i = 0; i < num_msgs; i++) { 30 | absl::StatusOr n = 31 | socket.Send(reinterpret_cast(memory), msg_size); 32 | if (!n.ok()) { 33 | fprintf(stderr, "Failed to read: %s\n", n.status().ToString().c_str()); 34 | exit(1); 35 | } 36 | } 37 | free(memory); 38 | } -------------------------------------------------------------------------------- /manual_tests/perf_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ipaddr= 4 | subspace=true 5 | localhost=true 6 | net=true 7 | 8 | if (( $# >= 1 )); then 9 | ipaddr=$1 10 | shift 11 | else 12 | echo "usage: perf_test ipaddr" 13 | exit 1 14 | fi 15 | 16 | for arg in "$*"; do 17 | case "$arg" in 18 | --no-subspace) 19 | subspace=false 20 | ;; 21 | --no-localhost) 22 | localhost=false 23 | ;; 24 | --no-net) 25 | net=false 26 | ;; 27 | esac 28 | done 29 | 30 | RUN_FAST= 31 | if [[ $(uname) == "Darwin" ]]; then 32 | RUN_FAST="taskpolicy -c utility" 33 | fi 34 | 35 | function run_subspace() { 36 | size=$1 37 | num=$2 38 | $RUN_FAST bazel-bin/manual_tests/perf_subspace_pub --reliable --num_msgs=$num --slot_size=$size & 39 | $RUN_FAST bazel-bin/manual_tests/perf_subspace_sub --reliable --num_msgs=$num --slot_size=$size --csv & 40 | wait 41 | } 42 | 43 | function run_tcp() { 44 | size=$1 45 | host=$2 46 | num=$3 47 | pkill perf_tcp_recv 48 | pkill perf_tcp_send 49 | ps -ef | grep perf_tcp_recv 50 | ps -ef | grep perf_tcp_send 51 | $RUN_FAST bazel-bin/manual_tests/perf_tcp_recv --num_msgs=$num --hostname=$host --msg_size=$size --csv & 52 | sleep 1 53 | $RUN_FAST bazel-bin/manual_tests/perf_tcp_send --num_msgs=$num --hostname=$host --msg_size=$size & 54 | wait 55 | sleep 1 56 | } 57 | 58 | 59 | DIR=${HOME}/Documents 60 | subspace_out=${DIR}/subspace.csv 61 | tcp_localhost_out=${DIR}/localhost.csv 62 | tcp_net_out=${DIR}/net.csv 63 | rm -f $subspace_out $tcp_localhost_out $tcp_net_out 64 | 65 | if [[ $subspace == true ]]; then 66 | echo Subspace 67 | size=1000 68 | num=1000000 69 | max=100000000 70 | while (( size < $max )); do 71 | echo $size 72 | run_subspace $size $num >> $subspace_out 73 | size=$((size*2)) 74 | done 75 | fi 76 | 77 | if [[ $localhost == true ]]; then 78 | echo TCP localhost 79 | size=1000 80 | num=10000 81 | max=8000000 82 | while (( size < $max )); do 83 | echo $size 84 | run_tcp $size localhost $num>> $tcp_localhost_out 85 | size=$((size*2)) 86 | done 87 | fi 88 | 89 | if [[ $net == true ]]; then 90 | echo TCP net 91 | size=1000 92 | num=10000 93 | max=8000000 94 | while (( size < $max )); do 95 | echo $size 96 | run_tcp $size $ipaddr $num>> $tcp_net_out 97 | size=$((size*2)) 98 | done 99 | fi 100 | 101 | -------------------------------------------------------------------------------- /manual_tests/pub.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Allison 2 | // All Rights Reserved 3 | // See LICENSE file for licensing information. 4 | 5 | #include "absl/flags/flag.h" 6 | #include "absl/flags/parse.h" 7 | #include "client/client.h" 8 | #include 9 | 10 | ABSL_FLAG(std::string, socket, "/tmp/subspace", 11 | "Name of Unix socket to listen on"); 12 | ABSL_FLAG(int, num_msgs, 1, "Number of messages to send"); 13 | ABSL_FLAG(double, frequency, 0, "Freqency to send at (Hz)"); 14 | ABSL_FLAG(bool, reliable, false, "Use reliable transport"); 15 | ABSL_FLAG(int, num_slots, 5, "Number of slots in channel"); 16 | 17 | int main(int argc, char **argv) { 18 | absl::ParseCommandLine(argc, argv); 19 | 20 | subspace::Client client; 21 | absl::Status init_status = client.Init(absl::GetFlag(FLAGS_socket)); 22 | if (!init_status.ok()) { 23 | fprintf(stderr, "Can't connect to Subspace server: %s\n", 24 | init_status.ToString().c_str()); 25 | exit(1); 26 | } 27 | bool reliable = absl::GetFlag(FLAGS_reliable); 28 | int num_slots = absl::GetFlag(FLAGS_num_slots); 29 | 30 | absl::StatusOr pub = client.CreatePublisher( 31 | "test", 256, num_slots, 32 | subspace::PublisherOptions().SetLocal(true).SetReliable(reliable)); 33 | if (!pub.ok()) { 34 | fprintf(stderr, "Can't create publisher: %s\n", 35 | pub.status().ToString().c_str()); 36 | exit(1); 37 | } 38 | int num_msgs = absl::GetFlag(FLAGS_num_msgs); 39 | double frequency = absl::GetFlag(FLAGS_frequency); 40 | 41 | int delay_ns = 0; 42 | 43 | for (;;) { 44 | if (frequency <= 0) { 45 | printf("Sending %d message%s as fast as possible\n", num_msgs, 46 | num_msgs == 1 ? "s" : ""); 47 | } else { 48 | delay_ns = 1000000000 / frequency; 49 | printf("Sending %d message%s at %gHz\n", num_msgs, 50 | num_msgs == 1 ? "s" : "", frequency); 51 | } 52 | struct timespec delay = {.tv_sec = delay_ns / 1000000000, 53 | .tv_nsec = delay_ns % 1000000000}; 54 | 55 | for (int i = 0; i < num_msgs; i++) { 56 | void *buf = nullptr; 57 | for (;;) { 58 | absl::StatusOr buffer = pub->GetMessageBuffer(); 59 | if (!buffer.ok()) { 60 | fprintf(stderr, "Can't get publisher buffer: %s\n", 61 | buffer.status().ToString().c_str()); 62 | exit(1); 63 | } 64 | if (*buffer == nullptr) { 65 | // Wait for publisher trigger. 66 | absl::Status wait_status = pub->Wait(); 67 | if (!wait_status.ok()) { 68 | fprintf(stderr, "Can't wait for publisher: %s", 69 | wait_status.ToString().c_str()); 70 | exit(1); 71 | } 72 | continue; 73 | } 74 | buf = *buffer; 75 | break; 76 | } 77 | size_t len = snprintf(reinterpret_cast(buf), 256, "%s", "hello"); 78 | absl::StatusOr status = 79 | pub->PublishMessage(len); 80 | if (!status.ok()) { 81 | fprintf(stderr, "Can't publish message: %s\n", 82 | status.status().ToString().c_str()); 83 | exit(1); 84 | } 85 | if (i < (num_msgs - 1)) { 86 | nanosleep(&delay, nullptr); 87 | } 88 | } 89 | printf("All messages sent, hit return to do it again, ^C to stop\n"); 90 | getchar(); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /manual_tests/sub.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Allison 2 | // All Rights Reserved 3 | // See LICENSE file for licensing information. 4 | 5 | #include "absl/flags/flag.h" 6 | #include "absl/flags/parse.h" 7 | #include "client/client.h" 8 | #include 9 | #include 10 | 11 | ABSL_FLAG(std::string, socket, "/tmp/subspace", 12 | "Name of Unix socket to listen on"); 13 | ABSL_FLAG(bool, reliable, false, "Use reliable transport"); 14 | 15 | int main(int argc, char **argv) { 16 | absl::ParseCommandLine(argc, argv); 17 | signal(SIGPIPE, SIG_IGN); 18 | 19 | subspace::Client client; 20 | 21 | absl::Status init_status = client.Init(absl::GetFlag(FLAGS_socket)); 22 | if (!init_status.ok()) { 23 | fprintf(stderr, "Can't connect to Subspace server: %s\n", init_status.ToString().c_str()); 24 | exit(1); 25 | } 26 | bool reliable = absl::GetFlag(FLAGS_reliable); 27 | 28 | absl::StatusOr sub = client.CreateSubscriber( 29 | "test", subspace::SubscriberOptions().SetReliable(reliable)); 30 | if (!sub.ok()) { 31 | fprintf(stderr, "Can't create subscriber: %s\n", 32 | sub.status().ToString().c_str()); 33 | exit(1); 34 | } 35 | 36 | sub->RegisterDroppedMessageCallback( 37 | [](subspace::Subscriber *s, int64_t n) { 38 | printf("Dropped %" PRId64 " messages\n", n); 39 | }); 40 | 41 | for (;;) { 42 | if (absl::Status wait_status = sub->Wait(); !wait_status.ok()) { 43 | fprintf(stderr, "Can't wait for subscriber: %s\n", wait_status.ToString().c_str()); 44 | exit(1); 45 | } 46 | for (;;) { 47 | absl::StatusOr msg = sub->ReadMessage(); 48 | if (!msg.ok()) { 49 | fprintf(stderr, "Can't read message: %s\n", 50 | msg.status().ToString().c_str()); 51 | exit(1); 52 | } 53 | if (msg->length == 0) { 54 | break; 55 | } 56 | int64_t ordinal = sub->GetCurrentOrdinal(); 57 | printf("Message %" PRId64 ": %s\n", ordinal, 58 | reinterpret_cast(msg->buffer)); 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /proto/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") 2 | load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") 3 | 4 | package(default_visibility = ["//visibility:public"]) 5 | 6 | proto_library( 7 | name = "subspace_proto", 8 | srcs = ["subspace.proto"], 9 | ) 10 | 11 | cc_proto_library( 12 | name = "subspace_cc_proto", 13 | deps = [":subspace_proto"], 14 | ) 15 | 16 | -------------------------------------------------------------------------------- /proto/subspace.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Allison 2 | // All Rights Reserved 3 | // See LICENSE file for licensing information. 4 | 5 | syntax = "proto3"; 6 | 7 | package subspace; 8 | 9 | message InitRequest { string client_name = 1; } 10 | 11 | // All Request and Response messages are carried over a Unix Domain 12 | // Socket between the client and the server. They are followed 13 | // by an array of file descriptors (SCM_RIGHTS message) that 14 | // are duped from the server. These fds are for shared memory 15 | // segments and trigger fds. The messages contain indices into 16 | // the fd array. 17 | message InitResponse { 18 | int32 scb_fd_index = 1; // Index into fds of SystemControlBlock fd. 19 | } 20 | 21 | message BufferInfo { 22 | int32 slot_size = 1; 23 | int32 fd_index = 2; 24 | } 25 | 26 | message CreatePublisherRequest { 27 | string channel_name = 1; 28 | int32 num_slots = 2; 29 | int32 slot_size = 3; 30 | bool is_local = 4; 31 | bool is_reliable = 5; 32 | bool is_bridge = 6; // This publisher is for the bridge. 33 | bytes type = 7; // Type of data carried on channel. 34 | bool is_fixed_size = 8; 35 | } 36 | 37 | message CreatePublisherResponse { 38 | string error = 1; 39 | int32 channel_id = 2; 40 | int32 publisher_id = 3; 41 | int32 ccb_fd_index = 4; 42 | repeated BufferInfo buffers = 5; 43 | int32 pub_poll_fd_index = 6; 44 | int32 pub_trigger_fd_index = 7; 45 | repeated int32 sub_trigger_fd_indexes = 8; 46 | int32 num_sub_updates = 9; 47 | bytes type = 10; 48 | } 49 | 50 | // This is used both to create a new subscriber and to reload 51 | // an existing one. If channel_id is not -1, it refers to 52 | // an existing subscriber to the channel. 53 | message CreateSubscriberRequest { 54 | string channel_name = 1; 55 | int32 subscriber_id = 2; // -1 if there is no existing subscriber. 56 | bool is_reliable = 3; 57 | bool is_bridge = 4; // This subscriber is for the bridge. 58 | bytes type = 5; // Type of data carried on channel. 59 | int32 max_shared_ptrs = 6; // Max number of shared_ptr objects. 60 | } 61 | 62 | message CreateSubscriberResponse { 63 | string error = 1; 64 | int32 channel_id = 2; 65 | int32 subscriber_id = 3; 66 | int32 ccb_fd_index = 4; 67 | repeated BufferInfo buffers = 5; 68 | int32 trigger_fd_index = 6; 69 | int32 poll_fd_index = 7; 70 | int32 slot_size = 8; // Might be zero if no publisher. 71 | int32 num_slots = 9; // Might be zero if no publisher. 72 | repeated int32 reliable_pub_trigger_fd_indexes = 10; 73 | int32 num_pub_updates = 11; 74 | bytes type = 12; 75 | } 76 | 77 | message GetTriggersRequest { string channel_name = 1; } 78 | 79 | message GetTriggersResponse { 80 | string error = 1; 81 | repeated int32 reliable_pub_trigger_fd_indexes = 2; 82 | repeated int32 sub_trigger_fd_indexes = 3; 83 | } 84 | 85 | message RemovePublisherRequest { 86 | string channel_name = 1; 87 | int32 publisher_id = 2; 88 | } 89 | 90 | message RemovePublisherResponse { string error = 1; } 91 | 92 | message RemoveSubscriberRequest { 93 | string channel_name = 1; 94 | int32 subscriber_id = 2; 95 | } 96 | 97 | message RemoveSubscriberResponse { string error = 1; } 98 | 99 | message ResizeRequest { 100 | string channel_name = 1; 101 | int32 new_slot_size = 2; 102 | } 103 | 104 | message ResizeResponse { 105 | string error = 1; 106 | repeated BufferInfo buffers = 2; 107 | int32 slot_size = 3; // Will be equal to new_slot_size if success. 108 | } 109 | 110 | message GetBuffersRequest { 111 | string channel_name = 1; 112 | } 113 | 114 | message GetBuffersResponse { 115 | string error = 1; 116 | repeated BufferInfo buffers = 2; 117 | } 118 | 119 | message Request { 120 | oneof request { 121 | InitRequest init = 1; 122 | CreatePublisherRequest create_publisher = 2; 123 | CreateSubscriberRequest create_subscriber = 3; 124 | GetTriggersRequest get_triggers = 4; 125 | RemovePublisherRequest remove_publisher = 5; 126 | RemoveSubscriberRequest remove_subscriber = 6; 127 | ResizeRequest resize = 7; 128 | GetBuffersRequest get_buffers = 8; 129 | } 130 | } 131 | 132 | message Response { 133 | oneof response { 134 | InitResponse init = 1; 135 | CreatePublisherResponse create_publisher = 2; 136 | CreateSubscriberResponse create_subscriber = 3; 137 | GetTriggersResponse get_triggers = 4; 138 | RemovePublisherResponse remove_publisher = 5; 139 | RemoveSubscriberResponse remove_subscriber = 6; 140 | ResizeResponse resize = 7; 141 | GetBuffersResponse get_buffers = 8; 142 | } 143 | } 144 | 145 | // These messages are carried on Subspace channels published by 146 | // the server. 147 | message ChannelInfo { 148 | string name = 1; 149 | int32 slot_size = 2; 150 | int32 num_slots = 3; 151 | bytes type = 4; 152 | } 153 | 154 | // This is published to the /subspace/ChannelDirectory channel. 155 | message ChannelDirectory { 156 | string server_id = 1; 157 | repeated ChannelInfo channels = 2; 158 | } 159 | 160 | message ChannelStats { 161 | string channel_name = 1; 162 | int64 total_bytes = 2; 163 | int64 total_messages = 3; 164 | int32 slot_size = 4; 165 | int32 num_slots = 5; 166 | int32 num_pubs = 6; 167 | int32 num_subs = 7; 168 | } 169 | 170 | // This is published to the /subspace/Statistics channel. 171 | message Statistics { 172 | string server_id = 1; 173 | int64 timestamp = 2; 174 | repeated ChannelStats channels = 3; 175 | } 176 | 177 | message ChannelAddress { 178 | bytes ip_address = 1; // In host byte order. 179 | int32 port = 2; // In host byte order. 180 | } 181 | 182 | // This is sent over the connected channel TCP bridge when the 183 | // bridged subscription is successful. 184 | message Subscribed { 185 | string channel_name = 1; 186 | int32 slot_size = 2; 187 | int32 num_slots = 3; 188 | bool reliable = 4; 189 | } 190 | 191 | // This message is sent over UDP. 192 | message Discovery { 193 | // Ask which server is publishing the named channel. 194 | message Query { string channel_name = 1; } 195 | 196 | // Advertise that the sender is publishing a channel. 197 | message Advertise { 198 | string channel_name = 1; 199 | bool reliable = 2; 200 | } 201 | 202 | // Subscribe to the given channel. The sender is listening 203 | // on the address specified in 'receiver'. 204 | message Subscribe { 205 | string channel_name = 1; 206 | ChannelAddress receiver = 2; 207 | bool reliable = 3; 208 | } 209 | 210 | string server_id = 1; 211 | int32 port = 2; // UDP port I'm listening on. 212 | 213 | oneof data { 214 | Query query = 3; 215 | Advertise advertise = 4; 216 | Subscribe subscribe = 5; 217 | } 218 | } 219 | -------------------------------------------------------------------------------- /server/BUILD.bazel: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | cc_library( 4 | name = "server", 5 | srcs = [ 6 | "server.cc", 7 | "client_handler.cc", 8 | "server_channel.cc", 9 | ], 10 | hdrs = [ 11 | "server.h", 12 | "client_handler.h", 13 | "server_channel.h", 14 | ], 15 | deps = [ 16 | "//common:subspace_common", 17 | "//client:subspace_client", 18 | "@com_google_absl//absl/container:flat_hash_map", 19 | "@com_google_absl//absl/container:flat_hash_set", 20 | "@com_google_absl//absl/status", 21 | "@com_google_absl//absl/status:statusor", 22 | "@com_google_absl//absl/strings:str_format", 23 | "@com_google_absl//absl/flags:flag", 24 | "@com_google_absl//absl/flags:parse", 25 | "@coroutines//:co", 26 | ], 27 | ) 28 | 29 | cc_binary( 30 | name = "subspace_server", 31 | srcs = [ 32 | "main.cc", 33 | ], 34 | deps = [ 35 | ":server", 36 | "@coroutines//:co", 37 | ], 38 | linkopts = select({ 39 | "//:macos_x86_64": [], 40 | "//:macos_arm64": [], 41 | "//:macos_default": [], 42 | "//conditions:default": [ "-lrt" ], 43 | }), 44 | ) 45 | -------------------------------------------------------------------------------- /server/client_handler.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Allison 2 | // All Rights Reserved 3 | // See LICENSE file for licensing information. 4 | 5 | #include "server/client_handler.h" 6 | #include "absl/strings/str_format.h" 7 | #include "server/server.h" 8 | 9 | namespace subspace { 10 | 11 | ClientHandler::~ClientHandler() { server_->RemoveAllUsersFor(this); } 12 | 13 | void ClientHandler::Run(co::Coroutine *c) { 14 | // The data is placed 4 bytes into the buffer. The first 4 15 | // bytes of the buffer are used by SendMessage and ReceiveMessage 16 | // for the length of the data. 17 | char *sendbuf = buffer_ + sizeof(int32_t); 18 | constexpr size_t kSendBufLen = sizeof(buffer_) - sizeof(int32_t); 19 | for (;;) { 20 | absl::StatusOr n_recv = 21 | socket_.ReceiveMessage(buffer_, sizeof(buffer_), c); 22 | if (!n_recv.ok()) { 23 | return; 24 | } 25 | subspace::Request request; 26 | if (request.ParseFromArray(buffer_, *n_recv)) { 27 | std::vector fds; 28 | subspace::Response response; 29 | if (absl::Status s = HandleMessage(request, response, fds); !s.ok()) { 30 | server_->logger_.Log(toolbelt::LogLevel::kError, "%s\n", 31 | s.ToString().c_str()); 32 | return; 33 | } 34 | 35 | if (!response.SerializeToArray(sendbuf, kSendBufLen)) { 36 | server_->logger_.Log(toolbelt::LogLevel::kError, 37 | "Failed to serialize response\n"); 38 | return; 39 | } 40 | size_t msglen = response.ByteSizeLong(); 41 | absl::StatusOr n_sent = socket_.SendMessage(sendbuf, msglen, c); 42 | if (!n_sent.ok()) { 43 | return; 44 | } 45 | if (absl::Status status = socket_.SendFds(fds, c); !status.ok()) { 46 | server_->logger_.Log(toolbelt::LogLevel::kError, "%s\n", 47 | status.ToString().c_str()); 48 | return; 49 | } 50 | } else { 51 | server_->logger_.Log(toolbelt::LogLevel::kError, 52 | "Failed to parse message\n"); 53 | return; 54 | } 55 | } 56 | } 57 | 58 | absl::Status 59 | ClientHandler::HandleMessage(const subspace::Request &req, 60 | subspace::Response &resp, 61 | std::vector &fds) { 62 | switch (req.request_case()) { 63 | case subspace::Request::kInit: 64 | HandleInit(req.init(), resp.mutable_init(), fds); 65 | break; 66 | case subspace::Request::kCreatePublisher: 67 | HandleCreatePublisher(req.create_publisher(), 68 | resp.mutable_create_publisher(), fds); 69 | break; 70 | 71 | case subspace::Request::kCreateSubscriber: 72 | HandleCreateSubscriber(req.create_subscriber(), 73 | resp.mutable_create_subscriber(), fds); 74 | break; 75 | 76 | case subspace::Request::kGetTriggers: 77 | HandleGetTriggers(req.get_triggers(), resp.mutable_get_triggers(), fds); 78 | break; 79 | 80 | case subspace::Request::kRemovePublisher: 81 | HandleRemovePublisher(req.remove_publisher(), 82 | resp.mutable_remove_publisher(), fds); 83 | break; 84 | 85 | case subspace::Request::kRemoveSubscriber: 86 | HandleRemoveSubscriber(req.remove_subscriber(), 87 | resp.mutable_remove_subscriber(), fds); 88 | break; 89 | 90 | case subspace::Request::kResize: 91 | HandleResize(req.resize(), resp.mutable_resize(), fds); 92 | break; 93 | 94 | case subspace::Request::kGetBuffers: 95 | HandleGetBuffers(req.get_buffers(), resp.mutable_get_buffers(), fds); 96 | break; 97 | 98 | case subspace::Request::REQUEST_NOT_SET: 99 | return absl::InternalError("Protocol error: unknown request"); 100 | } 101 | return absl::OkStatus(); 102 | } 103 | 104 | void ClientHandler::HandleInit(const subspace::InitRequest &req, 105 | subspace::InitResponse *response, 106 | std::vector &fds) { 107 | response->set_scb_fd_index(0); 108 | fds.push_back(server_->scb_fd_); 109 | client_name_ = req.client_name(); 110 | } 111 | 112 | void ClientHandler::HandleCreatePublisher( 113 | const subspace::CreatePublisherRequest &req, 114 | subspace::CreatePublisherResponse *response, 115 | std::vector &fds) { 116 | ServerChannel *channel = server_->FindChannel(req.channel_name()); 117 | if (channel == nullptr) { 118 | absl::StatusOr ch = server_->CreateChannel( 119 | req.channel_name(), req.slot_size(), req.num_slots(), req.type()); 120 | if (!ch.ok()) { 121 | response->set_error(ch.status().ToString()); 122 | return; 123 | } 124 | channel = *ch; 125 | } else if (channel->IsPlaceholder()) { 126 | // Channel exists, but it's just a placeholder. Remap the memory now 127 | // that we know the slots. 128 | absl::Status status = 129 | server_->RemapChannel(channel, req.slot_size(), req.num_slots()); 130 | if (!status.ok()) { 131 | response->set_error(status.ToString()); 132 | return; 133 | } 134 | } 135 | // Check that the channel types match, if they are provided and 136 | // already set in the channel. 137 | if (!req.type().empty() && !channel->Type().empty() && 138 | channel->Type() != req.type()) { 139 | response->set_error( 140 | absl::StrFormat("Inconsistent channel types for channel %s: " 141 | "type has been set as %s, not %s\n", 142 | req.channel_name(), channel->Type(), req.type())); 143 | return; 144 | } 145 | if (channel->Type().empty()) { 146 | channel->SetType(req.type()); 147 | } 148 | 149 | // Check capacity of channel for unreliable channels. 150 | if (!req.is_reliable()) { 151 | absl::Status cap_ok = channel->HasSufficientCapacity(0); 152 | if (!cap_ok.ok()) { 153 | response->set_error(absl::StrFormat( 154 | "Insufficient capcacity to add a new publisher to channel %s: %s", 155 | req.channel_name(), cap_ok.ToString())); 156 | return; 157 | } 158 | } 159 | 160 | int num_pubs, num_subs; 161 | channel->CountUsers(num_pubs, num_subs); 162 | 163 | // Check consistency of publisher parameters. 164 | if (num_pubs > 0) { 165 | if (req.is_fixed_size() != channel->IsFixedSize()) { 166 | response->set_error( 167 | absl::StrFormat("Inconsistent publisher parameters for channel %s: " 168 | "all publishers must be either fixed size or not", 169 | req.channel_name())); 170 | return; 171 | } 172 | 173 | // We check only the number of slots since the slot size can change 174 | // over time. 175 | if ((req.is_fixed_size() && channel->SlotSize() != req.slot_size()) || 176 | channel->NumSlots() != req.num_slots()) { 177 | response->set_error(absl::StrFormat( 178 | "Inconsistent publisher parameters for channel %s: " 179 | "existing: %d/%d, new: %d/%d", 180 | req.channel_name(), channel->SlotSize(), channel->NumSlots(), 181 | req.slot_size(), req.num_slots())); 182 | return; 183 | } 184 | 185 | if (channel->IsLocal() != req.is_local()) { 186 | response->set_error( 187 | absl::StrFormat("Inconsistent publisher parameters for channel %s: " 188 | "all publishers must be either local or not", 189 | req.channel_name())); 190 | return; 191 | } 192 | } 193 | 194 | // Create the publisher. 195 | absl::StatusOr publisher = 196 | channel->AddPublisher(this, req.is_reliable(), req.is_local(), 197 | req.is_bridge(), req.is_fixed_size()); 198 | if (!publisher.ok()) { 199 | response->set_error(publisher.status().ToString()); 200 | return; 201 | } 202 | response->set_channel_id(channel->GetChannelId()); 203 | response->set_type(channel->Type()); 204 | 205 | PublisherUser *pub = *publisher; 206 | response->set_publisher_id(pub->GetId()); 207 | 208 | // Copy the shared memory file descriptors. 209 | const SharedMemoryFds &channel_fds = channel->GetFds(); 210 | 211 | response->set_ccb_fd_index(0); 212 | fds.push_back(channel_fds.ccb); 213 | 214 | int fd_index = 1; 215 | for (const auto &buffer : channel_fds.buffers) { 216 | auto *info = response->add_buffers(); 217 | info->set_slot_size(buffer.slot_size); 218 | info->set_fd_index(fd_index++); 219 | fds.push_back(buffer.fd); 220 | } 221 | 222 | // Copy the publisher poll and triggers fds. 223 | response->set_pub_poll_fd_index(fd_index++); 224 | fds.push_back(pub->GetPollFd()); 225 | response->set_pub_trigger_fd_index(fd_index++); 226 | fds.push_back(pub->GetTriggerFd()); 227 | 228 | // Add subscriber trigger indexes. 229 | std::vector sub_fds = 230 | channel->GetSubscriberTriggerFds(); 231 | for (auto &fd : sub_fds) { 232 | response->add_sub_trigger_fd_indexes(fd_index++); 233 | fds.push_back(fd); 234 | } 235 | 236 | if (!req.is_bridge() && req.is_local()) { 237 | server_->SendAdvertise(req.channel_name(), req.is_reliable()); 238 | } 239 | ChannelCounters &counters = 240 | channel->RecordUpdate(/*is_pub=*/true, /*add=*/true, req.is_reliable()); 241 | response->set_num_sub_updates(counters.num_sub_updates); 242 | } 243 | 244 | void ClientHandler::HandleCreateSubscriber( 245 | const subspace::CreateSubscriberRequest &req, 246 | subspace::CreateSubscriberResponse *response, 247 | std::vector &fds) { 248 | ServerChannel *channel = server_->FindChannel(req.channel_name()); 249 | if (channel == nullptr) { 250 | // No channel exists, map an empty channel. 251 | absl::StatusOr ch = 252 | server_->CreateChannel(req.channel_name(), 0, 0, req.type()); 253 | if (!ch.ok()) { 254 | response->set_error(ch.status().ToString()); 255 | return; 256 | } 257 | channel = *ch; 258 | } else { 259 | // Check that the channel types match, if they are provided and 260 | // already set in the channel. 261 | if (!req.type().empty() && !channel->Type().empty() && 262 | channel->Type() != req.type()) { 263 | response->set_error( 264 | absl::StrFormat("Inconsistent channel types for channel %s: " 265 | "type has been set as %s, not %s\n", 266 | req.channel_name(), channel->Type(), req.type())); 267 | return; 268 | } 269 | if (channel->Type().empty()) { 270 | channel->SetType(req.type()); 271 | } 272 | } 273 | 274 | SubscriberUser *sub; 275 | if (req.subscriber_id() != -1) { 276 | // This is an exsiting subscriber. 277 | absl::StatusOr user = channel->GetUser(req.subscriber_id()); 278 | if (!user.ok()) { 279 | response->set_error(user.status().ToString()); 280 | return; 281 | } 282 | sub = static_cast(*user); 283 | } else { 284 | if (!req.is_reliable()) { 285 | absl::Status cap_ok = 286 | channel->HasSufficientCapacity(req.max_shared_ptrs()); 287 | if (!cap_ok.ok()) { 288 | response->set_error(absl::StrFormat( 289 | "Insufficient capcacity to add a new subscriber to channel %s: %s", 290 | req.channel_name(), cap_ok.ToString())); 291 | return; 292 | } 293 | } 294 | // Create the subscriber. 295 | absl::StatusOr subscriber = channel->AddSubscriber( 296 | this, req.is_reliable(), req.is_bridge(), req.max_shared_ptrs()); 297 | if (!subscriber.ok()) { 298 | response->set_error(subscriber.status().ToString()); 299 | return; 300 | } 301 | sub = *subscriber; 302 | } 303 | response->set_channel_id(channel->GetChannelId()); 304 | response->set_subscriber_id(sub->GetId()); 305 | response->set_type(channel->Type()); 306 | 307 | const SharedMemoryFds &channel_fds = channel->GetFds(); 308 | 309 | response->set_ccb_fd_index(0); 310 | fds.push_back(channel_fds.ccb); 311 | 312 | int fd_index = 1; 313 | for (const auto &buffer : channel_fds.buffers) { 314 | auto *info = response->add_buffers(); 315 | info->set_slot_size(buffer.slot_size); 316 | info->set_fd_index(fd_index++); 317 | fds.push_back(buffer.fd); 318 | } 319 | 320 | response->set_trigger_fd_index(fd_index++); 321 | fds.push_back(sub->GetTriggerFd()); 322 | 323 | response->set_poll_fd_index(fd_index++); 324 | fds.push_back(sub->GetPollFd()); 325 | 326 | response->set_slot_size(channel->SlotSize()); 327 | response->set_num_slots(channel->NumSlots()); 328 | 329 | // Add publisher trigger indexes. 330 | std::vector pub_fds = 331 | channel->GetReliablePublisherTriggerFds(); 332 | for (auto &fd : pub_fds) { 333 | response->add_reliable_pub_trigger_fd_indexes(fd_index++); 334 | fds.push_back(fd); 335 | } 336 | 337 | if (!req.is_bridge()) { 338 | // Send Query to subscribe to public channels on other servers. 339 | server_->SendQuery(req.channel_name()); 340 | } 341 | ChannelCounters &counters = 342 | channel->RecordUpdate(/*is_pub=*/false, /*add=*/true, req.is_reliable()); 343 | response->set_num_pub_updates(counters.num_pub_updates); 344 | } 345 | 346 | void ClientHandler::HandleGetTriggers( 347 | const subspace::GetTriggersRequest &req, 348 | subspace::GetTriggersResponse *response, 349 | std::vector &fds) { 350 | ServerChannel *channel = server_->FindChannel(req.channel_name()); 351 | if (channel == nullptr) { 352 | response->set_error( 353 | absl::StrFormat("No such channel %s", req.channel_name())); 354 | return; 355 | } 356 | int index = 0; 357 | std::vector pub_fds = 358 | channel->GetReliablePublisherTriggerFds(); 359 | for (auto &fd : pub_fds) { 360 | response->add_reliable_pub_trigger_fd_indexes(index++); 361 | fds.push_back(fd); 362 | } 363 | 364 | std::vector sub_fds = 365 | channel->GetSubscriberTriggerFds(); 366 | for (auto &fd : sub_fds) { 367 | response->add_sub_trigger_fd_indexes(index++); 368 | fds.push_back(fd); 369 | } 370 | } 371 | 372 | void ClientHandler::HandleRemovePublisher( 373 | const subspace::RemovePublisherRequest &req, 374 | subspace::RemovePublisherResponse *response, 375 | std::vector &fds) { 376 | ServerChannel *channel = server_->FindChannel(req.channel_name()); 377 | if (channel == nullptr) { 378 | response->set_error( 379 | absl::StrFormat("No such channel %s", req.channel_name())); 380 | return; 381 | } 382 | channel->RemoveUser(req.publisher_id()); 383 | } 384 | 385 | void ClientHandler::HandleRemoveSubscriber( 386 | const subspace::RemoveSubscriberRequest &req, 387 | subspace::RemoveSubscriberResponse *response, 388 | std::vector &fds) { 389 | ServerChannel *channel = server_->FindChannel(req.channel_name()); 390 | if (channel == nullptr) { 391 | response->set_error( 392 | absl::StrFormat("No such channel %s", req.channel_name())); 393 | return; 394 | } 395 | channel->RemoveUser(req.subscriber_id()); 396 | } 397 | 398 | void ClientHandler::HandleResize(const subspace::ResizeRequest &req, 399 | subspace::ResizeResponse *response, 400 | std::vector &fds) { 401 | ServerChannel *channel = server_->FindChannel(req.channel_name()); 402 | if (channel == nullptr) { 403 | response->set_error( 404 | absl::StrFormat("No such channel %s", req.channel_name())); 405 | return; 406 | } 407 | 408 | // If channel is already bigger than requested, don't resize. 409 | if (req.new_slot_size() <= channel->SlotSize()) { 410 | response->set_slot_size(channel->SlotSize()); 411 | return; 412 | } 413 | absl::StatusOr fd = 414 | channel->ExtendBuffers(req.new_slot_size()); 415 | if (!fd.ok()) { 416 | response->set_error(absl::StrFormat( 417 | "Failed to resize channel %s to %d byte: %s", req.channel_name(), 418 | req.new_slot_size(), fd.status().ToString())); 419 | return; 420 | } 421 | response->set_slot_size(channel->SlotSize()); 422 | 423 | channel->AddBuffer(req.new_slot_size(), std::move(*fd)); 424 | const SharedMemoryFds &channel_fds = channel->GetFds(); 425 | 426 | int fd_index = 0; 427 | for (const auto &buffer : channel_fds.buffers) { 428 | auto *info = response->add_buffers(); 429 | info->set_slot_size(buffer.slot_size); 430 | info->set_fd_index(fd_index++); 431 | fds.push_back(buffer.fd); 432 | } 433 | } 434 | 435 | void ClientHandler::HandleGetBuffers( 436 | const subspace::GetBuffersRequest &req, 437 | subspace::GetBuffersResponse *response, 438 | std::vector &fds) { 439 | ServerChannel *channel = server_->FindChannel(req.channel_name()); 440 | if (channel == nullptr) { 441 | response->set_error( 442 | absl::StrFormat("No such channel %s", req.channel_name())); 443 | return; 444 | } 445 | const SharedMemoryFds &channel_fds = channel->GetFds(); 446 | 447 | int fd_index = 0; 448 | for (const auto &buffer : channel_fds.buffers) { 449 | auto *info = response->add_buffers(); 450 | info->set_slot_size(buffer.slot_size); 451 | info->set_fd_index(fd_index++); 452 | fds.push_back(buffer.fd); 453 | } 454 | } 455 | 456 | } // namespace subspace 457 | -------------------------------------------------------------------------------- /server/client_handler.h: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Allison 2 | // All Rights Reserved 3 | // See LICENSE file for licensing information. 4 | 5 | #ifndef __SERVER_CLIENT_HANDLER_H 6 | #define __SERVER_CLIENT_HANDLER_H 7 | 8 | #include "absl/status/status.h" 9 | #include "common/channel.h" 10 | #include "coroutine.h" 11 | #include "proto/subspace.pb.h" 12 | #include "toolbelt/sockets.h" 13 | #include 14 | #include 15 | 16 | namespace subspace { 17 | 18 | class Server; 19 | 20 | class ClientHandler { 21 | public: 22 | ClientHandler(Server *server, toolbelt::UnixSocket socket) 23 | : server_(server), socket_(std::move(socket)) {} 24 | ~ClientHandler(); 25 | 26 | // Run the client handler receiver in a coroutine. Terminates 27 | // when the connection to the client is closed. 28 | void Run(co::Coroutine *c); 29 | 30 | private: 31 | absl::Status HandleMessage(const subspace::Request &req, 32 | subspace::Response &resp, 33 | std::vector &fds); 34 | 35 | // These individual handler functions set any errors in the response 36 | // message instead of returning them to the caller. This allows the 37 | // connection to remain open to the client and the client will be 38 | // able to display or handle the error as appropriate. 39 | void HandleInit(const subspace::InitRequest &req, 40 | subspace::InitResponse *response, 41 | std::vector &fds); 42 | void HandleCreatePublisher(const subspace::CreatePublisherRequest &req, 43 | subspace::CreatePublisherResponse *response, 44 | std::vector &fds); 45 | void HandleCreateSubscriber(const subspace::CreateSubscriberRequest &req, 46 | subspace::CreateSubscriberResponse *response, 47 | std::vector &fds); 48 | void HandleGetTriggers(const subspace::GetTriggersRequest &req, 49 | subspace::GetTriggersResponse *response, 50 | std::vector &fds); 51 | 52 | void HandleRemovePublisher(const subspace::RemovePublisherRequest &req, 53 | subspace::RemovePublisherResponse *response, 54 | std::vector &fds); 55 | void HandleRemoveSubscriber(const subspace::RemoveSubscriberRequest &req, 56 | subspace::RemoveSubscriberResponse *response, 57 | std::vector &fds); 58 | void HandleResize(const subspace::ResizeRequest &req, 59 | subspace::ResizeResponse *response, 60 | std::vector &fds); 61 | void HandleGetBuffers(const subspace::GetBuffersRequest &req, 62 | subspace::GetBuffersResponse *response, 63 | std::vector &fds); 64 | Server *server_; 65 | toolbelt::UnixSocket socket_; 66 | char buffer_[kMaxMessage]; 67 | std::string client_name_; 68 | }; 69 | 70 | } // namespace subspace 71 | 72 | #endif // __SERVER_CLIENT_HANDLER_H -------------------------------------------------------------------------------- /server/main.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Allison 2 | // All Rights Reserved 3 | // See LICENSE file for licensing information. 4 | 5 | #include "absl/flags/flag.h" 6 | #include "absl/flags/parse.h" 7 | #include "server.h" 8 | #include 9 | #include 10 | #include 11 | 12 | static co::CoroutineScheduler* g_scheduler; 13 | void Signal(int sig) { 14 | printf("\nAll coroutines:\n"); 15 | g_scheduler->Show(); 16 | signal(sig, SIG_DFL); 17 | raise(sig); 18 | } 19 | 20 | ABSL_FLAG(std::string, socket, "/tmp/subspace", 21 | "Name of Unix socket to listen on"); 22 | ABSL_FLAG(int, disc_port, 6502, "Discovery UDP port"); 23 | ABSL_FLAG(int, peer_port, 6502, "Discovery peer UDP port"); 24 | ABSL_FLAG(std::string, log_level, "info", "Log level"); 25 | ABSL_FLAG(std::string, interface, "", "Discovery network interface"); 26 | ABSL_FLAG(bool, local, false, "Use local computer only"); 27 | ABSL_FLAG(int, notify_fd, -1, "File descriptor to notify of startup"); 28 | 29 | int main(int argc, char **argv) { 30 | absl::ParseCommandLine(argc, argv); 31 | 32 | co::CoroutineScheduler scheduler; 33 | 34 | g_scheduler = &scheduler; // For signal handler. 35 | signal(SIGPIPE, SIG_IGN); 36 | signal(SIGQUIT, Signal); 37 | 38 | subspace::Server server( 39 | scheduler, absl::GetFlag(FLAGS_socket), absl::GetFlag(FLAGS_interface), 40 | absl::GetFlag(FLAGS_disc_port), absl::GetFlag(FLAGS_peer_port), 41 | absl::GetFlag(FLAGS_local), absl::GetFlag(FLAGS_notify_fd)); 42 | 43 | server.SetLogLevel(absl::GetFlag(FLAGS_log_level)); 44 | absl::Status s = server.Run(); 45 | if (!s.ok()) { 46 | fprintf(stderr, "Error running Subspace server: %s\n", s.ToString().c_str()); 47 | exit(1); 48 | } 49 | } -------------------------------------------------------------------------------- /server/server.h: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Allison 2 | // All Rights Reserved 3 | // See LICENSE file for licensing information. 4 | 5 | #ifndef __SERVER_SERVER_H 6 | #define __SERVER_SERVER_H 7 | 8 | #include "absl/container/flat_hash_map.h" 9 | #include "absl/container/flat_hash_set.h" 10 | #include "absl/status/status.h" 11 | #include "absl/status/statusor.h" 12 | #include "client_handler.h" 13 | #include "coroutine.h" 14 | #include "proto/subspace.pb.h" 15 | #include "server/server_channel.h" 16 | #include "toolbelt/bitset.h" 17 | #include "toolbelt/fd.h" 18 | #include "toolbelt/logging.h" 19 | #include 20 | #include 21 | 22 | namespace subspace { 23 | 24 | // Values written to the notify_fd when the server is ready and 25 | // is stopped. 26 | constexpr int64_t kServerReady = 1; 27 | constexpr int64_t kServerStopped = 2; 28 | 29 | // The Subspace server. 30 | // This is a single-threaded, coroutine-based server that maintains shared 31 | // memory IPC channels and communicates with other servers to allow for 32 | // cross-computer IPC. 33 | class Server { 34 | public: 35 | // The notify_fd is a file descriptor that the server will write to 36 | // when it is ready to run (after the socket has been created) and when 37 | // it is shutting down. It will write 8 bytes to it so it can be 38 | // either a pipe or an eventfd. If notify_fd is -1 then it won't be used. 39 | // The values are written in host byte order. 40 | Server(co::CoroutineScheduler &scheduler, const std::string &socket_name, 41 | const std::string &interface, int disc_port, int peer_port, bool local, 42 | int notify_fd = -1); 43 | ~Server(); 44 | void SetLogLevel(const std::string &level) { logger_.SetLogLevel(level); } 45 | absl::Status Run(); 46 | void Stop(); 47 | 48 | private: 49 | friend class ClientHandler; 50 | friend class ServerChannel; 51 | static constexpr size_t kDiscoveryBufferSize = 1024; 52 | 53 | absl::Status HandleIncomingConnection(toolbelt::UnixSocket &listen_socket, 54 | co::Coroutine *c); 55 | 56 | // Create a channel in both process and shared memory. For a placeholder 57 | // subscriber, the channel parameters are not known, so slot_size and 58 | // num_slots will be zero. 59 | absl::StatusOr CreateChannel(const std::string &channel_name, 60 | int slot_size, int num_slots, 61 | std::string type); 62 | absl::Status RemapChannel(ServerChannel *channel, int slot_size, 63 | int num_slots); 64 | ServerChannel *FindChannel(const std::string &channel_name); 65 | void RemoveChannel(ServerChannel *channel); 66 | void RemoveAllUsersFor(ClientHandler *handler); 67 | void CloseHandler(ClientHandler *handler); 68 | void ListenerCoroutine(toolbelt::UnixSocket& listen_socket, co::Coroutine *c); 69 | void ChannelDirectoryCoroutine(co::Coroutine *c); 70 | void SendChannelDirectory(); 71 | void StatisticsCoroutine(co::Coroutine *c); 72 | void DiscoveryReceiverCoroutine(co::Coroutine *c); 73 | void PublisherCoroutine(co::Coroutine *c); 74 | void SendQuery(const std::string &channel_name); 75 | void SendAdvertise(const std::string &channel_name, bool reliable); 76 | void BridgeTransmitterCoroutine(ServerChannel *channel, bool pub_reliable, 77 | bool sub_reliable, 78 | toolbelt::InetAddress subscriber, 79 | co::Coroutine *c); 80 | void BridgeReceiverCoroutine(std::string channel_name, bool sub_reliable, 81 | toolbelt::InetAddress publisher, 82 | co::Coroutine *c); 83 | void SubscribeOverBridge(ServerChannel *channel, bool reliable, 84 | toolbelt::InetAddress publisher); 85 | void IncomingQuery(const Discovery::Query &query, 86 | const toolbelt::InetAddress &sender); 87 | void IncomingAdvertise(const Discovery::Advertise &advertise, 88 | const toolbelt::InetAddress &sender); 89 | void IncomingSubscribe(const Discovery::Subscribe &subscribe, 90 | const toolbelt::InetAddress &sender); 91 | void GratuitousAdvertiseCoroutine(co::Coroutine *c); 92 | absl::Status SendSubscribeMessage(const std::string &channel_name, 93 | bool reliable, 94 | toolbelt::InetAddress publisher, 95 | toolbelt::TCPSocket &receiver_listener, 96 | char *buffer, size_t buffer_size, 97 | co::Coroutine *c); 98 | std::string socket_name_; 99 | std::vector> client_handlers_; 100 | bool running_ = false; 101 | std::string server_id_; 102 | std::string hostname_; 103 | std::string interface_; 104 | int discovery_port_; 105 | int discovery_peer_port_; 106 | bool local_; 107 | toolbelt::FileDescriptor notify_fd_; 108 | 109 | absl::flat_hash_map> channels_; 110 | 111 | SystemControlBlock *scb_; 112 | toolbelt::FileDescriptor scb_fd_; 113 | toolbelt::BitSet channel_ids_; 114 | co::CoroutineScheduler &co_scheduler_; 115 | 116 | // All coroutines are owned by this set. 117 | absl::flat_hash_set> coroutines_; 118 | 119 | toolbelt::TriggerFd channel_directory_trigger_fd_; 120 | toolbelt::InetAddress discovery_addr_; 121 | toolbelt::UDPSocket discovery_transmitter_; 122 | toolbelt::UDPSocket discovery_receiver_; 123 | toolbelt::Logger logger_; 124 | }; 125 | 126 | } // namespace subspace 127 | 128 | #endif // __SERVER_SERVER_H 129 | -------------------------------------------------------------------------------- /server/server_channel.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Allison 2 | // All Rights Reserved 3 | // See LICENSE file for licensing information. 4 | 5 | #include "server/server_channel.h" 6 | #include "absl/strings/str_format.h" 7 | #include "server/server.h" 8 | 9 | namespace subspace { 10 | 11 | ServerChannel::~ServerChannel() { 12 | // Clear the channel counters in the SCB. 13 | memset(&GetScb()->counters[GetChannelId()], 0, sizeof(ChannelCounters)); 14 | } 15 | 16 | std::vector 17 | ServerChannel::GetSubscriberTriggerFds() const { 18 | std::vector r; 19 | for (auto &user : users_) { 20 | if (user == nullptr) { 21 | continue; 22 | } 23 | if (user->IsSubscriber()) { 24 | r.push_back(user->GetTriggerFd()); 25 | } 26 | } 27 | return r; 28 | } 29 | 30 | std::vector 31 | ServerChannel::GetReliablePublisherTriggerFds() const { 32 | std::vector r; 33 | for (auto &user : users_) { 34 | if (user == nullptr) { 35 | continue; 36 | } 37 | if (user->IsPublisher() && user->IsReliable()) { 38 | r.push_back(user->GetTriggerFd()); 39 | } 40 | } 41 | return r; 42 | } 43 | 44 | absl::StatusOr 45 | ServerChannel::AddPublisher(ClientHandler *handler, bool is_reliable, 46 | bool is_local, bool is_bridge, bool is_fixed_size) { 47 | absl::StatusOr user_id = user_ids_.Allocate("publisher"); 48 | if (!user_id.ok()) { 49 | return user_id.status(); 50 | } 51 | std::unique_ptr pub = std::make_unique( 52 | handler, *user_id, is_reliable, is_local, is_bridge, is_fixed_size); 53 | absl::Status status = pub->Init(); 54 | if (!status.ok()) { 55 | return status; 56 | } 57 | PublisherUser *result = pub.get(); 58 | if (*user_id >= users_.size()) { 59 | users_.resize(*user_id + 1); 60 | } 61 | users_[*user_id] = std::move(pub); 62 | return result; 63 | } 64 | 65 | absl::StatusOr 66 | ServerChannel::AddSubscriber(ClientHandler *handler, bool is_reliable, 67 | bool is_bridge, int max_shared_ptrs) { 68 | absl::StatusOr user_id = user_ids_.Allocate("subscriber"); 69 | if (!user_id.ok()) { 70 | return user_id.status(); 71 | } 72 | std::unique_ptr sub = std::make_unique( 73 | handler, *user_id, is_reliable, is_bridge, max_shared_ptrs); 74 | absl::Status status = sub->Init(); 75 | if (!status.ok()) { 76 | return status; 77 | } 78 | SubscriberUser *result = sub.get(); 79 | if (*user_id >= users_.size()) { 80 | users_.resize(*user_id + 1); 81 | } 82 | users_[*user_id] = std::move(sub); 83 | 84 | return result; 85 | } 86 | 87 | void ServerChannel::TriggerAllSubscribers() { 88 | for (auto &user : users_) { 89 | if (user == nullptr) { 90 | continue; 91 | } 92 | if (user->IsSubscriber()) { 93 | user->Trigger(); 94 | } 95 | } 96 | } 97 | 98 | void ServerChannel::RemoveUser(int user_id) { 99 | for (auto &user : users_) { 100 | if (user == nullptr) { 101 | continue; 102 | } 103 | if (user->GetId() == user_id) { 104 | CleanupSlots(user->GetId(), user->IsReliable()); 105 | user_ids_.Clear(user->GetId()); 106 | RecordUpdate(user->IsPublisher(), /*add=*/false, user->IsReliable()); 107 | if (user->IsPublisher()) { 108 | TriggerAllSubscribers(); 109 | } 110 | user.reset(); 111 | return; 112 | } 113 | } 114 | } 115 | 116 | void ServerChannel::RemoveAllUsersFor(ClientHandler *handler) { 117 | for (auto &user : users_) { 118 | if (user == nullptr) { 119 | continue; 120 | } 121 | if (user->GetHandler() == handler) { 122 | CleanupSlots(user->GetId(), user->IsReliable()); 123 | user_ids_.Clear(user->GetId()); 124 | RecordUpdate(user->IsPublisher(), /*add=*/false, user->IsReliable()); 125 | if (user->IsPublisher()) { 126 | TriggerAllSubscribers(); 127 | } 128 | user.reset(); 129 | } 130 | } 131 | } 132 | 133 | void ServerChannel::CountUsers(int &num_pubs, int &num_subs) const { 134 | num_pubs = num_subs = 0; 135 | for (auto &user : users_) { 136 | if (user == nullptr) { 137 | continue; 138 | } 139 | if (user->IsPublisher()) { 140 | num_pubs++; 141 | } else { 142 | num_subs++; 143 | } 144 | } 145 | } 146 | 147 | // Channel is public if there are any public publishers. 148 | bool ServerChannel::IsLocal() const { 149 | for (auto &user : users_) { 150 | if (user == nullptr) { 151 | continue; 152 | } 153 | if (user->IsPublisher()) { 154 | PublisherUser *pub = static_cast(user.get()); 155 | if (pub->IsLocal()) { 156 | return true; 157 | } 158 | } 159 | } 160 | return false; 161 | } 162 | 163 | // Channel is reliable if there are any reliable publishers. 164 | bool ServerChannel::IsReliable() const { 165 | for (auto &user : users_) { 166 | if (user == nullptr) { 167 | continue; 168 | } 169 | if (user->IsPublisher()) { 170 | PublisherUser *pub = static_cast(user.get()); 171 | if (pub->IsReliable()) { 172 | return true; 173 | } 174 | } 175 | } 176 | return false; 177 | } 178 | 179 | // Channel is fixed_size if there are any fixed size publishers. If one is 180 | // fixed size, they all must be. 181 | bool ServerChannel::IsFixedSize() const { 182 | for (auto &user : users_) { 183 | if (user == nullptr) { 184 | continue; 185 | } 186 | if (user->IsPublisher()) { 187 | PublisherUser *pub = static_cast(user.get()); 188 | if (pub->IsFixedSize()) { 189 | return true; 190 | } 191 | } 192 | } 193 | return false; 194 | } 195 | 196 | 197 | bool ServerChannel::IsBridgePublisher() const { 198 | int num_pubs = 0; 199 | int num_bridge_pubs = 0; 200 | for (auto &user : users_) { 201 | if (user == nullptr) { 202 | continue; 203 | } 204 | if (user->IsPublisher()) { 205 | num_pubs++; 206 | PublisherUser *pub = static_cast(user.get()); 207 | if (pub->IsBridge()) { 208 | num_bridge_pubs++; 209 | } 210 | } 211 | } 212 | return num_pubs == num_bridge_pubs; 213 | } 214 | 215 | bool ServerChannel::IsBridgeSubscriber() const { 216 | int num_subs = 0; 217 | int num_bridge_subs = 0; 218 | for (auto &user : users_) { 219 | if (user == nullptr) { 220 | continue; 221 | } 222 | if (user->IsSubscriber()) { 223 | num_subs++; 224 | SubscriberUser *sub = static_cast(user.get()); 225 | if (sub->IsBridge()) { 226 | num_bridge_subs++; 227 | } 228 | } 229 | } 230 | return num_subs == num_bridge_subs; 231 | } 232 | 233 | absl::Status ServerChannel::HasSufficientCapacity(int new_max_ptrs) const { 234 | if (NumSlots() == 0) { 235 | return absl::OkStatus(); 236 | } 237 | // Count number of publishers and subscribers. 238 | int num_pubs, num_subs; 239 | CountUsers(num_pubs, num_subs); 240 | 241 | // Add in the total shared ptr maximums. 242 | int max_shared_ptrs = new_max_ptrs; 243 | for (auto &user : users_) { 244 | if (user == nullptr) { 245 | continue; 246 | } 247 | if (user->IsSubscriber()) { 248 | SubscriberUser *sub = static_cast(user.get()); 249 | max_shared_ptrs += sub->MaxSharedPtrs(); 250 | } 251 | } 252 | int slots_needed = num_pubs + num_subs + max_shared_ptrs + 1; 253 | if (slots_needed <= (NumSlots() - 1)) { 254 | return absl::OkStatus(); 255 | } 256 | max_shared_ptrs -= new_max_ptrs; // Adjust for error message. 257 | 258 | return absl::InternalError( 259 | absl::StrFormat("there are %d slots with %d publisher%s and %d " 260 | "subscriber%s with %d shared pointer%s; you need at least %d slots", 261 | NumSlots(), num_pubs, (num_pubs == 1 ? "" : "s"), 262 | num_subs, (num_subs == 1 ? "" : "s"), max_shared_ptrs, 263 | (max_shared_ptrs == 1 ? "" : "s"), 264 | slots_needed+1)); 265 | } 266 | 267 | void ServerChannel::GetChannelInfo(subspace::ChannelInfo *info) { 268 | info->set_name(Name()); 269 | info->set_slot_size(SlotSize()); 270 | info->set_num_slots(NumSlots()); 271 | info->set_type(Type()); 272 | } 273 | 274 | void ServerChannel::GetChannelStats(subspace::ChannelStats *stats) { 275 | stats->set_channel_name(Name()); 276 | int64_t total_bytes, total_messages; 277 | GetStatsCounters(total_bytes, total_messages); 278 | stats->set_total_bytes(total_bytes); 279 | stats->set_total_messages(total_messages); 280 | stats->set_slot_size(SlotSize()); 281 | stats->set_num_slots(NumSlots()); 282 | 283 | int num_pubs, num_subs; 284 | CountUsers(num_pubs, num_subs); 285 | stats->set_num_pubs(num_pubs); 286 | stats->set_num_subs(num_subs); 287 | } 288 | 289 | ChannelCounters &ServerChannel::RecordUpdate(bool is_pub, bool add, 290 | bool reliable) { 291 | SystemControlBlock *scb = GetScb(); 292 | int channel_id = GetChannelId(); 293 | ChannelCounters &counters = scb->counters[channel_id]; 294 | int inc = add ? 1 : -1; 295 | if (is_pub) { 296 | SetNumUpdates(++counters.num_pub_updates); 297 | counters.num_pubs += inc; 298 | if (reliable) { 299 | counters.num_reliable_pubs += inc; 300 | } 301 | } else { 302 | SetNumUpdates(++counters.num_sub_updates); 303 | counters.num_subs += inc; 304 | if (reliable) { 305 | counters.num_reliable_subs += inc; 306 | } 307 | } 308 | return counters; 309 | } 310 | 311 | void ServerChannel::AddBuffer(int slot_size, toolbelt::FileDescriptor fd) { 312 | shared_memory_fds_.buffers.push_back({slot_size, std::move(fd)}); 313 | } 314 | 315 | } // namespace subspace -------------------------------------------------------------------------------- /server/server_channel.h: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Allison 2 | // All Rights Reserved 3 | // See LICENSE file for licensing information. 4 | 5 | #ifndef __SERVER_SERVER_CHANNEL_H 6 | #define __SERVER_SERVER_CHANNEL_H 7 | 8 | #include "absl/container/flat_hash_set.h" 9 | #include "absl/status/status.h" 10 | #include "absl/status/statusor.h" 11 | #include "common/channel.h" 12 | #include "proto/subspace.pb.h" 13 | #include "toolbelt/bitset.h" 14 | #include "toolbelt/fd.h" 15 | #include "toolbelt/sockets.h" 16 | #include "toolbelt/triggerfd.h" 17 | #include 18 | #include 19 | 20 | namespace subspace { 21 | constexpr int kMaxUsers = kMaxSlotOwners; 22 | 23 | class ClientHandler; 24 | class Server; 25 | 26 | // A user is a publisher or subscriber on a channel. Each user has a 27 | // unique (per channel) user id. A user might have a trigger fd 28 | // associated with it (subscribers always have one, but only 29 | // reliable publishers have one). 30 | class User { 31 | public: 32 | User(ClientHandler *handler, int id, bool is_reliable, bool is_bridge) 33 | : handler_(handler), id_(id), is_reliable_(is_reliable), 34 | is_bridge_(is_bridge) {} 35 | virtual ~User() = default; 36 | 37 | absl::Status Init() { return trigger_fd_.Open(); } 38 | 39 | int GetId() const { return id_; } 40 | toolbelt::FileDescriptor &GetPollFd() { return trigger_fd_.GetPollFd(); } 41 | toolbelt::FileDescriptor &GetTriggerFd() { 42 | return trigger_fd_.GetTriggerFd(); 43 | } 44 | virtual bool IsSubscriber() const { return false; } 45 | virtual bool IsPublisher() const { return false; } 46 | ClientHandler *GetHandler() const { return handler_; } 47 | bool IsReliable() const { return is_reliable_; } 48 | bool IsBridge() const { return is_bridge_; } 49 | void Trigger() { trigger_fd_.Trigger(); } 50 | 51 | private: 52 | ClientHandler *handler_; 53 | int id_; 54 | toolbelt::TriggerFd trigger_fd_; 55 | bool is_reliable_; 56 | bool is_bridge_; // This is used to send or receive over a bridge. 57 | }; 58 | 59 | class SubscriberUser : public User { 60 | public: 61 | SubscriberUser(ClientHandler *handler, int id, bool is_reliable, 62 | bool is_bridge, int max_shared_ptrs) 63 | : User(handler, id, is_reliable, is_bridge), 64 | max_shared_ptrs_(max_shared_ptrs) {} 65 | bool IsSubscriber() const override { return true; } 66 | int MaxSharedPtrs() const { return max_shared_ptrs_; } 67 | 68 | private: 69 | int max_shared_ptrs_; 70 | }; 71 | 72 | class PublisherUser : public User { 73 | public: 74 | PublisherUser(ClientHandler *handler, int id, bool is_reliable, bool is_local, 75 | bool is_bridge, bool is_fixed_size) 76 | : User(handler, id, is_reliable, is_bridge), is_local_(is_local), 77 | is_fixed_size_(is_fixed_size) {} 78 | 79 | bool IsPublisher() const override { return true; } 80 | bool IsLocal() const { return is_local_; } 81 | bool IsFixedSize() const { return is_fixed_size_; } 82 | 83 | private: 84 | bool is_local_; 85 | bool is_fixed_size_; 86 | }; 87 | 88 | // This is endpoint transmitting the data for a channel. It holds an internet 89 | // address and whether the transmitter is or not. It is absl hashable. 90 | class ChannelTransmitter { 91 | public: 92 | ChannelTransmitter(const toolbelt::InetAddress &addr, bool reliable) 93 | : addr_(addr), reliable_(reliable) {} 94 | 95 | private: 96 | toolbelt::InetAddress addr_; 97 | bool reliable_; 98 | 99 | // Provide support for Abseil hashing. 100 | friend bool operator==(const ChannelTransmitter &a, 101 | const ChannelTransmitter &b); 102 | template 103 | friend H AbslHashValue(H h, const ChannelTransmitter &a); 104 | }; 105 | 106 | inline bool operator==(const ChannelTransmitter &a, 107 | const ChannelTransmitter &b) { 108 | return a.addr_ == b.addr_ && a.reliable_ == b.reliable_; 109 | } 110 | 111 | template inline H AbslHashValue(H h, const ChannelTransmitter &a) { 112 | H addr_hash = AbslHashValue(std::move(h), a.addr_); 113 | return H::combine(std::move(addr_hash), a.reliable_); 114 | } 115 | 116 | // This is a channel maintained by the server. The server creates the shared 117 | // memory for the channel and distributes the file descriptor associated with 118 | // it. 119 | class ServerChannel : public Channel { 120 | public: 121 | ServerChannel(int id, const std::string &name, int num_slots, 122 | std::string type) 123 | : Channel(name, num_slots, id, std::move(type)) {} 124 | 125 | ~ServerChannel(); 126 | 127 | absl::StatusOr AddPublisher(ClientHandler *handler, 128 | bool is_reliable, bool is_local, 129 | bool is_bridge, bool is_fixed_size); 130 | absl::StatusOr AddSubscriber(ClientHandler *handler, 131 | bool is_reliable, 132 | bool is_bridge, 133 | int max_shared_ptrs); 134 | 135 | // Get the file descriptors for all subscriber triggers. 136 | std::vector GetSubscriberTriggerFds() const; 137 | 138 | // Get the file descriptors for all reliable publisher triggers. 139 | std::vector GetReliablePublisherTriggerFds() const; 140 | 141 | // Translate a user id into a User pointer. The pointer ownership 142 | // is kept by the ServerChannel. 143 | absl::StatusOr GetUser(int id) { 144 | if (id < 0 || id >= users_.size()) { 145 | return absl::InternalError("Invalid user id"); 146 | } 147 | return users_[id].get(); 148 | } 149 | 150 | // Record an update to a channel in the SCB. Args: 151 | // is_pub: this is a publisher 152 | // add: the publisher or subscriber is being added 153 | // reliable: change the reliable counters. 154 | ChannelCounters &RecordUpdate(bool is_pub, bool add, bool reliable); 155 | ChannelCounters &RecordResize(); 156 | 157 | void RemoveUser(int user_id); 158 | void RemoveAllUsersFor(ClientHandler *handler); 159 | bool IsEmpty() const { return user_ids_.IsEmpty(); } 160 | absl::Status HasSufficientCapacity(int new_max_ptrs) const; 161 | void CountUsers(int &num_pubs, int &num_subs) const; 162 | void GetChannelInfo(subspace::ChannelInfo *info); 163 | void GetChannelStats(subspace::ChannelStats *stats); 164 | void TriggerAllSubscribers(); 165 | 166 | // This is true if all publishers are bridge publishers. 167 | bool IsBridgePublisher() const; 168 | bool IsBridgeSubscriber() const; 169 | 170 | // Determine if the given address is registered as a bridge 171 | // publisher. 172 | bool IsBridged(const toolbelt::InetAddress &addr, bool reliable) const { 173 | return bridged_publishers_.contains(ChannelTransmitter(addr, reliable)); 174 | } 175 | 176 | void AddBridgedAddress(const toolbelt::InetAddress &addr, bool reliable) { 177 | bridged_publishers_.emplace(addr, reliable); 178 | } 179 | 180 | void RemoveBridgedAddress(const toolbelt::InetAddress &addr, bool reliable) { 181 | bridged_publishers_.erase(ChannelTransmitter(addr, reliable)); 182 | } 183 | 184 | bool IsLocal() const; 185 | bool IsReliable() const; 186 | bool IsFixedSize() const; 187 | 188 | void SetSharedMemoryFds(SharedMemoryFds fds) { 189 | shared_memory_fds_ = std::move(fds); 190 | } 191 | 192 | const SharedMemoryFds &GetFds() { return shared_memory_fds_; } 193 | 194 | // Add a buffer (slot size and memory fd) to the shared_memory_fds. 195 | void AddBuffer(int slot_size, toolbelt::FileDescriptor fd); 196 | 197 | private: 198 | std::vector> users_; 199 | toolbelt::BitSet user_ids_; 200 | absl::flat_hash_set bridged_publishers_; 201 | SharedMemoryFds shared_memory_fds_; 202 | }; 203 | } // namespace subspace 204 | #endif // __SERVER_SERVER_CHANNEL_H 205 | --------------------------------------------------------------------------------