├── .gitignore ├── benches ├── ipc_bench.sh ├── bench.rs └── ipc_bench.rs ├── Cargo.toml ├── examples ├── ipc_read.rs ├── ipc_write.rs └── basics.rs ├── .github └── workflows │ └── on_pull_request.yml ├── LICENSE ├── README.md └── src ├── tests.rs └── lib.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /benches/ipc_bench.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cargo bench --bench ipc_bench -- reader & 6 | sleep 1 7 | cargo bench --bench ipc_bench -- writer 8 | wait 9 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cueue" 3 | version = "0.4.0" 4 | edition = "2021" 5 | authors = ["Benedek Thaler"] 6 | license = "MIT" 7 | repository = "https://github.com/erenon/cueue" 8 | description = "High performance SPSC circular byte buffer with batch operations" 9 | keywords = ["spsc", "queue", "ringbuffer"] 10 | categories = ["data-structures"] 11 | readme = "README.md" 12 | 13 | [dependencies] 14 | libc = "0.2.132" 15 | 16 | [[bench]] 17 | name = "ipc_bench" 18 | harness = false 19 | -------------------------------------------------------------------------------- /benches/bench.rs: -------------------------------------------------------------------------------- 1 | #![feature(test)] 2 | 3 | use cueue::cueue; 4 | 5 | extern crate test; 6 | use self::test::Bencher; 7 | 8 | #[bench] 9 | fn bench_write(b: &mut Bencher) { 10 | let (mut w, mut r) = cueue(16).unwrap(); 11 | 12 | let rt = std::thread::spawn(move || { 13 | while !r.is_abandoned() { 14 | let _rr = r.read_chunk(); 15 | r.commit(); 16 | } 17 | }); 18 | 19 | b.iter(move || { 20 | let buf = loop { 21 | let buf = w.write_chunk(); 22 | if buf.len() >= 16 { 23 | break buf; 24 | } 25 | }; 26 | unsafe { 27 | std::ptr::copy_nonoverlapping(b"123456789abcdefh", buf.as_mut_ptr(), 16); 28 | } 29 | w.commit(16); 30 | }); 31 | 32 | rt.join().unwrap(); 33 | } 34 | -------------------------------------------------------------------------------- /examples/ipc_read.rs: -------------------------------------------------------------------------------- 1 | //! Example of inter-process communication 2 | //! 3 | //! Start `ipc_write` first, then this example second, while the writer is still running. 4 | 5 | use std::ffi::CString; 6 | use std::io::Write; 7 | 8 | fn main() -> std::io::Result<()> { 9 | let path = CString::new("cueue_ipc").unwrap(); 10 | let f = unsafe { libc::shm_open(path.as_ptr(), libc::O_RDWR, 0) }; 11 | if f < 0 { 12 | return Err(std::io::Error::last_os_error()); 13 | } 14 | 15 | let (_, mut r) = cueue::cueue_in_fd(f, None)?; 16 | 17 | let mut out = std::io::stdout().lock(); 18 | loop { 19 | let buf = r.read_chunk(); 20 | if !buf.is_empty() { 21 | out.write_all(buf)?; 22 | r.commit(); 23 | } else { 24 | std::thread::sleep(std::time::Duration::from_millis(100)); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /.github/workflows/on_pull_request.yml: -------------------------------------------------------------------------------- 1 | name: On Pull Request 2 | 3 | on: pull_request 4 | 5 | jobs: 6 | build_linux: 7 | name: Build on Linux 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - name: Build 12 | run: | 13 | cargo fmt --check 14 | cargo clippy 15 | cargo test 16 | cargo test --release 17 | cargo doc 18 | 19 | build_macos: 20 | name: Build on macOS 21 | runs-on: macos-latest 22 | steps: 23 | - uses: actions/checkout@v2 24 | - name: Build 25 | run: | 26 | cargo fmt --check 27 | cargo clippy 28 | cargo test 29 | cargo test --release 30 | cargo doc 31 | 32 | build_windows: 33 | name: Build on Windows (stub) 34 | runs-on: windows-latest 35 | steps: 36 | - uses: actions/checkout@v2 37 | - name: Build 38 | run: | 39 | cargo build 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2022 Benedek Thaler 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /examples/ipc_write.rs: -------------------------------------------------------------------------------- 1 | //! Example of inter-process communication 2 | //! 3 | //! Start this example first, then `ipc_read`. 4 | 5 | use std::ffi::CString; 6 | use std::io::{Read, Write}; 7 | 8 | #[cfg(target_os = "linux")] 9 | type Mode = libc::mode_t; 10 | 11 | #[cfg(target_os = "macos")] 12 | type Mode = libc::c_uint; 13 | 14 | fn main() -> std::io::Result<()> { 15 | let path = CString::new("cueue_ipc").unwrap(); 16 | let mode = (libc::S_IRUSR | libc::S_IWUSR) as Mode; 17 | let f = unsafe { libc::shm_open(path.as_ptr(), libc::O_RDWR | libc::O_CREAT, mode) }; 18 | if f < 0 { 19 | return Err(std::io::Error::last_os_error()); 20 | } 21 | 22 | let (mut w, _) = cueue::cueue_in_fd(f, Some(1 << 20))?; 23 | 24 | loop { 25 | print!("> "); 26 | std::io::stdout().flush()?; 27 | let buf = w.write_chunk(); 28 | match std::io::stdin().read(buf) { 29 | Ok(0) | Err(_) => break, 30 | Ok(n) => { 31 | w.commit(n); 32 | } 33 | } 34 | } 35 | 36 | unsafe { 37 | libc::shm_unlink(path.as_ptr()); 38 | } 39 | 40 | Ok(()) 41 | } 42 | -------------------------------------------------------------------------------- /examples/basics.rs: -------------------------------------------------------------------------------- 1 | //! An example of constructing, writing and reading a cueue 2 | 3 | fn main() { 4 | // Create a cueue with capacity at least 1M. 5 | // (The actual capacity will be rounded up to match system requirements, if needed) 6 | // w and r are the write and read handles of the cueue, respectively. 7 | // These handles can be sent between threads, but cannot be duplicated. 8 | let (mut w, mut r) = cueue::cueue(1 << 20).unwrap(); 9 | 10 | // To write a cueue, first we need to get a writable slice from it: 11 | let buf = w.write_chunk(); 12 | 13 | // Check if there are 9 bytes free for writing in the cueue. 14 | if buf.len() >= 3 + 3 + 3 { 15 | // If yes, write whatever we want 16 | buf[..9].copy_from_slice(b"foobarbaz"); 17 | 18 | // When done, make the written are available for reading. 19 | // Without this, the reader will not see the written but not committed changes. 20 | w.commit(9); 21 | } 22 | 23 | // Now read whatever is in the queue 24 | let read_result = r.read_chunk(); 25 | assert_eq!(read_result, b"foobarbaz"); 26 | println!("Read {}", String::from_utf8_lossy(read_result)); 27 | // Mark the previously returned slice consumed, making it available for writing. 28 | r.commit(); 29 | } 30 | -------------------------------------------------------------------------------- /benches/ipc_bench.rs: -------------------------------------------------------------------------------- 1 | //! Send timestamps from one process to the other, print send-recv latency (nanoseconds) of each message 2 | //! 3 | //! See ipc_bench.sh 4 | 5 | const NAME: &str = "cueue_ipc_bench"; 6 | 7 | const N: usize = 100_000; 8 | 9 | const MSG_SIZE: usize = 20; 10 | 11 | const DELAY: std::time::Duration = std::time::Duration::from_micros(10); 12 | 13 | use std::ffi::CString; 14 | 15 | fn main() -> std::io::Result<()> { 16 | let runtime = DELAY * N as u32; 17 | if runtime >= std::time::Duration::from_secs(10) { 18 | eprintln!("Expected runtime: {runtime:?}"); 19 | } 20 | 21 | let mode = std::env::args().nth(1).unwrap(); 22 | if mode == "writer" { 23 | writer_main() 24 | } else { 25 | reader_main() 26 | } 27 | } 28 | 29 | fn writer_main() -> std::io::Result<()> { 30 | let name = CString::new(NAME).unwrap(); 31 | let f = unsafe { libc::shm_open(name.as_ptr(), libc::O_RDWR, 0) }; 32 | if f < 0 { 33 | return Err(std::io::Error::last_os_error()); 34 | } 35 | 36 | let (mut w, _) = cueue::cueue_in_fd(f, None)?; 37 | 38 | for _ in 0..N { 39 | let buf = w.write_chunk(); 40 | if buf.len() >= MSG_SIZE { 41 | let ns = monotonic_nanoseconds(); 42 | buf[0..8].copy_from_slice(&ns.to_le_bytes()); 43 | w.commit(MSG_SIZE); 44 | std::thread::sleep(DELAY); 45 | } 46 | } 47 | 48 | Ok(()) 49 | } 50 | 51 | fn reader_main() -> std::io::Result<()> { 52 | let mut latencies = Vec::with_capacity(N); 53 | 54 | let name = CString::new(NAME).unwrap(); 55 | let mode = libc::S_IRUSR | libc::S_IWUSR; 56 | let f = unsafe { libc::shm_open(name.as_ptr(), libc::O_RDWR | libc::O_CREAT, mode) }; 57 | if f < 0 { 58 | return Err(std::io::Error::last_os_error()); 59 | } 60 | 61 | let (_, mut r) = cueue::cueue_in_fd(f, Some(1 << 20))?; 62 | 63 | while latencies.len() < N { 64 | let buf = r.read_chunk(); 65 | let recv = monotonic_nanoseconds(); 66 | let mut ns = [0u8; 8]; 67 | let mut offset = 0; 68 | while offset + MSG_SIZE <= buf.len() { 69 | ns.copy_from_slice(&buf[offset..offset + 8]); 70 | let sent = u64::from_le_bytes(ns); 71 | assert!( 72 | sent < recv, 73 | "back to the future: sent: {sent}, recv: {recv}" 74 | ); 75 | latencies.push(recv - sent); 76 | offset += MSG_SIZE; 77 | } 78 | r.commit(); 79 | } 80 | 81 | let mut out = std::io::stdout().lock(); 82 | for l in latencies { 83 | use std::io::Write; 84 | write!(out, "{l}\n")?; 85 | } 86 | 87 | Ok(()) 88 | } 89 | 90 | fn monotonic_nanoseconds() -> u64 { 91 | let ts = unsafe { 92 | let mut ts = std::mem::MaybeUninit::::uninit(); 93 | libc::clock_gettime(libc::CLOCK_MONOTONIC, ts.as_mut_ptr()); 94 | ts.assume_init() 95 | }; 96 | (ts.tv_sec * 1_000_000_000 + ts.tv_nsec) as u64 97 | 98 | // The bench spends 80% of its time in clock_gettime. 99 | // To get more accurate results, use TSC instead: 100 | // 101 | // use core::arch::x86_64::_rdtsc; 102 | // unsafe { _rdtsc() } 103 | } 104 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cueue 2 | 3 | A high performance, single-producer, single-consumer, bounded circular buffer 4 | of contiguous elements, that supports lock-free atomic batch operations, 5 | suitable for both inter-thread and inter-process communication. 6 | 7 | ## Example 8 | 9 | ```rust 10 | fn main() { 11 | let (mut w, mut r) = cueue::cueue(1 << 20).unwrap(); 12 | 13 | let buf = w.write_chunk(); 14 | assert!(buf.len() >= 9); 15 | buf[..9].copy_from_slice(b"foobarbaz"); 16 | w.commit(9); 17 | 18 | let read_result = r.read_chunk(); 19 | assert_eq!(read_result, b"foobarbaz"); 20 | r.commit(); 21 | } 22 | ``` 23 | 24 | A bounded `cueue` of requested capacity is referenced by a single Writer and a single Reader. 25 | The Writer can request space to write (`write_chunk`), 26 | limited by the queue capacity minus the already committed but unread space. 27 | Requested space can written to, then committed (`end_write`). 28 | A special feature of this container is that stored elements are always initialized, 29 | (in the beginning, defaulted, therefore `T` must implement `Default`), and only 30 | dropped when the queue is dropped. Therefore, the writer can reuse previously 31 | written, but then consumed elements (useful if the elements own e.g: heap allocated memory), 32 | and in those cases, contention on the producers heap lock is avoided (that is otherwise present, 33 | if the consumer drops heap allocated elements continuously that the producer allocated). 34 | 35 | The Reader can check out the written elements (`read_chunk`), process it at will, 36 | then mark it as consumed (`commit`). The returned slice of elements might be a result 37 | of multiple writer commits (i.e: the reading is batched), but it never include uncommitted elements 38 | (i.e: write commits are atomic). This prevents the reader observing partial messages. 39 | 40 | ## Use-case 41 | 42 | This data structure is designed to allow one thread (actor) sending variable-sized messages (bytes) 43 | to a different thread (actor), that processes the messages in batches (e.g: writes them to a file, 44 | sends them over the network, etc.). For example, asynchronous logging. 45 | 46 | Alternative options: 47 | 48 | - Use a standard channel of Strings (or `Vec`). This is slow, because strings require memory allocations, 49 | and with one thread allocating and the other deallocating, quickly yields to contention on the heap lock. 50 | 51 | - Use a standard channel of fixed size arrays. Works, but bounds the size of the messages and 52 | wastes memory. 53 | 54 | - Use two ringbuffers of `Vec` containers (one for sending data, one for reusing the consumed vectors). 55 | Does not allow efficient reading (separate messages are not contiguous). 56 | Requires to estimate the max number of messages in flight, instead of the max sum of size of messages. 57 | 58 | This data structure uses a single array of the user specified capacity. 59 | At any given time, this array is sliced into three logical parts: allocated for writing, 60 | ready for reading, unwritten. (Any maximum two of the three can be zero sized) 61 | 62 | `write_chunk` joins the unwritten part to the part already allocated for writing: 63 | the result is limited by the capacity minus the space ready for reading. 64 | `Writer::commit` makes the written space ready for reading, zeroing the slice allocated for writing. 65 | `read_chunk` determines the boundary of the space ready for reading, 66 | `Reader::commit` marks this space unwritten. Thanks for the truly circular nature of `cueue`, 67 | the writer and reader can freely chase each other around. 68 | 69 | ## How Does it Work 70 | 71 | The `cueue` constructor creates a memory area, and maps it into virtual memory twice, 72 | the two maps next to each other. This means that for the resulting `map` of capacity `cap`, 73 | `map[0]` and `map[cap]`, refers to the same byte. (In general, `map[N]` and `map[cap+N]` are the same 74 | for every `0 <= N < cap` indices) 75 | 76 | With this double map, there's no need to wrap around, this maximises the useful capacity of the queue 77 | during any point of usage, and simplifies the indexing logic of the code. Synchronization 78 | between writer and reader is done by atomic operations, there are no mutexes or lock ASM instruction prefixes 79 | (on the tested platforms: x86 and M1). 80 | 81 | This structure also allows inter-process communication using shared memory, and data recovery from coredumps. 82 | 83 | ## Limitations 84 | 85 | - Supported platforms: Linux (3.17) and macOS 86 | - rust 1.63 87 | - Uses `unsafe` operations 88 | 89 | ## Build and Test 90 | 91 | ```shell 92 | $ cargo build 93 | $ cargo test 94 | $ cargo run --example basics 95 | $ cargo fmt 96 | $ cargo clippy 97 | $ cargo bench 98 | $ cargo doc --open 99 | ``` 100 | 101 | ## Acknowledgments 102 | 103 | This is a rust port of the [binlog][] C++ [cueue][cpp-cueue]. 104 | The interface names are changed to match [rtrb][], to make it more familiar for Rust developers. 105 | 106 | [binlog]: https://github.com/erenon/binlog 107 | [cpp-cueue]: https://github.com/erenon/binlog/blob/hiperf-macos/include/binlog/detail/Cueue.hpp 108 | [rtrb]: https://github.com/mgeier/rtrb 109 | -------------------------------------------------------------------------------- /src/tests.rs: -------------------------------------------------------------------------------- 1 | use crate::*; 2 | 3 | #[test] 4 | fn test_next_power_two() { 5 | assert_eq!(1, next_power_two(0).unwrap()); 6 | assert_eq!(1, next_power_two(1).unwrap()); 7 | assert_eq!(2, next_power_two(2).unwrap()); 8 | assert_eq!(4, next_power_two(3).unwrap()); 9 | assert_eq!(4, next_power_two(4).unwrap()); 10 | assert_eq!(8, next_power_two(5).unwrap()); 11 | assert_eq!(4096, next_power_two(4095).unwrap()); 12 | assert_eq!(4096, next_power_two(4096).unwrap()); 13 | 14 | assert_eq!(1 << 63, next_power_two(1 << 63).unwrap()); 15 | assert!(next_power_two((1 << 63) + 1).is_err()); 16 | } 17 | 18 | #[test] 19 | fn test_capacity() { 20 | let (w, r) = cueue::(16).unwrap(); 21 | assert_eq!(w.capacity(), r.capacity()); 22 | assert!(w.capacity() >= 4096); 23 | } 24 | 25 | #[test] 26 | fn test_writer() { 27 | let (mut w, r) = cueue::(16).unwrap(); 28 | 29 | let cap = w.capacity(); 30 | 31 | let buf = w.write_chunk(); 32 | assert_eq!(buf.len(), cap); 33 | w.commit(0); 34 | 35 | let buf = w.write_chunk(); 36 | assert_eq!(buf.len(), cap); 37 | w.commit(3); 38 | 39 | let buf = w.write_chunk(); 40 | assert_eq!(buf.len(), cap - 3); 41 | 42 | assert!(!w.is_abandoned()); 43 | std::mem::drop(r); 44 | assert!(w.is_abandoned()); 45 | } 46 | 47 | #[test] 48 | fn test_reader() { 49 | let (mut w, mut r) = cueue(16).unwrap(); 50 | 51 | let empty = r.read_chunk(); 52 | assert_eq!(empty.len(), 0); 53 | r.commit(); 54 | 55 | let buf = w.write_chunk(); 56 | buf[..3].copy_from_slice(b"foo"); 57 | w.commit(3); 58 | 59 | let foo = r.read_chunk(); 60 | assert_eq!(foo, b"foo"); 61 | r.commit(); 62 | 63 | assert!(!r.is_abandoned()); 64 | std::mem::drop(w); 65 | assert!(r.is_abandoned()); 66 | } 67 | 68 | #[test] 69 | fn test_limited_read_chunk() { 70 | let (mut w, mut r) = cueue(16).unwrap(); 71 | 72 | let empty = r.limited_read_chunk(8); 73 | assert_eq!(empty.len(), 0); 74 | r.commit(); 75 | 76 | let buf = w.write_chunk(); 77 | buf[..3].copy_from_slice(b"foo"); 78 | w.commit(3); 79 | 80 | let foo = r.limited_read_chunk(1); 81 | assert_eq!(foo, b"f"); 82 | r.commit(); 83 | 84 | let foo = r.limited_read_chunk(4); 85 | assert_eq!(foo, b"oo"); 86 | let foo = r.limited_read_chunk(2); 87 | assert_eq!(foo, b"oo"); 88 | r.commit(); 89 | 90 | let empty = r.limited_read_chunk(2); 91 | assert_eq!(empty.len(), 0); 92 | } 93 | 94 | #[test] 95 | fn test_full() { 96 | let (mut w, mut r) = cueue::(16).unwrap(); 97 | 98 | let buf = w.write_chunk(); 99 | let buflen = buf.len(); 100 | assert_eq!(buf.len(), w.capacity()); 101 | w.commit(buflen); 102 | 103 | let empty = w.write_chunk(); 104 | assert_eq!(empty.len(), 0); 105 | 106 | let full = r.read_chunk(); 107 | assert_eq!(full.len(), buflen); 108 | assert_eq!(full.len(), r.capacity()); 109 | } 110 | 111 | #[test] 112 | fn test_reuse() { 113 | let (mut w, mut r) = cueue(16).unwrap(); 114 | 115 | // fill the queue with strings 116 | let buf = w.write_chunk(); 117 | for s in buf.into_iter() { 118 | *s = "foobar"; 119 | } 120 | let buflen = buf.len(); 121 | w.commit(buflen); 122 | 123 | // consume everything 124 | let full = r.read_chunk(); 125 | assert_eq!(full.len(), buflen); 126 | r.commit(); 127 | 128 | // try writing again 129 | let buf = w.write_chunk(); 130 | assert_eq!(buf[0], "foobar"); 131 | } 132 | 133 | #[test] 134 | fn test_push() { 135 | let (mut w, _) = cueue(16).unwrap(); 136 | let cap = w.capacity(); 137 | 138 | for i in 0..cap { 139 | assert_eq!(w.push(i), Ok(())); 140 | } 141 | 142 | assert_eq!(w.push(0), Err(0)); 143 | } 144 | 145 | #[test] 146 | fn test_push_string() { 147 | let (mut w, _) = cueue(16).unwrap(); 148 | let cap = w.capacity(); 149 | 150 | for i in 0..cap { 151 | assert_eq!(w.push(i.to_string()), Ok(())); 152 | } 153 | 154 | assert_eq!(w.push("foo".to_string()), Err("foo".to_string())); 155 | } 156 | 157 | #[test] 158 | fn test_cueue_threaded_w_r() { 159 | let (mut w, mut r) = cueue(16).unwrap(); 160 | let maxi = 1_000_000; 161 | 162 | let wt = std::thread::spawn(move || { 163 | let mut msg: u8 = 0; 164 | for _ in 0..maxi { 165 | let buf = loop { 166 | let buf = w.write_chunk(); 167 | if buf.len() > 0 { 168 | break buf; 169 | } 170 | }; 171 | buf[0] = msg; 172 | w.commit(1); 173 | 174 | msg = msg.wrapping_add(1); 175 | } 176 | }); 177 | 178 | let rt = std::thread::spawn(move || { 179 | let mut emsg: u8 = 0; 180 | let mut i = 0; 181 | while i < maxi { 182 | let rr = r.read_chunk(); 183 | for msg in rr { 184 | assert_eq!(*msg, emsg); 185 | emsg = emsg.wrapping_add(1); 186 | i += 1; 187 | } 188 | r.commit(); 189 | } 190 | }); 191 | 192 | wt.join().unwrap(); 193 | rt.join().unwrap(); 194 | } 195 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! A high performance, single-producer, single-consumer, bounded circular buffer 2 | //! of contiguous elements, that supports lock-free atomic batch operations, 3 | //! suitable for inter-thread communication. 4 | //! 5 | //!``` 6 | //! let (mut w, mut r) = cueue::cueue(1 << 20).unwrap(); 7 | //! 8 | //! let buf = w.write_chunk(); 9 | //! assert!(buf.len() >= 9); 10 | //! buf[..9].copy_from_slice(b"foobarbaz"); 11 | //! w.commit(9); 12 | //! 13 | //! let read_result = r.read_chunk(); 14 | //! assert_eq!(read_result, b"foobarbaz"); 15 | //! r.commit(); 16 | //!``` 17 | //! 18 | //! Elements in the queue are always initialized, and not dropped until the queue is dropped. 19 | //! This allows re-use of elements (useful for elements with heap allocated contents), 20 | //! and prevents contention on the senders heap (by avoiding the consumer freeing memory 21 | //! the sender allocated). 22 | 23 | #[cfg(any(target_os = "linux", target_os = "macos"))] 24 | use std::ffi::CString; 25 | #[cfg(any(target_os = "linux", target_os = "macos"))] 26 | use std::os::unix::io::{AsRawFd, FromRawFd, OwnedFd, RawFd}; 27 | use std::sync::atomic::Ordering; 28 | 29 | #[cfg(any(target_os = "linux", target_os = "macos"))] 30 | use libc::{c_void, ftruncate, mmap, munmap, sysconf}; 31 | #[cfg(any(target_os = "linux", target_os = "macos"))] 32 | use libc::{ 33 | MAP_ANONYMOUS, MAP_FAILED, MAP_FIXED, MAP_PRIVATE, MAP_SHARED, PROT_READ, PROT_WRITE, 34 | _SC_PAGESIZE, 35 | }; 36 | 37 | fn errno_with_hint(hint: &str) -> std::io::Error { 38 | std::io::Error::new(std::io::Error::last_os_error().kind(), hint) 39 | } 40 | 41 | /// Create a file descriptor that points to a location in memory. 42 | #[cfg(target_os = "linux")] 43 | unsafe fn memoryfile() -> std::io::Result { 44 | let name = CString::new("cueue").unwrap(); 45 | let memfd = libc::memfd_create(name.as_ptr(), 0); 46 | if memfd < 0 { 47 | return Err(errno_with_hint("memfd_create")); 48 | } 49 | Ok(OwnedFd::from_raw_fd(memfd)) 50 | } 51 | 52 | #[cfg(target_os = "macos")] 53 | unsafe fn memoryfile() -> std::io::Result { 54 | let path = CString::new("/tmp/cueue_XXXXXX").unwrap(); 55 | let path_cstr = path.into_raw(); 56 | let tmpfd = libc::mkstemp(path_cstr); 57 | let path = CString::from_raw(path_cstr); 58 | if tmpfd < 0 { 59 | return Err(errno_with_hint("mkstemp")); 60 | } 61 | let memfd = libc::shm_open(path.as_ptr(), libc::O_RDWR | libc::O_CREAT | libc::O_EXCL); 62 | libc::unlink(path.as_ptr()); 63 | libc::close(tmpfd); 64 | if memfd < 0 { 65 | return Err(errno_with_hint("shm_open")); 66 | } 67 | 68 | Ok(OwnedFd::from_raw_fd(memfd)) 69 | } 70 | 71 | #[cfg(not(any(target_os = "linux", target_os = "macos")))] 72 | unsafe fn memoryfile() { 73 | todo!("Only Linux and macOS are supported so far"); 74 | } 75 | 76 | /// A chunk of memory allocated using mmap. 77 | /// 78 | /// Deallocates the memory on Drop. 79 | #[cfg(any(target_os = "linux", target_os = "macos"))] 80 | struct MemoryMap { 81 | map: *mut c_void, 82 | size: usize, 83 | } 84 | 85 | #[cfg(any(target_os = "linux", target_os = "macos"))] 86 | impl MemoryMap { 87 | fn new(map: *mut c_void, size: usize) -> Self { 88 | Self { map, size } 89 | } 90 | 91 | fn failed(&self) -> bool { 92 | self.map == MAP_FAILED 93 | } 94 | 95 | fn ptr(&self) -> *mut u8 { 96 | self.map as *mut u8 97 | } 98 | } 99 | 100 | #[cfg(any(target_os = "linux", target_os = "macos"))] 101 | impl Drop for MemoryMap { 102 | fn drop(&mut self) { 103 | if !self.failed() { 104 | unsafe { 105 | munmap(self.map, self.size); 106 | } 107 | } 108 | } 109 | } 110 | 111 | #[cfg(not(any(target_os = "linux", target_os = "macos")))] 112 | struct MemoryMap {} 113 | 114 | #[cfg(not(any(target_os = "linux", target_os = "macos")))] 115 | impl MemoryMap { 116 | fn ptr(&self) -> *mut u8 { 117 | todo!("Only Linux and macOS are supported so far"); 118 | } 119 | } 120 | 121 | struct MemoryMapInitialized { 122 | map: MemoryMap, 123 | buf: *mut T, 124 | cap: usize, 125 | } 126 | 127 | impl MemoryMapInitialized 128 | where 129 | T: Default, 130 | { 131 | fn new(map: MemoryMap, buf: *mut T, cap: usize) -> Self { 132 | for i in 0..cap { 133 | unsafe { 134 | buf.add(i).write(T::default()); 135 | } 136 | } 137 | Self { map, buf, cap } 138 | } 139 | 140 | #[inline] 141 | fn controlblock(&self) -> *mut ControlBlock { 142 | self.map.ptr().cast::() 143 | } 144 | } 145 | 146 | impl Drop for MemoryMapInitialized { 147 | fn drop(&mut self) { 148 | for i in 0..self.cap { 149 | unsafe { 150 | self.buf.add(i).drop_in_place(); 151 | } 152 | } 153 | } 154 | } 155 | 156 | /// Platform specific flags that increase performance, but not required. 157 | #[cfg(target_os = "linux")] 158 | fn platform_flags() -> i32 { 159 | libc::MAP_POPULATE 160 | } 161 | 162 | #[cfg(not(target_os = "linux"))] 163 | fn platform_flags() -> i32 { 164 | 0 165 | } 166 | 167 | /// Map a `size` chunk of `fd` at `offset` twice, next to each other in virtual memory 168 | /// Also map [0,offset) for the control block. 169 | /// The size of the file pointed by `fd` must be >= offset + size. 170 | #[cfg(any(target_os = "linux", target_os = "macos"))] 171 | unsafe fn doublemap(fd: RawFd, offset: usize, size: usize) -> std::io::Result { 172 | // Create a map, offset + twice the size, to get a suitable virtual address which will work with MAP_FIXED 173 | let rw = PROT_READ | PROT_WRITE; 174 | let mapsize = offset + size * 2; 175 | let map = MemoryMap::new( 176 | mmap( 177 | std::ptr::null_mut(), 178 | mapsize, 179 | rw, 180 | MAP_PRIVATE | MAP_ANONYMOUS, 181 | -1, 182 | 0, 183 | ), 184 | mapsize, 185 | ); 186 | if map.failed() { 187 | return Err(errno_with_hint("mmap 1")); 188 | } 189 | 190 | let cb = mmap( 191 | map.ptr() as *mut c_void, 192 | offset, 193 | rw, 194 | MAP_SHARED | MAP_FIXED, 195 | fd, 196 | 0, 197 | ); 198 | if cb == MAP_FAILED { 199 | return Err(errno_with_hint("mmap cb")); 200 | } 201 | 202 | // Map f twice, put maps next to each other with MAP_FIXED 203 | // MAP_SHARED is required to have the changes propagated between maps 204 | let first_addr = map.ptr().add(offset) as *mut c_void; 205 | let first_map = mmap( 206 | first_addr, 207 | size, 208 | rw, 209 | MAP_SHARED | MAP_FIXED | platform_flags(), 210 | fd, 211 | offset as i64, 212 | ); 213 | if first_map != first_addr { 214 | return Err(errno_with_hint("mmap 2")); 215 | } 216 | 217 | let second_addr = map.ptr().add(offset + size) as *mut c_void; 218 | let second_map = mmap( 219 | second_addr, 220 | size, 221 | rw, 222 | MAP_SHARED | MAP_FIXED, 223 | fd, 224 | offset as i64, 225 | ); 226 | if second_map != second_addr { 227 | return Err(errno_with_hint("mmap 3")); 228 | } 229 | 230 | // man mmap: 231 | // If the memory region specified by addr and len overlaps 232 | // pages of any existing mapping(s), then the overlapped part 233 | // of the existing mapping(s) will be discarded. 234 | // -> No need to munmap `first_map` and `second_map`, drop(map) will do both 235 | 236 | Ok(map) 237 | } 238 | 239 | #[cfg(not(any(target_os = "linux", target_os = "macos")))] 240 | unsafe fn doublemap() { 241 | todo!("Only Linux and macOS are supported so far"); 242 | } 243 | 244 | /// Returns smallest power of 2 not smaller than `n`, 245 | /// or an error if the expected result cannot be represented by the return type. 246 | fn next_power_two(n: usize) -> std::io::Result { 247 | if n == 0 { 248 | return Ok(1); 249 | } 250 | 251 | let mut m = n - 1; 252 | let mut result = 1; 253 | while m != 0 { 254 | m >>= 1; 255 | result <<= 1; 256 | } 257 | 258 | if result >= n { 259 | Ok(result) 260 | } else { 261 | Err(std::io::Error::other("next_power_two")) 262 | } 263 | } 264 | 265 | /// Force an AtomicU64 to a separate cache-line to avoid false-sharing. 266 | /// This wrapper is needed as I was unable to specify alignment for individual fields. 267 | #[repr(align(128))] 268 | #[derive(Default)] 269 | struct CacheLineAlignedAU64(std::sync::atomic::AtomicU64); 270 | 271 | /// The shared metadata of a Cueue. 272 | /// 273 | /// Cueue is empty if R == W 274 | /// Cueue is full if W == R+capacity 275 | /// Invariant: W >= R 276 | /// Invariant: R + capacity >= W 277 | #[repr(C)] 278 | struct ControlBlock { 279 | write_position: CacheLineAlignedAU64, 280 | read_position: CacheLineAlignedAU64, 281 | capacity: u64, 282 | } 283 | 284 | impl ControlBlock { 285 | pub fn new(capacity: usize) -> Self { 286 | ControlBlock { 287 | write_position: CacheLineAlignedAU64(0.into()), 288 | read_position: CacheLineAlignedAU64(0.into()), 289 | capacity: capacity as u64, 290 | } 291 | } 292 | } 293 | 294 | /// Writer of a Cueue. 295 | /// 296 | /// See examples/ for usage. 297 | pub struct Writer { 298 | mem: std::sync::Arc>, 299 | cb: *mut ControlBlock, 300 | mask: u64, 301 | 302 | buffer: *mut T, 303 | write_begin: *mut T, 304 | write_capacity: usize, 305 | } 306 | 307 | impl Writer 308 | where 309 | T: Default, 310 | { 311 | fn new(mem: std::sync::Arc>, buffer: *mut T) -> Self { 312 | let cb = mem.controlblock(); 313 | let capacity = unsafe { (*cb).capacity }; 314 | Self { 315 | mem, 316 | cb, 317 | mask: capacity - 1, 318 | buffer, 319 | write_begin: std::ptr::null_mut(), 320 | write_capacity: 0, 321 | } 322 | } 323 | 324 | /// Maximum number of elements the referenced `cueue` can hold. 325 | #[inline] 326 | pub fn capacity(&self) -> usize { 327 | (self.mask + 1) as usize 328 | } 329 | 330 | /// Get a writable slice of maximum available size. 331 | /// 332 | /// The elements in the returned slice are either default initialized 333 | /// (never written yet) or are the result of previous writes. 334 | /// The writer is free to overwrite or reuse them. 335 | /// 336 | /// After write, `commit` must be called, to make the written elements 337 | /// available for reading. 338 | pub fn write_chunk(&mut self) -> &mut [T] { 339 | let w = self.write_pos().load(Ordering::Relaxed); 340 | let r = self.read_pos().load(Ordering::Acquire); 341 | 342 | debug_assert!(r <= w); 343 | debug_assert!(r + self.capacity() as u64 >= w); 344 | 345 | let wi = w & self.mask; 346 | self.write_capacity = (self.capacity() as u64 - (w.wrapping_sub(r))) as usize; 347 | 348 | unsafe { 349 | self.write_begin = self.buffer.offset(wi as isize); 350 | std::slice::from_raw_parts_mut(self.write_begin, self.write_capacity) 351 | } 352 | } 353 | 354 | /// Make `n` number of elements, written to the slice returned by `write_chunk` 355 | /// available for reading. 356 | /// 357 | /// `n` is checked: if too large, gets truncated to the maximum committable size. 358 | /// 359 | /// Returns the number of committed elements. 360 | pub fn commit(&mut self, n: usize) -> usize { 361 | let m = usize::min(self.write_capacity, n); 362 | unsafe { 363 | self.unchecked_commit(m); 364 | } 365 | m 366 | } 367 | 368 | unsafe fn unchecked_commit(&mut self, n: usize) { 369 | let w = self.write_pos().load(Ordering::Relaxed); 370 | self.write_begin = self.write_begin.add(n); 371 | self.write_capacity -= n; 372 | self.write_pos().store(w + n as u64, Ordering::Release); 373 | } 374 | 375 | /// Returns true, if the Reader counterpart was dropped. 376 | pub fn is_abandoned(&self) -> bool { 377 | std::sync::Arc::strong_count(&self.mem) < 2 378 | } 379 | 380 | /// Write and commit a single element, or return it if the queue was full. 381 | pub fn push(&mut self, t: T) -> Result<(), T> { 382 | let chunk = self.write_chunk(); 383 | if !chunk.is_empty() { 384 | chunk[0] = t; 385 | self.commit(1); 386 | Ok(()) 387 | } else { 388 | Err(t) 389 | } 390 | } 391 | 392 | #[inline] 393 | fn write_pos(&self) -> &std::sync::atomic::AtomicU64 { 394 | unsafe { &(*self.cb).write_position.0 } 395 | } 396 | 397 | #[inline] 398 | fn read_pos(&self) -> &std::sync::atomic::AtomicU64 { 399 | unsafe { &(*self.cb).read_position.0 } 400 | } 401 | } 402 | 403 | unsafe impl Send for Writer {} 404 | 405 | /// Reader of a Cueue. 406 | /// 407 | /// See examples/ for usage. 408 | pub struct Reader { 409 | mem: std::sync::Arc>, 410 | cb: *mut ControlBlock, 411 | mask: u64, 412 | 413 | buffer: *const T, 414 | read_begin: *const T, 415 | read_size: u64, 416 | } 417 | 418 | impl Reader 419 | where 420 | T: Default, 421 | { 422 | fn new(mem: std::sync::Arc>, buffer: *const T) -> Self { 423 | let cb = mem.controlblock(); 424 | let capacity = unsafe { (*cb).capacity }; 425 | Self { 426 | mem, 427 | cb, 428 | mask: capacity - 1, 429 | buffer, 430 | read_begin: std::ptr::null(), 431 | read_size: 0, 432 | } 433 | } 434 | 435 | /// Maximum number of elements the referenced `cueue` can hold. 436 | #[inline] 437 | pub fn capacity(&self) -> usize { 438 | (self.mask + 1) as usize 439 | } 440 | 441 | /// Return a slice of elements written and committed by the Writer. 442 | pub fn read_chunk(&mut self) -> &[T] { 443 | let w = self.write_pos().load(Ordering::Acquire); 444 | let r = self.read_pos().load(Ordering::Relaxed); 445 | 446 | debug_assert!(r <= w); 447 | debug_assert!(r + self.capacity() as u64 >= w); 448 | 449 | let ri = r & self.mask; 450 | 451 | self.read_size = w - r; 452 | 453 | unsafe { 454 | self.read_begin = self.buffer.offset(ri as isize); 455 | std::slice::from_raw_parts(self.read_begin, self.read_size as usize) 456 | } 457 | } 458 | 459 | /// Return a slice of elements written and committed by the Writer. 460 | /// 461 | /// The length of the returned slice will be less or equal than `n`. 462 | pub fn limited_read_chunk(&mut self, n: u64) -> &[T] { 463 | let w = self.write_pos().load(Ordering::Acquire); 464 | let r = self.read_pos().load(Ordering::Relaxed); 465 | 466 | debug_assert!(r <= w); 467 | debug_assert!(r + self.capacity() as u64 >= w); 468 | 469 | let ri = r & self.mask; 470 | let rs = n.min(w - r); 471 | 472 | self.read_size = rs; 473 | 474 | unsafe { 475 | self.read_begin = self.buffer.add(ri as usize); 476 | std::slice::from_raw_parts(self.read_begin, self.read_size as usize) 477 | } 478 | } 479 | 480 | /// Mark the slice previously acquired by `read_chunk` as consumed, 481 | /// making it available for writing. 482 | pub fn commit(&mut self) { 483 | let r = self.read_pos().load(Ordering::Relaxed); 484 | let rs = self.read_size; 485 | self.read_pos().store(r + rs, Ordering::Release); 486 | } 487 | 488 | /// Returns true, if the Writer counterpart was dropped. 489 | pub fn is_abandoned(&self) -> bool { 490 | std::sync::Arc::strong_count(&self.mem) < 2 491 | } 492 | 493 | #[inline] 494 | fn write_pos(&self) -> &std::sync::atomic::AtomicU64 { 495 | unsafe { &(*self.cb).write_position.0 } 496 | } 497 | 498 | #[inline] 499 | fn read_pos(&self) -> &std::sync::atomic::AtomicU64 { 500 | unsafe { &(*self.cb).read_position.0 } 501 | } 502 | } 503 | 504 | unsafe impl Send for Reader {} 505 | 506 | /// Create a single-producer, single-consumer `Cueue`. 507 | /// 508 | /// The `requested_capacity` is a lower bound of the actual capacity 509 | /// of the constructed queue: it might be rounded up to match system requirements 510 | /// (power of two, multiple of page size). 511 | /// 512 | /// `requested_capacity` must not be bigger than 2^63. 513 | /// 514 | /// On success, returns a `(Writer, Reader)` pair, that share the ownership 515 | /// of the underlying circular array. 516 | #[cfg(any(target_os = "linux", target_os = "macos"))] 517 | pub fn cueue(requested_capacity: usize) -> std::io::Result<(Writer, Reader)> 518 | where 519 | T: Default, 520 | { 521 | let f = unsafe { memoryfile()? }; 522 | cueue_in_fd(f.as_raw_fd(), Some(requested_capacity)) 523 | } 524 | 525 | // TODO bad: already_initialized vs. requested_capacity 526 | 527 | /// Like `cueue`, but takes a file descriptor `f`, to put the queue into. 528 | /// 529 | /// If `requested_capacity` is Some, it'll initialize the queue for the 530 | /// given size, otherwise it assumes the queue is already initialized. 531 | /// 532 | /// This can be used with a file that is setup for inter-process communication, 533 | /// see the `ipc_write` and `ipc_read` examples. Each handle (Writer and Reader) 534 | /// must be used in a single process only: the queue is unidirectional. 535 | #[cfg(any(target_os = "linux", target_os = "macos"))] 536 | pub fn cueue_in_fd( 537 | f: RawFd, 538 | requested_capacity: Option, 539 | ) -> std::io::Result<(Writer, Reader)> 540 | where 541 | T: Default, 542 | { 543 | let pagesize = unsafe { sysconf(_SC_PAGESIZE) as usize }; 544 | 545 | if std::mem::size_of::() > pagesize { 546 | return Err(std::io::Error::other( 547 | "ControlBlock does not fit in a single page", 548 | )); 549 | } 550 | 551 | let initmap; 552 | let buf; 553 | 554 | if let Some(requested_capacity) = requested_capacity { 555 | // create the queue 556 | let cap = next_power_two(usize::max(requested_capacity, pagesize))?; 557 | let bufsize = cap * std::mem::size_of::(); 558 | 559 | unsafe { 560 | if ftruncate(f.as_raw_fd(), (pagesize + bufsize) as i64) != 0 { 561 | return Err(errno_with_hint("ftruncate")); 562 | } 563 | let map = doublemap(f.as_raw_fd(), pagesize, bufsize)?; 564 | buf = map.ptr().add(pagesize).cast::(); 565 | 566 | // initialize control block 567 | let cbp = map.ptr() as *mut ControlBlock; 568 | cbp.write(ControlBlock::new(cap)); 569 | 570 | // default initialize elems. 571 | // this is required to make sure writer always sees initialized elements 572 | initmap = MemoryMapInitialized::new(map, buf, cap) 573 | } 574 | } else { 575 | // the queue is already created, attach to it 576 | let cap = unsafe { 577 | let mut cb = std::mem::MaybeUninit::::uninit(); 578 | let cbsize = std::mem::size_of::(); 579 | let rs = libc::read(f, cb.as_mut_ptr() as *mut c_void, cbsize); 580 | if rs < cbsize as isize { 581 | return Err(std::io::Error::other( 582 | "Failed to read control block from file", 583 | )); 584 | } 585 | 586 | cb.assume_init().capacity as usize 587 | }; 588 | let bufsize = cap * std::mem::size_of::(); 589 | 590 | unsafe { 591 | let map = doublemap(f.as_raw_fd(), pagesize, bufsize)?; 592 | buf = map.ptr().add(pagesize).cast::(); 593 | initmap = MemoryMapInitialized { map, buf, cap } 594 | } 595 | } 596 | 597 | let shared_map = std::sync::Arc::new(initmap); 598 | 599 | Ok(( 600 | Writer::new(shared_map.clone(), buf), 601 | Reader::new(shared_map, buf), 602 | )) 603 | } 604 | 605 | #[cfg(not(any(target_os = "linux", target_os = "macos")))] 606 | pub fn cueue(requested_capacity: usize) -> std::io::Result<(Writer, Reader)> 607 | where 608 | T: Default, 609 | { 610 | todo!("Only Linux and macOS are supported so far"); 611 | } 612 | 613 | #[cfg(test)] 614 | mod tests; 615 | --------------------------------------------------------------------------------