├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .rustfmt ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE ├── README.md ├── benches ├── reader.rs └── writer.rs ├── fuzz ├── .gitignore ├── Cargo.toml └── fuzz_targets │ ├── read.rs │ └── write.rs ├── src ├── lib.rs ├── read.rs └── write.rs └── tests ├── read.rs └── write.rs /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | on: 3 | pull_request: 4 | push: 5 | branches: 6 | - master 7 | 8 | # Make sure CI fails on all warnings, including clippy lints 9 | env: 10 | RUSTFLAGS: "-Dwarnings" 11 | 12 | jobs: 13 | test: 14 | name: test 15 | runs-on: ${{ matrix.os }} 16 | strategy: 17 | matrix: 18 | build: [pinned, stable, beta, nightly, macos, win-msvc, win-gnu] 19 | include: 20 | - build: pinned 21 | os: ubuntu-22.04 22 | rust: 1.38.0 23 | - build: stable 24 | os: ubuntu-22.04 25 | rust: stable 26 | - build: beta 27 | os: ubuntu-22.04 28 | rust: beta 29 | - build: nightly 30 | os: ubuntu-22.04 31 | rust: nightly 32 | - build: macos 33 | os: macos-12 34 | rust: stable 35 | - build: win-msvc 36 | os: windows-2022 37 | rust: stable 38 | - build: win-gnu 39 | os: windows-2022 40 | rust: stable-x86_64-gnu 41 | steps: 42 | - name: Checkout repository 43 | uses: actions/checkout@v3 44 | - name: Install Rust 45 | uses: dtolnay/rust-toolchain@master 46 | with: 47 | toolchain: ${{ matrix.rust }} 48 | - run: cargo build --verbose 49 | - run: cargo doc --verbose 50 | - run: cargo test --verbose 51 | - if: matrix.build == 'nightly' 52 | run: cargo bench --verbose --no-run 53 | 54 | clippy: 55 | name: clippy 56 | runs-on: ubuntu-22.04 57 | steps: 58 | - name: Checkout repository 59 | uses: actions/checkout@v3 60 | - name: Install Rust 61 | uses: dtolnay/rust-toolchain@master 62 | with: 63 | toolchain: nightly 64 | components: clippy 65 | - name: Clippy check 66 | run: cargo clippy --all-targets 67 | 68 | rustfmt: 69 | name: rustfmt 70 | runs-on: ubuntu-22.04 71 | steps: 72 | - name: Checkout repository 73 | uses: actions/checkout@v3 74 | - name: Install Rust 75 | uses: dtolnay/rust-toolchain@master 76 | with: 77 | toolchain: stable 78 | components: rustfmt 79 | - name: Check formatting 80 | run: cargo fmt --check 81 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | Cargo.lock 3 | *~ 4 | -------------------------------------------------------------------------------- /.rustfmt: -------------------------------------------------------------------------------- 1 | max_width = 80 -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | # [0.3.0] - 2020-09-15 4 | This release most importantly fixes error handling, which was partially broken 5 | in the 0.2 release. Users should upgrade. 6 | - Fix error handling (fixes #2, thanks @winstonewert) and improve documentation 7 | - Make flush calls respect the order relative to write calls 8 | (5bb26be195bec2f609cf3f7712c6d47e0b19f407) 9 | - More extensive documentation 10 | - The crate now relies on the 2018 Rust edition 11 | - Misc. smaller improvements 12 | 13 | # [0.2.0] - 2018-12-25 14 | - Initial release 15 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "thread_io" 3 | version = "0.3.1" 4 | rust-version = "1.38" 5 | authors = ["markschl "] 6 | description = "Crate for performing I/O in background thread" 7 | license = "MIT" 8 | repository = "https://github.com/markschl/thread_io" 9 | documentation = "https://docs.rs/thread_io" 10 | readme = "README.md" 11 | keywords = ["io", "thread", "concurrency"] 12 | edition = "2018" 13 | 14 | [dependencies] 15 | crossbeam-channel = "0.5" 16 | crossbeam-utils = "0.8" 17 | 18 | [features] 19 | crossbeam_channel = [] 20 | 21 | [badges] 22 | travis = { repository = "https://travis-ci.org/markschl/thread_io" } 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 markschl 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # thread_io 2 | 3 | [![docs.rs](https://docs.rs/thread_io/badge.svg)](https://docs.rs/thread_io/latest/thread_io) 4 | [![crates.io](https://img.shields.io/crates/v/thread_io.svg)](https://crates.io/crates/thread_io) 5 | [![Build status](https://github.com/markschl/thread_io/workflows/ci/badge.svg)](https://github.com/markschl/thread_io/actions) 6 | 7 | This crate allows to easily wrap readers and writers in a background thread. 8 | This can be useful e.g. with readers and writers for compression formats to 9 | reduce load on the main thread. 10 | 11 | `thread_io` uses channels (optionally from the 12 | [crossbeam crate](https://docs.rs/crossbeam/latest/crossbeam/channel/index.html)) 13 | for communicating and exchanging chunks of data with the background reader / 14 | writer. 15 | 16 | **[Reader API documentation](https://docs.rs/thread_io/latest/thread_io/read)** 17 | 18 | **[Writer API documentation](https://docs.rs/thread_io/latest/thread_io/write)** 19 | 20 | The minimum Rust version is *1.38.0*. 21 | 22 | ## Examples 23 | 24 | ### Reading 25 | 26 | The following code counts the number of lines containing *spam* in a gzip 27 | compressed file. Decompression is done in a background thread using the `flate2` 28 | library, and the decompressed data is sent to a reader supplied to a closure in 29 | the main thread. The speed gain should be highest if decompression and text 30 | searching use about the same amount of CPU time. 31 | 32 | The resulting line number should be the same as the output of 33 | `zcat file.txt.gz | grep 'spam' | wc -l`. 34 | 35 | ```rust 36 | use io::prelude::*; 37 | use io; 38 | use fs::File; 39 | use thread_io::read::reader; 40 | use flate2::read::GzDecoder; 41 | 42 | // size of buffers sent across threads 43 | const BUF_SIZE: usize = 256 * 1024; 44 | // length of queue with buffers pre-filled in background thread 45 | const QUEUE_LEN: usize = 5; 46 | 47 | let f = File::open("file.txt.gz").unwrap(); 48 | let gz = GzDecoder::new(f); 49 | let search_term = "spam"; 50 | 51 | let found = reader( 52 | BUF_SIZE, 53 | QUEUE_LEN, 54 | gz, 55 | |reader| { 56 | let mut buf_reader = io::BufReader::new(reader); 57 | let mut found = 0; 58 | let mut line = String::new(); 59 | while buf_reader.read_line(&mut line)? > 0 { 60 | if line.contains(search_term) { 61 | found += 1; 62 | } 63 | line.clear(); 64 | } 65 | Ok::<_, io::Error>(found) 66 | } 67 | ) 68 | .expect("decoding error"); 69 | 70 | println!("Found '{}' in {} lines.", search_term, found); 71 | ``` 72 | 73 | Note that this is an example for illustration. To increase performance, one 74 | could read lines into a `Vec` buffer (instead of `String`) and search for 75 | *spam* e.g. using `memchr` from the [memchr crate](https://crates.io/crates/memchr). 76 | 77 | The compiler sometimes needs a hint about the exact error type returned from 78 | `func`, in this case this was done by specifying `Ok::<_, io::Error>()` as 79 | return value. 80 | 81 | `thread_io::read::reader` requires the underlying reader to implement `Send`. 82 | Unfortunately, this is not always the case, such as with `io::StdinLock`. 83 | There is the [`thread_io::read::reader_init`](https://docs.rs/thread_io/latest/thread_io/read/fn.reader_init.html) 84 | function to handle such cases. 85 | 86 | ### Writing 87 | 88 | Writing to a gzip compressed file in a background thread works similarly as 89 | reading. The following code writes all lines containing *spam* to a compressed 90 | file. The contents of the compressed output file `file.txt.gz` should be the 91 | same as if running `grep 'spam' file.txt | gzip -c > file.txt.gz` 92 | 93 | ```rust 94 | use fs::File; 95 | use io::prelude::*; 96 | use io; 97 | use thread_io::write::writer; 98 | use flate2::write::{GzEncoder}; 99 | use flate2::Compression; 100 | 101 | const BUF_SIZE: usize = 256 * 1024; 102 | const QUEUE_LEN: usize = 5; 103 | 104 | let infile = File::open("file.txt").unwrap(); 105 | let outfile = File::create("file.txt.gz").unwrap(); 106 | let mut gz_out = GzEncoder::new(outfile, Compression::default()); 107 | let search_term = "spam"; 108 | 109 | writer( 110 | BUF_SIZE, 111 | QUEUE_LEN, 112 | &mut gz_out, 113 | |writer| { 114 | // This function runs in the main thread, all writes are written to 115 | // 'gz_out' in the background 116 | let mut buf_infile = io::BufReader::new(infile); 117 | let mut line = String::new(); 118 | while buf_infile.read_line(&mut line)? > 0 { 119 | if line.contains(search_term) { 120 | writer.write(line.as_bytes()).expect("write error"); 121 | } 122 | line.clear(); 123 | } 124 | Ok::<_, io::Error>(()) 125 | }, 126 | ) 127 | .expect("encoding error"); 128 | gz_out.finish().expect("finishing failed"); 129 | ``` 130 | 131 | More details on the exact behavior and more flexible functions e.g. for dealing 132 | with *non-Send* writer types can be found in the documentation of the 133 | [write module](https://docs.rs/thread_io/latest/thread_io/write). 134 | 135 | After `func` returns, the background writer *always* calls 136 | `io::Write::flush`, making sure that possible flushing errors are caught before 137 | the file goes out of scope. 138 | 139 | ## Notes on errors 140 | 141 | Two types of errors may occur when using the readers and writers of this crate: 142 | 143 | * **`io::Error`** returned from `io::Read::read` / `io::Write::write` calls. 144 | This error cannot be returned *instantly*, instead it is pushed to a queue and 145 | will be returned in a subsequent read or write call. The delay depends on the 146 | `queuelen` parameter of the reading / writing functions, but also on the 147 | `bufsize` parameter and the size of the reading / writing buffer. 148 | * The `func` closure allows returning **custom errors** of any type, which may 149 | occur in the user program *after* reading from the background reader or 150 | *before* writing to the background writer. With the `thread_io` writer, there 151 | is the additional required trait bound `From` due to the way the 152 | writer works. 153 | 154 | Both with reading and writing, custom user errors are prioritized over eventual 155 | `io::Error`s. 156 | For example, it is possible that while parsing a file, a syntax error occurs, 157 | which the programmer returns from the `func` closure. Around the same time, 158 | `io::Error` may occur as well, e.g. because the GZIP file is truncated. If this 159 | error is still in the queue waiting to be reported as the syntax error happens, 160 | ultimately the syntax error will be returned and the `io::Error` discarded. 161 | 162 | After the func closure ends (with or without an error), a signal is placed in 163 | a queue telling the background thread to stop processing. However, `queuelen` 164 | reads or writes will be done before processing ultimately stops. 165 | 166 | More details on error handling are found in the documentation of the 167 | [read](https://docs.rs/thread_io/latest/thread_io/read) and 168 | [write](https://docs.rs/thread_io/latest/thread_io/write) modules. 169 | 170 | ## Crossbeam channels 171 | 172 | It is possible to use 173 | [the channel implementation from the crossbeam crate](https://docs.rs/crossbeam/latest/crossbeam/channel/index.html) 174 | by specifying the `crossbeam_channel` feature. The few tests I have done didn't 175 | show any performance gain over using the channels from the standard library. 176 | 177 | ## Similar projects 178 | 179 | [**fastq-rs**](https://github.com/aseyboldt/fastq-rs) provides a very similar 180 | functionality as `thread_io::read::reader` in its 181 | [`thread_reader`](https://docs.rs/fastq/latest/fastq/fn.thread_reader.html) 182 | module. 183 | -------------------------------------------------------------------------------- /benches/reader.rs: -------------------------------------------------------------------------------- 1 | #![feature(test)] 2 | #![allow(non_snake_case, unused_variables)] 3 | 4 | extern crate test; 5 | extern crate thread_io; 6 | 7 | use std::io::{Cursor, ErrorKind, Read}; 8 | 9 | fn get_data(len: usize) -> Vec { 10 | b"The quick brown fox jumps over the lazy dog" 11 | .iter() 12 | .cycle() 13 | .take(len) 14 | .cloned() 15 | .collect() 16 | } 17 | 18 | static DATA_LEN: usize = 1 << 26; 19 | 20 | macro_rules! bench { 21 | ($name:ident, $bufsize:expr, $queuelen:expr) => { 22 | #[bench] 23 | fn $name(b: &mut test::Bencher) { 24 | let data = get_data(DATA_LEN); 25 | b.bytes = data.len() as u64; 26 | b.iter(move || { 27 | let mut buf = vec![0; $bufsize]; 28 | thread_io::read::reader($bufsize, $queuelen, Cursor::new(&data), |rdr| { 29 | loop { 30 | if let Err(e) = rdr.read_exact(&mut buf) { 31 | if e.kind() == ErrorKind::UnexpectedEof { 32 | break; 33 | } else { 34 | return Err(e); 35 | } 36 | } 37 | } 38 | Ok(()) 39 | }) 40 | .unwrap(); 41 | }); 42 | } 43 | }; 44 | } 45 | 46 | macro_rules! bench_native { 47 | ($name:ident, $bufsize:expr) => { 48 | #[bench] 49 | fn $name(b: &mut test::Bencher) { 50 | let data = get_data(DATA_LEN); 51 | b.bytes = data.len() as u64; 52 | b.iter(move || { 53 | let mut buf = vec![0; $bufsize]; 54 | let mut rdr = Cursor::new(&data); 55 | loop { 56 | if let Err(e) = rdr.read_exact(&mut buf) { 57 | if e.kind() == ErrorKind::UnexpectedEof { 58 | break; 59 | } else { 60 | panic!("{}", e); 61 | } 62 | } 63 | } 64 | }); 65 | } 66 | }; 67 | } 68 | 69 | bench!(read_thread_32k_2, 1 << 15, 2); 70 | bench!(read_thread_64k_2, 1 << 16, 2); 71 | bench!(read_thread_128k_2, 1 << 17, 2); 72 | bench!(read_thread_256k_2, 1 << 18, 2); 73 | bench!(read_thread_512k_2, 1 << 19, 2); 74 | bench!(read_thread_512k_3, 1 << 19, 3); 75 | bench!(read_thread_512k_4, 1 << 19, 4); 76 | bench!(read_thread_512k_5, 1 << 19, 5); 77 | bench!(read_thread_1m_2, 1 << 20, 2); 78 | bench!(read_thread_2m_2, 1 << 21, 2); 79 | 80 | bench_native!(read_native_8k, 1 << 13); 81 | bench_native!(read_native_64k, 1 << 16); 82 | bench_native!(read_native_256k, 1 << 18); 83 | bench_native!(read_native_512k, 1 << 19); 84 | bench_native!(read_native_1m, 1 << 20); 85 | -------------------------------------------------------------------------------- /benches/writer.rs: -------------------------------------------------------------------------------- 1 | #![feature(test)] 2 | #![allow(non_snake_case, unused_variables)] 3 | 4 | extern crate test; 5 | extern crate thread_io; 6 | 7 | use std::io::{sink, Write}; 8 | 9 | fn get_data(len: usize) -> Vec { 10 | b"The quick brown fox jumps over the lazy dog" 11 | .iter() 12 | .cycle() 13 | .take(len) 14 | .cloned() 15 | .collect() 16 | } 17 | 18 | static DATA_LEN: usize = 1 << 26; 19 | static CHUNK_SIZE: usize = 1024; 20 | 21 | macro_rules! bench { 22 | ($name:ident, $bufsize:expr, $queuelen:expr) => { 23 | #[bench] 24 | fn $name(b: &mut test::Bencher) { 25 | let data = get_data(DATA_LEN); 26 | b.bytes = data.len() as u64; 27 | let mut out = sink(); 28 | b.iter(move || { 29 | thread_io::write::writer($bufsize, $queuelen, &mut out, |w| { 30 | for chunk in data.chunks(CHUNK_SIZE) { 31 | w.write_all(chunk)?; 32 | } 33 | Ok::<(), ::std::io::Error>(()) 34 | }) 35 | .unwrap(); 36 | }); 37 | } 38 | }; 39 | } 40 | 41 | macro_rules! bench_native { 42 | ($name:ident, $bufsize:expr) => { 43 | #[bench] 44 | fn $name(b: &mut test::Bencher) { 45 | let data = get_data(DATA_LEN); 46 | b.bytes = data.len() as u64; 47 | let mut out = sink(); 48 | b.iter(move || { 49 | for chunk in data.chunks(CHUNK_SIZE) { 50 | out.write_all(chunk).expect("Write error"); 51 | } 52 | }); 53 | } 54 | }; 55 | } 56 | 57 | bench!(write_thread_68k_3, 1024 * 68, 3); 58 | bench!(write_thread_256k_3, 1 << 18, 3); 59 | bench!(write_thread_512k_2, 1 << 19, 2); 60 | bench!(write_thread_512k_3, 1 << 19, 3); 61 | bench!(write_thread_512k_4, 1 << 19, 4); 62 | bench!(write_thread_512k_5, 1 << 19, 5); 63 | bench!(write_thread_1m_3, 1 << 20, 3); 64 | bench!(write_thread_2m_3, 1 << 21, 3); 65 | bench!(write_thread_4m_3, 1 << 22, 3); 66 | 67 | bench_native!(write_native_256k, 1 << 18); 68 | bench_native!(write_native_512k, 1 << 19); 69 | bench_native!(write_native_1m, 1 << 20); 70 | -------------------------------------------------------------------------------- /fuzz/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target 3 | corpus 4 | artifacts 5 | -------------------------------------------------------------------------------- /fuzz/Cargo.toml: -------------------------------------------------------------------------------- 1 | 2 | [package] 3 | name = "thread_io-fuzz" 4 | version = "0.0.2" 5 | authors = ["markschl "] 6 | publish = false 7 | edition = "2018" 8 | metadata = { cargo-fuzz = true } 9 | 10 | [dependencies] 11 | thread_io = { path = ".." } 12 | libfuzzer-sys = { version = "0.3", features = ["arbitrary-derive"] } 13 | arbitrary = { version = "0.4", features = ["derive"] } 14 | 15 | [profile.release] 16 | overflow-checks = true 17 | debug-assertions = true 18 | 19 | # Prevent this from interfering with workspaces 20 | [workspace] 21 | members = ["."] 22 | 23 | [[bin]] 24 | name = "read" 25 | path = "fuzz_targets/read.rs" 26 | 27 | [[bin]] 28 | name = "write" 29 | path = "fuzz_targets/write.rs" 30 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/read.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use std::io::{self, Read}; 4 | use std::cmp::{min, max}; 5 | use libfuzzer_sys::fuzz_target; 6 | use libfuzzer_sys::arbitrary::Arbitrary; 7 | 8 | #[derive(Arbitrary, Debug)] 9 | pub struct Config { 10 | channel_bufsize: u16, 11 | queuelen: u8, 12 | out_bufsize: u16, 13 | data_len: u16, 14 | // The list of reads to be done: 15 | // - 0 means ErrorKind::Interrupted 16 | // - > 0 means n bytes will be read into target buffer 17 | chunk_sizes: Vec, 18 | } 19 | 20 | // runs the thread_io::read::reader using different buffer sizes and queue lengths 21 | // using a mock reader (as in tests::read, but more variables are changed randomly) to ensure that 22 | // the returned data will always be the same 23 | fuzz_target!(|cfg: Config| { 24 | //println!("{:?}", cfg); 25 | let mut cfg = cfg; 26 | cfg.channel_bufsize = max(1, cfg.channel_bufsize); 27 | cfg.queuelen = max(1, cfg.queuelen); 28 | cfg.out_bufsize = max(1, cfg.out_bufsize); 29 | 30 | if cfg.chunk_sizes.len() == 0 { 31 | return; 32 | } 33 | 34 | let data: Vec = (0..cfg.data_len).map(|_| 0).collect(); 35 | 36 | // test the mock reader itself 37 | let c = cfg.chunk_sizes.iter().map(|&c| c as usize); 38 | let rdr = Reader::new(&data, c); 39 | assert_eq!(read_chunks(rdr, cfg.out_bufsize as usize).unwrap(), data); 40 | 41 | // test threaded reader 42 | let c = cfg.chunk_sizes.iter().map(|&c| c as usize); 43 | let rdr = Reader::new(&data, c); 44 | let out = thread_io::read::reader( 45 | cfg.channel_bufsize as usize, 46 | cfg.queuelen as usize, 47 | rdr, 48 | |r| read_chunks(r, cfg.out_bufsize as usize) 49 | ).unwrap(); 50 | 51 | assert_eq!(out, data); 52 | 53 | }); 54 | 55 | 56 | struct Reader<'a, C: Iterator> { 57 | data: &'a [u8], 58 | chunk_sizes: C, 59 | } 60 | 61 | impl<'a, C: Iterator> Reader<'a, C> { 62 | fn new(data: &'a [u8], chunk_sizes: C) -> Self { 63 | Reader { 64 | data: data, 65 | chunk_sizes: chunk_sizes, 66 | } 67 | } 68 | } 69 | 70 | impl<'a, C: Iterator> Read for Reader<'a, C> { 71 | fn read(&mut self, buf: &mut [u8]) -> io::Result { 72 | let chunk_size = self.chunk_sizes.next(); 73 | if chunk_size == Some(0) { 74 | return Err(io::Error::new(io::ErrorKind::Interrupted, "interrupted")); 75 | } 76 | let chunk_size = chunk_size.unwrap_or_else(|| max(self.data.len(), 0)); 77 | let amt = min(self.data.len(), min(buf.len(), chunk_size)); 78 | let (a, b) = self.data.split_at(amt); 79 | buf[..amt].copy_from_slice(a); 80 | self.data = b; 81 | Ok(amt) 82 | } 83 | } 84 | 85 | // Repeatedly reads from rdr into a buffer of size `chunksize`. The buffer contents are 86 | // appended to the output until EOF occurs, and the output is returned. 87 | fn read_chunks(mut rdr: R, out_buf_size: usize) -> io::Result> { 88 | let mut out = vec![]; 89 | let mut buf = vec![0; out_buf_size]; 90 | loop { 91 | let res = rdr.read(buf.as_mut_slice()); 92 | let n = match res { 93 | Ok(n) => n, 94 | Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, 95 | Err(e) => return Err(e), 96 | }; 97 | out.extend_from_slice(&buf[..n]); 98 | if n == 0 { 99 | break; 100 | } 101 | } 102 | Ok(out) 103 | } 104 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/write.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use std::io::{self, Write}; 4 | use std::cmp::{min, max}; 5 | use libfuzzer_sys::fuzz_target; 6 | use libfuzzer_sys::arbitrary::Arbitrary; 7 | 8 | #[derive(Arbitrary, Debug)] 9 | pub struct Config { 10 | channel_bufsize: u16, 11 | queuelen: u8, 12 | writer_bufsize: u16, 13 | data: Vec, 14 | } 15 | 16 | // runs the thread_io::read::reader using different buffer sizes and queue lengths 17 | // using a mock reader (as in tests::read, but more variables are changed randomly) to ensure that 18 | // the returned cfg.data will always be the same 19 | fuzz_target!(|cfg: Config| { 20 | //println!("{:?}", cfg); 21 | let mut cfg = cfg; 22 | cfg.channel_bufsize = max(1, cfg.channel_bufsize); 23 | cfg.queuelen = max(1, cfg.queuelen); 24 | cfg.writer_bufsize = max(1, cfg.writer_bufsize); 25 | 26 | // Test writer 27 | let w = thread_io::write::writer_finish( 28 | cfg.channel_bufsize as usize, 29 | cfg.queuelen as usize, 30 | Writer::new(false, false, cfg.writer_bufsize as usize), 31 | |w| w.write(&cfg.data), 32 | |w| w 33 | ).unwrap().1; 34 | assert_eq!(w.data(), cfg.data.as_slice()); 35 | 36 | // Test case in which write fails 37 | let w = Writer::new(true, false, cfg.writer_bufsize as usize); 38 | let res = thread_io::write::writer( 39 | cfg.channel_bufsize as usize, 40 | cfg.queuelen as usize, 41 | w, 42 | |w| w.write(&cfg.data) 43 | ); 44 | if let Err(e) = res { 45 | assert_eq!(&format!("{}", e), "write err"); 46 | } else { 47 | panic!("write should fail"); 48 | } 49 | 50 | // Test case in which flushing fails 51 | let w = Writer::new(false, true, cfg.writer_bufsize as usize); 52 | let res = thread_io::write::writer( 53 | cfg.channel_bufsize as usize, 54 | cfg.queuelen as usize, 55 | w, 56 | |w| w.flush() 57 | ); 58 | if let Err(e) = res { 59 | assert_eq!(&format!("{}", e), "flush err"); 60 | } else { 61 | panic!("flush should fail"); 62 | } 63 | }); 64 | 65 | /// a writer that only writes its cfg.data to `Writer::cfg.data` upon `flush()` 66 | #[derive(Clone)] 67 | struct Writer { 68 | cache: Vec, 69 | data: Vec, 70 | write_fails: bool, 71 | flush_fails: bool, 72 | bufsize: usize, 73 | } 74 | 75 | impl Writer { 76 | fn new(write_fails: bool, flush_fails: bool, bufsize: usize) -> Writer { 77 | Writer { 78 | cache: vec![], 79 | data: vec![], 80 | write_fails: write_fails, 81 | flush_fails: flush_fails, 82 | bufsize: bufsize, 83 | } 84 | } 85 | 86 | fn data(&self) -> &[u8] { 87 | &self.data 88 | } 89 | } 90 | 91 | impl Write for Writer { 92 | fn write(&mut self, buffer: &[u8]) -> io::Result { 93 | if self.write_fails { 94 | return Err(io::Error::new(io::ErrorKind::Other, "write err")) 95 | } 96 | self.cache.write(&buffer[..min(buffer.len(), self.bufsize)]) 97 | } 98 | 99 | fn flush(&mut self) -> io::Result<()> { 100 | if self.flush_fails { 101 | Err(io::Error::new(io::ErrorKind::Other, "flush err")) 102 | } else { 103 | self.data.extend_from_slice(&self.cache); 104 | self.cache.clear(); 105 | Ok(()) 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod read; 2 | pub mod write; 3 | 4 | fn unwrap_or_resume_unwind(value: Result>) -> V { 5 | match value { 6 | Ok(value) => value, 7 | Err(error) => std::panic::resume_unwind(error), 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/read.rs: -------------------------------------------------------------------------------- 1 | //! This module contains functions for reading in a background thread. 2 | //! 3 | //! * The simplest to use is the [`reader`](fn.reader.html) function. It accepts 4 | //! any `io::Read` instance that implements `Send`. 5 | //! * The [`reader_init`](fn.reader_init.html) function handles cases where the 6 | //! wrapped reader cannot be sent safely across the thread boundary by 7 | //! providing a closure for initializing the reader in the background thread. 8 | //! 9 | //! # Error handling 10 | //! 11 | //! * `io::Error`s occuring during reading in the background are returned by 12 | //! the `read` method of the [reader in the main thread](struct.Reader.html) 13 | //! as expected, but with a delay of at east one call. 14 | //! * Reading errors cause the background reader to stop, *except* for errors 15 | //! of kind `io::ErrorKind::Interrupted`. In this case reading continues in 16 | //! background, allowing the user to resume reading after the error occurred. 17 | //! * The `func` closure running in the main thread allows returning errors of 18 | //! any type. If a reading error happens around the same time in the 19 | //! background thread and does not reach the main thread due to the reporting 20 | //! delay, it will be discarded and the error from `func` returned instead. 21 | //! * *panics* in the background reader are correctly forwarded to the main 22 | //! thread, but are also given lower priority if an error is returned from 23 | //! `func`. 24 | 25 | #[cfg(feature = "crossbeam_channel")] 26 | use crossbeam_channel::{unbounded as channel, Receiver, Sender}; 27 | use std::io::{self, Cursor, Read}; 28 | #[cfg(not(feature = "crossbeam_channel"))] 29 | use std::sync::mpsc::{channel, Receiver, Sender}; 30 | 31 | #[derive(Debug)] 32 | struct Buffer { 33 | data: Box<[u8]>, 34 | end: usize, 35 | // io::ErrorKind::Interrupted 36 | interrupted: bool, 37 | } 38 | 39 | impl Buffer { 40 | #[inline] 41 | fn new(size: usize) -> Buffer { 42 | assert!(size > 0); 43 | Buffer { 44 | data: vec![0; size].into_boxed_slice(), 45 | end: 0, 46 | interrupted: false, 47 | } 48 | } 49 | 50 | /// Fill the whole buffer, using multiple reads if necessary. This means, that upon EOF, 51 | /// read may be called again once before n = 0 is returned in the main thread. 52 | #[inline] 53 | fn refill(&mut self, mut reader: R) -> io::Result<()> { 54 | let mut n_read = 0; 55 | let mut buf = &mut *self.data; 56 | self.interrupted = false; 57 | 58 | while !buf.is_empty() { 59 | match reader.read(buf) { 60 | Ok(n) => { 61 | if n == 0 { 62 | // EOF 63 | break; 64 | } 65 | let tmp = buf; 66 | buf = &mut tmp[n..]; 67 | n_read += n; 68 | } 69 | Err(ref e) if e.kind() == io::ErrorKind::Interrupted => { 70 | self.interrupted = true; 71 | break; 72 | } 73 | Err(e) => return Err(e), 74 | }; 75 | } 76 | 77 | self.end = n_read; 78 | 79 | Ok(()) 80 | } 81 | } 82 | 83 | /// The reader in the main thread 84 | #[derive(Debug)] 85 | pub struct Reader { 86 | full_recv: Receiver>, 87 | empty_send: Sender>, 88 | buffer: Option, 89 | pos: usize, 90 | } 91 | 92 | impl Reader { 93 | #[inline] 94 | fn new( 95 | full_recv: Receiver>, 96 | empty_send: Sender>, 97 | bufsize: usize, 98 | queuelen: usize, 99 | ) -> Self { 100 | for _ in 0..queuelen { 101 | empty_send.send(Some(Buffer::new(bufsize))).ok(); 102 | } 103 | 104 | Reader { 105 | full_recv, 106 | empty_send, 107 | buffer: None, 108 | pos: 0, 109 | } 110 | } 111 | 112 | #[inline] 113 | fn done(&self) { 114 | self.empty_send.send(None).ok(); 115 | } 116 | 117 | // assumes that self.buffer is not None. Returns a tuple of the read result 118 | // and a flag indicating if a new buffer should be received (cannot be done 119 | // here due to borrow checker) 120 | #[inline] 121 | fn _read(&mut self, buf: &mut [u8]) -> (io::Result, bool) { 122 | let source = self.buffer.as_mut().unwrap(); 123 | 124 | if source.interrupted && self.pos == source.end { 125 | return (Err(io::Error::from(io::ErrorKind::Interrupted)), true); 126 | } 127 | 128 | let n = Cursor::new(&source.data[self.pos..source.end]) 129 | .read(buf) 130 | .unwrap(); 131 | self.pos += n; 132 | 133 | (Ok(n), self.pos == source.end && !source.interrupted) 134 | } 135 | } 136 | 137 | impl io::Read for Reader { 138 | fn read(&mut self, buf: &mut [u8]) -> io::Result { 139 | if self.buffer.is_none() { 140 | self.buffer = Some(self.full_recv.recv().ok().unwrap()?); 141 | } 142 | 143 | let (rv, recv_next) = self._read(buf); 144 | 145 | if recv_next { 146 | self.empty_send.send(self.buffer.take()).ok(); 147 | self.pos = 0; 148 | } 149 | 150 | rv 151 | } 152 | } 153 | 154 | #[derive(Debug)] 155 | struct BackgroundReader { 156 | empty_recv: Receiver>, 157 | full_send: Sender>, 158 | } 159 | 160 | impl BackgroundReader { 161 | #[inline] 162 | fn new(empty_recv: Receiver>, full_send: Sender>) -> Self { 163 | BackgroundReader { 164 | empty_recv, 165 | full_send, 166 | } 167 | } 168 | 169 | #[inline] 170 | fn serve(&mut self, mut reader: R) { 171 | while let Ok(Some(mut buffer)) = self.empty_recv.recv() { 172 | match buffer.refill(&mut reader) { 173 | Ok(_) => { 174 | self.full_send.send(Ok(buffer)).ok(); 175 | } 176 | Err(e) => { 177 | self.full_send.send(Err(e)).ok(); 178 | break; 179 | } 180 | } 181 | } 182 | } 183 | } 184 | 185 | /// Sends `reader` to a background thread and provides a reader in the main 186 | /// thread, which obtains data from the background reader. 187 | /// 188 | /// The background reader fills buffers of a given size (`bufsize`) and submits 189 | /// them to the main thread through a channel. The queue length of the channel 190 | /// can be configured using the `queuelen` parameter (must be ≥ 1). As a 191 | /// consequence, errors will not be returned immediately, but after some delay. 192 | /// The reader in the background thread will stop if an error occurs, except for 193 | /// errors of kind `io::ErrorKind::Interrupted`. In this case, reading continues 194 | /// in the background, but the error is still returned. 195 | /// 196 | /// # Example: 197 | /// 198 | /// ``` 199 | /// use thread_io::read::reader; 200 | /// use std::io::Read; 201 | /// 202 | /// let text = b"The quick brown fox jumps over the lazy dog"; 203 | /// let mut target = vec![]; 204 | /// 205 | /// reader(16, 2, &text[..], |rdr| { 206 | /// rdr.read_to_end(&mut target) 207 | /// }).expect("read failed"); 208 | /// 209 | /// assert_eq!(target.as_slice(), &text[..]); 210 | /// ``` 211 | pub fn reader(bufsize: usize, queuelen: usize, reader: R, func: F) -> Result 212 | where 213 | F: FnOnce(&mut Reader) -> Result, 214 | R: io::Read + Send, 215 | E: Send, 216 | { 217 | reader_init(bufsize, queuelen, || Ok(reader), func) 218 | } 219 | 220 | /// Like [`reader()`](fn.reader.html), but the wrapped reader is initialized 221 | /// in a closure (`init_reader`) in the background thread. This allows using 222 | /// readers that don't implement `Send` 223 | /// 224 | /// # Example: 225 | /// 226 | /// ``` 227 | /// use thread_io::read::reader_init; 228 | /// use std::io::{self, Read}; 229 | /// 230 | /// let mut input = io::stdin(); 231 | /// 232 | /// // StdinLock does not implement Send 233 | /// reader_init(16, 2, || Ok(input.lock()), |rdr| { 234 | /// let mut s = String::new(); 235 | /// let _ = rdr.read_to_string(&mut s).expect("read error"); 236 | /// // ... 237 | /// Ok::<_, io::Error>(()) 238 | /// }).expect("read failed"); 239 | /// ``` 240 | pub fn reader_init( 241 | bufsize: usize, 242 | queuelen: usize, 243 | init_reader: I, 244 | func: F, 245 | ) -> Result 246 | where 247 | I: Send + FnOnce() -> Result, 248 | F: FnOnce(&mut Reader) -> Result, 249 | R: io::Read, 250 | E: Send, 251 | { 252 | assert!(queuelen >= 1); 253 | assert!(bufsize > 0); 254 | 255 | let (full_send, full_recv) = channel(); 256 | let (empty_send, empty_recv) = channel(); 257 | 258 | let mut reader = Reader::new(full_recv, empty_send, bufsize, queuelen); 259 | let mut background_reader = BackgroundReader::new(empty_recv, full_send); 260 | 261 | crossbeam_utils::thread::scope(|scope| { 262 | let handle = scope.spawn(move |_| { 263 | let mut inner = init_reader()?; 264 | background_reader.serve(&mut inner); 265 | Ok::<_, E>(()) 266 | }); 267 | 268 | let out = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| func(&mut reader))); 269 | 270 | reader.done(); 271 | 272 | // We deliberately ensure that errors from the background reading thread 273 | // are given priority. This does NOT include errors returned from the 274 | // actual I/O which are returned via the channels to the reader. 275 | // It includes errors returned by init_reader() and panics that occured 276 | // while reading. 277 | // Either of those cases will have cause the reader to be in an 278 | // unworkable state. Consequently, we want to surface the error that 279 | // caused this. 280 | crate::unwrap_or_resume_unwind(handle.join())?; 281 | crate::unwrap_or_resume_unwind(out) 282 | }) 283 | .unwrap() 284 | } 285 | -------------------------------------------------------------------------------- /src/write.rs: -------------------------------------------------------------------------------- 1 | //! This module contains functions for writing in a background thread. 2 | //! 3 | //! * The simplest to use is the [`writer`](fn.writer.html) function. It accepts 4 | //! any `io::Write` instance that implements `Send`. 5 | //! * The [`writer_init`](fn.writer_init.html) function handles cases where the 6 | //! wrapped writer cannot be sent safely across the thread boundary by providing 7 | //! a closure for initializing the writer in the background thread. 8 | //! * [`writer_finish`](fn.writer_finish.html) and 9 | //! [`writer_init_finish`](fn.writer_init_finish.html) provide an additional 10 | //! `finish` closure, which allows for executing final code after all writing is 11 | //! done, or also for returning the wrapped writer to the main thread. Usually, 12 | //! `write` calls happen in the background even *after* `func` returned, 13 | //! finalizing code should thus *always* go after the `writer...()` function 14 | //! call or be placed in the `finish` closure (useful for non-`Send` writers). 15 | //! 16 | //! # Error handling 17 | //! 18 | //! * `io::Error`s occurring during writing in the background are returned by 19 | //! the `write` method of the [writer in the main thread](struct.Writer.html) 20 | //! as expected, but with a delay of at east one call. 21 | //! * The `func` closure running in the main thread allows returning errors of 22 | //! any type. However, in contrast to the functions in the `read` module, the 23 | //! error needs to implement `From` because additional `write` 24 | //! calls can happen in the background *after* `func` is already finished, 25 | //! and eventual errors have to be returned as well. 26 | //! * If an error is returned from `func` in the main thread, and around the 27 | //! same time a writing error happens in the background thread, which does not 28 | //! reach the main thread due to the reporting delay, the latter will be 29 | //! discarded and the error from `func` returned instead. 30 | //! Due to this, there is no guarantee on how much data is still written 31 | //! after the custom error occurs. 32 | //! * *panics* in the background writer are correctly forwarded to the main 33 | //! thread, but are also given lower priority if an error is returned from 34 | //! `func`. 35 | //! 36 | //! # Flushing 37 | //! 38 | //! Trying to mimick the expected functionality of `io::Write` as closely as 39 | //! possible, the [writer](struct.Writer.html) provided in the `func` 40 | //! closure in the main thread also forwards `flush()` calls to the background 41 | //! writer. `write` and `flush` calls are guaranteed to be in the same order as 42 | //! in the main thread. A `flush` call will always trigger the writer in the 43 | //! main thread to send any buffered data to the background *before* the actual 44 | //! flushing is queued. 45 | //! 46 | //! In addition, `flush()` is **always** called after all writing is done. 47 | //! It is assumed that usually no more data will be written after this, and a 48 | //! call to `flush()` ensures that that possible flushing errors don't go 49 | //! unnoticed like they would if the file is closed automatically when dropped. 50 | //! With `File`, it is still possible that when going out of scope, unreported 51 | //! errors occur when writing OS-internal metadata. To address this, 52 | //! `File::sync_all` or `File::sync_data` can be called after the `writer` 53 | //! call finished, or in the case of non-`Send` writers in the `finish` closure 54 | //! of `writer_finish` / `writer_init_finish`. 55 | 56 | use std::io::{self, Write}; 57 | use std::mem::replace; 58 | 59 | #[cfg(feature = "crossbeam_channel")] 60 | use crossbeam_channel::{unbounded as channel, Receiver, Sender}; 61 | #[cfg(not(feature = "crossbeam_channel"))] 62 | use std::sync::mpsc::{channel, Receiver, Sender}; 63 | 64 | #[derive(Debug)] 65 | enum Message { 66 | Buffer(io::Cursor>), 67 | Flush, 68 | Done, 69 | } 70 | 71 | /// The writer in the main thread 72 | #[derive(Debug)] 73 | pub struct Writer { 74 | empty_recv: Receiver>>, 75 | full_send: Sender, 76 | buffer: io::Cursor>, 77 | } 78 | 79 | impl Writer { 80 | #[inline] 81 | fn new( 82 | empty_recv: Receiver>>, 83 | full_send: Sender, 84 | bufsize: usize, 85 | ) -> Self { 86 | let buffer = io::Cursor::new(vec![0; bufsize].into_boxed_slice()); 87 | Writer { 88 | empty_recv, 89 | full_send, 90 | buffer, 91 | } 92 | } 93 | 94 | /// Sends the current buffer to the background thread for writing. Returns 95 | /// `false` if the background thread has terminated, which could have the 96 | /// following reasons: 97 | /// 1) There was a write/flush error or panic. These have to be handled 98 | /// accordingly, here we do nothing more. 99 | /// 2) Message::Done has already been sent, send_to_background() was called 100 | /// after done()). This should not happen in the current code. 101 | /// 3) Writer initialization failed. 102 | #[inline] 103 | fn send_to_background(&mut self) -> io::Result { 104 | if let Ok(empty) = self.empty_recv.recv() { 105 | let full = replace(&mut self.buffer, io::Cursor::new(empty?)); 106 | if self.full_send.send(Message::Buffer(full)).is_ok() { 107 | return Ok(true); 108 | } 109 | } 110 | Ok(false) 111 | } 112 | 113 | #[inline] 114 | fn done(&mut self) -> io::Result<()> { 115 | // send last buffer 116 | self.send_to_background()?; 117 | // Tell the background writer to finish up if it didn't already 118 | self.full_send.send(Message::Done).ok(); 119 | Ok(()) 120 | } 121 | 122 | // Checks all items in the empty_recv queue for a possible error, 123 | // throwing away all buffers. This should thus only be called at the 124 | // very end. 125 | #[inline] 126 | fn fetch_error(&self) -> io::Result<()> { 127 | for res in &self.empty_recv { 128 | res?; 129 | } 130 | Ok(()) 131 | } 132 | } 133 | 134 | impl Write for Writer { 135 | fn write(&mut self, buffer: &[u8]) -> io::Result { 136 | let mut written = 0; 137 | while written < buffer.len() { 138 | let n = self.buffer.write(&buffer[written..])?; 139 | written += n; 140 | if n == 0 && !self.send_to_background()? { 141 | break; 142 | } 143 | } 144 | Ok(written) 145 | } 146 | 147 | fn flush(&mut self) -> io::Result<()> { 148 | self.send_to_background()?; 149 | self.full_send.send(Message::Flush).ok(); 150 | Ok(()) 151 | } 152 | } 153 | 154 | #[derive(Debug)] 155 | struct BackgroundWriter { 156 | full_recv: Receiver, 157 | empty_send: Sender>>, 158 | } 159 | 160 | impl BackgroundWriter { 161 | #[inline] 162 | fn new( 163 | full_recv: Receiver, 164 | empty_send: Sender>>, 165 | bufsize: usize, 166 | queuelen: usize, 167 | ) -> Self { 168 | for _ in 0..queuelen { 169 | empty_send 170 | .send(Ok(vec![0; bufsize].into_boxed_slice())) 171 | .ok(); 172 | } 173 | BackgroundWriter { 174 | full_recv, 175 | empty_send, 176 | } 177 | } 178 | 179 | #[inline] 180 | fn listen(&mut self, mut writer: W) -> bool { 181 | while let Ok(msg) = self.full_recv.recv() { 182 | match msg { 183 | Message::Buffer(buf) => { 184 | let pos = buf.position() as usize; 185 | let buffer = buf.into_inner(); 186 | let res = writer.write_all(&buffer[..pos]); 187 | let is_err = res.is_err(); 188 | self.empty_send.send(res.map(|_| buffer)).ok(); 189 | if is_err { 190 | return false; 191 | } 192 | } 193 | Message::Flush => { 194 | if let Err(e) = writer.flush() { 195 | self.empty_send.send(Err(e)).ok(); 196 | return false; 197 | } 198 | } 199 | Message::Done => break, 200 | } 201 | } 202 | true 203 | } 204 | } 205 | 206 | /// Sends `writer` to a background thread and provides another writer in the 207 | /// main thread, which then submits its data to the background writer. 208 | /// 209 | /// The writer in the closure (`func`) fills buffers of a given size (`bufsize`) 210 | /// and submits them to the background writer through a channel. The queue 211 | /// length of the channel can be configured using the `queuelen` parameter (must 212 | /// be ≥ 1). As a consequence, errors will not be returned immediately, but 213 | /// after some delay, or after writing is finished and the closure ends. 214 | /// 215 | /// Also note that the last `write()` in the background can happen **after** the 216 | /// closure has ended. Finalizing actions should be done in the `finish` closure 217 | /// supplied to [`writer_finish`](fn.writer_finish.html) or 218 | /// [`writer_init_finish`](fn.writer_init_finish.html). 219 | /// 220 | /// # Example: 221 | /// 222 | /// ``` 223 | /// use thread_io::write::writer; 224 | /// use std::io::Write; 225 | /// 226 | /// let text = b"The quick brown fox jumps over the lazy dog"; 227 | /// let mut buf = vec![0; text.len()]; 228 | /// 229 | /// writer(16, 2, &mut buf[..], |writer| { 230 | /// writer.write_all(&text[..]) 231 | /// }).expect("write failed"); 232 | /// 233 | /// assert_eq!(&buf[..], &text[..]); 234 | /// ``` 235 | pub fn writer(bufsize: usize, queuelen: usize, writer: W, func: F) -> Result 236 | where 237 | F: FnOnce(&mut Writer) -> Result, 238 | W: Write + Send, 239 | E: Send + From, 240 | { 241 | writer_init(bufsize, queuelen, || Ok(writer), func) 242 | } 243 | 244 | /// Like [`writer`](fn.writer.html), but the wrapped writer is initialized in a 245 | /// closure (`init_writer()`) in the background thread. This allows using 246 | /// writers that don't implement `Send`. 247 | /// 248 | /// # Example: 249 | /// 250 | /// ``` 251 | /// use thread_io::write::writer_init; 252 | /// use std::io::{self, Write}; 253 | /// 254 | /// let text = b"The quick brown fox jumps over the lazy dog"; 255 | /// let mut output = io::stdout(); 256 | /// 257 | /// // StdoutLock does not implement Send 258 | /// writer_init(16, 2, || Ok(output.lock()), |writer| { 259 | /// writer.write_all(&text[..]) 260 | /// }).expect("write failed"); 261 | /// ``` 262 | pub fn writer_init( 263 | bufsize: usize, 264 | queuelen: usize, 265 | init_writer: I, 266 | func: F, 267 | ) -> Result 268 | where 269 | I: Send + FnOnce() -> Result, 270 | F: FnOnce(&mut Writer) -> Result, 271 | W: Write, 272 | E: Send + From, 273 | { 274 | writer_init_finish(bufsize, queuelen, init_writer, func, |_| ()).map(|(o, _)| o) 275 | } 276 | 277 | /// Like `writer`, but accepts another closure taking the wrapped writer by 278 | /// value before it goes out of scope (and there is no error). 279 | /// Useful for performing finalizing actions or returning the wrapped writer to 280 | /// the main thread (if it implements `Send`). 281 | /// 282 | /// The output values of `func` and the `finish` closure are returned in a 283 | /// tuple. 284 | /// 285 | /// # Example: 286 | /// 287 | /// ``` 288 | /// use thread_io::write::writer_finish; 289 | /// use std::io::Write; 290 | /// 291 | /// let text = b"The quick brown fox jumps over the lazy dog"; 292 | /// let mut output = vec![]; 293 | /// 294 | /// // `output` is moved to background thread 295 | /// let (_, output) = writer_finish(16, 2, output, 296 | /// |out| out.write_all(&text[..]), 297 | /// |out| out // output is returned to main thread 298 | /// ).expect("write failed"); 299 | /// 300 | /// assert_eq!(&output[..], &text[..]); 301 | /// ``` 302 | pub fn writer_finish( 303 | bufsize: usize, 304 | queuelen: usize, 305 | writer: W, 306 | func: F, 307 | finish: F2, 308 | ) -> Result<(O, O2), E> 309 | where 310 | F: FnOnce(&mut Writer) -> Result, 311 | W: Write + Send, 312 | F2: Send + FnOnce(W) -> O2, 313 | O2: Send, 314 | E: Send + From, 315 | { 316 | writer_init_finish(bufsize, queuelen, || Ok(writer), func, finish) 317 | } 318 | 319 | /// This method takes both an initializing closure (see 320 | /// [`writer_init`](fn.writer_init.html)), and a closure for finalizing or 321 | /// returning data back to the main thread (see 322 | /// [`writer_finish`](fn.writer_finish.html)). 323 | /// 324 | /// The output values of `func` and the `finish` closure are returned as a 325 | /// tuple. 326 | pub fn writer_init_finish( 327 | bufsize: usize, 328 | queuelen: usize, 329 | init_writer: I, 330 | func: F, 331 | finish: F2, 332 | ) -> Result<(O, O2), E> 333 | where 334 | I: Send + FnOnce() -> Result, 335 | F: FnOnce(&mut Writer) -> Result, 336 | W: Write, 337 | F2: Send + FnOnce(W) -> O2, 338 | O2: Send, 339 | E: Send + From, 340 | { 341 | assert!(queuelen >= 1); 342 | assert!(bufsize > 0); 343 | 344 | let (full_send, full_recv) = channel(); 345 | let (empty_send, empty_recv) = channel(); 346 | 347 | let mut writer = Writer::new(empty_recv, full_send, bufsize); 348 | let mut background_writer = BackgroundWriter::new(full_recv, empty_send, bufsize, queuelen); 349 | 350 | crossbeam_utils::thread::scope(|scope| { 351 | let handle = scope.spawn::<_, Result<_, E>>(move |_| { 352 | let mut inner = init_writer()?; 353 | if background_writer.listen(&mut inner) { 354 | // writing finished witout error 355 | return Ok(Some(finish(inner))); 356 | } 357 | Ok(None) 358 | }); 359 | 360 | let out = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { 361 | let out = func(&mut writer)?; 362 | // we always flush before returning 363 | writer.flush()?; 364 | Ok::<_, E>(out) 365 | })); 366 | 367 | let writer_result = writer.done(); 368 | 369 | let handle = handle.join(); 370 | 371 | // Prefer errors from the background thread. This doesn't include actual 372 | // I/O errors from the writing because those are sent via the channel to 373 | // the main thread. Instead, it returns errors from init_writer or 374 | // panics from the writing thread. If either of those happen, writing in 375 | // the main thread will fail but we want to return the underlying reason. 376 | let of = crate::unwrap_or_resume_unwind(handle)?; 377 | let out = crate::unwrap_or_resume_unwind(out)?; 378 | 379 | // Report write errors that happened after the main thread stopped writing. 380 | writer_result?; 381 | // Return errors that may have occurred when writing the last few chunks. 382 | writer.fetch_error()?; 383 | 384 | Ok((out, of.unwrap())) 385 | }) 386 | .unwrap() 387 | } 388 | -------------------------------------------------------------------------------- /tests/read.rs: -------------------------------------------------------------------------------- 1 | use std::cmp::min; 2 | use std::io::{self, Read}; 3 | use thread_io::read::*; 4 | 5 | struct Reader<'a> { 6 | data: &'a [u8], 7 | block_size: usize, 8 | // used to test what happens to errors that are 9 | // stuck in the queue 10 | fails_after: usize, 11 | interrups_after: usize, 12 | panic: bool, 13 | } 14 | 15 | impl<'a> Reader<'a> { 16 | fn new( 17 | data: &'a [u8], 18 | block_size: usize, 19 | fails_after: Option, 20 | interrups_after: Option, 21 | panic: bool, 22 | ) -> Reader { 23 | Reader { 24 | data, 25 | block_size, 26 | fails_after: fails_after.unwrap_or(usize::max_value()), 27 | interrups_after: interrups_after.unwrap_or(usize::max_value()), 28 | panic, 29 | } 30 | } 31 | } 32 | 33 | impl<'a> Read for Reader<'a> { 34 | fn read(&mut self, buf: &mut [u8]) -> io::Result { 35 | if self.fails_after == 0 { 36 | if self.panic { 37 | panic!("read err"); 38 | } else { 39 | return Err(io::Error::new(io::ErrorKind::Other, "read err")); 40 | } 41 | } 42 | if self.interrups_after == 0 { 43 | self.interrups_after = usize::max_value(); 44 | return Err(io::Error::new(io::ErrorKind::Interrupted, "interrupted")); 45 | } 46 | self.fails_after -= 1; 47 | self.interrups_after -= 1; 48 | let amt = min(self.data.len(), min(buf.len(), self.block_size)); 49 | let (a, b) = self.data.split_at(amt); 50 | buf[..amt].copy_from_slice(a); 51 | self.data = b; 52 | Ok(amt) 53 | } 54 | } 55 | 56 | // Repeatedly reads from rdr into a buffer of size `chunksize`. The buffer contents are 57 | // appended to the output until EOF occurs, and the output is returned. 58 | fn read_chunks(mut rdr: R, chunksize: usize, resume: bool) -> io::Result> { 59 | let mut out = vec![]; 60 | let mut buf = vec![0; chunksize]; 61 | loop { 62 | match rdr.read(buf.as_mut_slice()) { 63 | Ok(n) => { 64 | out.extend_from_slice(&buf[..n]); 65 | if n == 0 { 66 | break; 67 | } 68 | } 69 | Err(e) => { 70 | if resume && e.kind() == io::ErrorKind::Interrupted { 71 | continue; 72 | } 73 | return Err(e); 74 | } 75 | } 76 | } 77 | Ok(out) 78 | } 79 | 80 | #[test] 81 | fn read() { 82 | let text = b"The quick brown fox"; 83 | let n = text.len() + 3; 84 | 85 | for channel_bufsize in (1..n).step_by(2) { 86 | for rdr_block_size in (1..n).step_by(2) { 87 | for out_bufsize in (1..n).step_by(2) { 88 | for queuelen in (1..n).step_by(2) { 89 | // test the mock reader itself 90 | let rdr = Reader::new(text, rdr_block_size, None, None, false); 91 | assert_eq!( 92 | read_chunks(rdr, out_bufsize, false).unwrap().as_slice(), 93 | &text[..] 94 | ); 95 | 96 | // test threaded reader 97 | let rdr = Reader::new(text, rdr_block_size, None, None, false); 98 | let out = reader(channel_bufsize, queuelen, rdr, |r| { 99 | read_chunks(r, out_bufsize, false) 100 | }) 101 | .unwrap(); 102 | 103 | if out.as_slice() != &text[..] { 104 | panic!( 105 | "left != right at channel bufsize: {}, reader bufsize: {}, final reader bufsize {}, queue length: {}\nleft: {:?}\nright: {:?}", 106 | channel_bufsize, rdr_block_size, out_bufsize, queuelen, &out, &text[..] 107 | ); 108 | } 109 | } 110 | } 111 | } 112 | } 113 | } 114 | 115 | #[test] 116 | fn read_fail() { 117 | let text = b"The quick brown fox"; 118 | let len = text.len(); 119 | 120 | for channel_bufsize in 1..len { 121 | for queuelen in 1..len { 122 | let mut out = vec![0]; 123 | let rdr = Reader::new( 124 | text, 125 | channel_bufsize, 126 | Some(len / channel_bufsize), 127 | None, 128 | false, 129 | ); 130 | let res: io::Result<_> = reader(channel_bufsize, queuelen, rdr, |r| { 131 | while r.read(&mut out)? > 0 {} 132 | Ok(()) 133 | }); 134 | 135 | if let Err(e) = res { 136 | assert_eq!(&format!("{}", e), "read err"); 137 | } else { 138 | panic!( 139 | "read should fail at bufsize: {}, queue length: {}", 140 | channel_bufsize, queuelen 141 | ); 142 | } 143 | } 144 | } 145 | } 146 | 147 | #[test] 148 | #[should_panic(expected = "read err")] 149 | fn read_panic() { 150 | let text = b"The quick brown fox"; 151 | let rdr = Reader::new(text, 1, Some(1), None, true); 152 | let _res: io::Result<_> = reader(1, 1, rdr, |r| { 153 | r.read_to_end(&mut Vec::new())?; 154 | Ok(()) 155 | }); 156 | } 157 | 158 | #[test] 159 | fn read_fail_processing() { 160 | let text = b"The quick brown fox"; 161 | 162 | let rdr = Reader::new(text, 1, Some(1), None, false); 163 | let res: Result<(), &'static str> = reader(1, 1, rdr, |_r| Err("gave up")); 164 | 165 | if let Err(e) = res { 166 | assert_eq!(&e.to_string(), "gave up"); 167 | } else { 168 | panic!("read should fail"); 169 | } 170 | } 171 | 172 | #[test] 173 | fn read_interrupted() { 174 | let text = b"The quick brown fox"; 175 | let rdr = Reader::new(text, 2, None, Some(1), false); 176 | assert_eq!(read_chunks(rdr, 3, true).unwrap().as_slice(), &text[..]); 177 | } 178 | 179 | #[test] 180 | #[should_panic(expected = "gave up")] 181 | fn read_panic_processing() { 182 | let text = b"The quick brown fox"; 183 | 184 | let rdr = Reader::new(text, 1, Some(1), None, false); 185 | let _res: Result<(), &'static str> = reader(1, 1, rdr, |_r| panic!("gave up")); 186 | } 187 | 188 | #[test] 189 | fn reader_init_fail() { 190 | let e = io::Error::new(io::ErrorKind::Other, "init err"); 191 | let res = reader_init( 192 | 5, 193 | 2, 194 | || Err::<&[u8], _>(e), 195 | |reader| { 196 | reader.read_to_end(&mut Vec::new())?; 197 | Ok(()) 198 | }, 199 | ); 200 | if let Err(e) = res { 201 | assert_eq!(&format!("{}", e), "init err"); 202 | } else { 203 | panic!("init should fail"); 204 | } 205 | } 206 | 207 | #[test] 208 | #[should_panic(expected = "init panic")] 209 | fn reader_init_panic() { 210 | reader_init::<&[u8], _, _, _, _>(5, 2, || panic!("init panic"), |_| Ok::<_, ()>(())).unwrap(); 211 | } 212 | -------------------------------------------------------------------------------- /tests/write.rs: -------------------------------------------------------------------------------- 1 | use std::cmp::min; 2 | use std::io::{self, Write}; 3 | use thread_io::write::*; 4 | 5 | /// a writer that only writes its data to `Writer::data` upon `flush()` 6 | #[derive(Clone)] 7 | struct Writer { 8 | cache: Vec, 9 | data: Vec, 10 | write_fails: bool, 11 | flush_fails: bool, 12 | bufsize: usize, 13 | } 14 | 15 | impl Writer { 16 | fn new(write_fails: bool, flush_fails: bool, bufsize: usize) -> Writer { 17 | Writer { 18 | cache: vec![], 19 | data: vec![], 20 | write_fails, 21 | flush_fails, 22 | bufsize, 23 | } 24 | } 25 | 26 | fn data(&self) -> &[u8] { 27 | &self.data 28 | } 29 | } 30 | 31 | impl Write for Writer { 32 | fn write(&mut self, buffer: &[u8]) -> io::Result { 33 | if self.write_fails { 34 | return Err(io::Error::new(io::ErrorKind::Other, "write err")); 35 | } 36 | self.cache.write(&buffer[..min(buffer.len(), self.bufsize)]) 37 | } 38 | 39 | fn flush(&mut self) -> io::Result<()> { 40 | if self.flush_fails { 41 | Err(io::Error::new(io::ErrorKind::Other, "flush err")) 42 | } else { 43 | self.data.extend_from_slice(&self.cache); 44 | self.cache.clear(); 45 | Ok(()) 46 | } 47 | } 48 | } 49 | 50 | #[test] 51 | fn write_thread() { 52 | let text = b"The quick brown fox jumps over the lazy dog"; 53 | let n = text.len() + 3; 54 | 55 | for channel_bufsize in (1..n).step_by(2) { 56 | for writer_bufsize in (1..n).step_by(2) { 57 | for queuelen in (1..n).step_by(2) { 58 | let mut w = Writer::new(false, false, writer_bufsize); 59 | writer(channel_bufsize, queuelen, &mut w, |w| w.write(text)) 60 | .expect("writing should not fail"); 61 | if w.data() != &text[..] { 62 | panic!( 63 | "write test failed: {:?} != {:?} at channel buffer size {}, writer bufsize {}, queue length {}", 64 | String::from_utf8_lossy(w.data()), String::from_utf8_lossy(&text[..]), 65 | channel_bufsize, writer_bufsize, queuelen 66 | ); 67 | } 68 | } 69 | } 70 | } 71 | } 72 | 73 | #[test] 74 | fn writer_flush() { 75 | let text = b"was it written?"; 76 | let err_msg = "oops, it failed"; 77 | 78 | // Write without flushing by returning an error after the write 79 | let mut w = Writer::new(false, false, 1); 80 | let res: Result<(), _> = writer(1, 1, &mut w, |w| { 81 | w.write_all(text).unwrap(); 82 | Err(io::Error::new(io::ErrorKind::Other, err_msg)) 83 | }); 84 | assert!(res.is_err()); 85 | assert!(w.data().is_empty()); 86 | 87 | // If flushing before the error, the data should be there. 88 | let mut w = Writer::new(false, false, 1); 89 | let res: Result<(), _> = writer(1, 1, &mut w, |w| { 90 | w.write_all(text).unwrap(); 91 | w.flush().unwrap(); 92 | w.write_all(b"rest not flushed!").unwrap(); 93 | Err(io::Error::new(io::ErrorKind::Other, err_msg)) 94 | }); 95 | assert!(res.is_err()); 96 | assert!(w.data() == text); 97 | } 98 | 99 | #[test] 100 | fn writer_init_fail() { 101 | let e = io::Error::new(io::ErrorKind::Other, "init err"); 102 | let res = writer_init( 103 | 5, 104 | 2, 105 | || Err::<&mut [u8], _>(e), 106 | |writer| writer.write(b"let the cows come home"), 107 | ); 108 | if let Err(e) = res { 109 | assert_eq!(&format!("{}", e), "init err"); 110 | } else { 111 | panic!("init should fail"); 112 | } 113 | } 114 | 115 | #[test] 116 | #[should_panic(expected = "init panic")] 117 | fn writer_init_panic() { 118 | writer_init::<&mut Vec, _, _, _, _>( 119 | 5, 120 | 2, 121 | || panic!("init panic"), 122 | |writer| writer.write(b"let the cows come home"), 123 | ) 124 | .unwrap(); 125 | } 126 | 127 | #[test] 128 | fn write_fail() { 129 | let text = b"The quick brown fox jumps over the lazy dog"; 130 | let n = text.len() + 3; 131 | 132 | for channel_bufsize in (1..n).step_by(2) { 133 | for writer_bufsize in (1..n).step_by(2) { 134 | for queuelen in (1..n).step_by(2) { 135 | // Fail writing 136 | let w = Writer::new(true, false, writer_bufsize); 137 | let res = writer(channel_bufsize, queuelen, w, |w| w.write_all(text)); 138 | if let Err(e) = res { 139 | assert_eq!(&format!("{}", e), "write err"); 140 | } else { 141 | panic!("write should fail"); 142 | } 143 | 144 | // Fail flushing 145 | let w = Writer::new(false, true, writer_bufsize); 146 | let res = writer(channel_bufsize, queuelen, w, |w| w.flush()); 147 | if let Err(e) = res { 148 | assert_eq!(&format!("{}", e), "flush err"); 149 | } else { 150 | panic!("flush should fail"); 151 | } 152 | } 153 | } 154 | } 155 | } 156 | 157 | #[test] 158 | fn write_source_fail() { 159 | let w = Writer::new(true, false, 1); 160 | let res: std::io::Result<()> = writer(1, 1, w, |_w| { 161 | Err(std::io::Error::from(std::io::ErrorKind::AddrInUse)) 162 | }); 163 | 164 | if let Err(e) = res { 165 | assert_eq!(e.kind(), std::io::ErrorKind::AddrInUse); 166 | } else { 167 | panic!("expected error") 168 | } 169 | } 170 | 171 | #[test] 172 | #[should_panic(expected = "all out of bubblegum")] 173 | fn write_source_panic() { 174 | let w = Writer::new(true, false, 1); 175 | let _res: std::io::Result<()> = writer(1, 1, w, |_w| panic!("all out of bubblegum")); 176 | } 177 | --------------------------------------------------------------------------------