├── .gitignore ├── .travis.yml ├── Cargo.toml ├── LICENSE ├── examples └── ppipe_example.rs ├── src └── lib.rs └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | rust: 3 | - stable 4 | - nightly 5 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | 3 | name = "ppipe" 4 | version = "0.2.0" 5 | authors = ["Michael J Welsh "] 6 | license = "MIT" 7 | readme = "README.md" 8 | repository = "https://github.com/MichaelJWelsh/ppipe.git" 9 | homepage = "https://github.com/MichaelJWelsh/ppipe" 10 | description = """ 11 | An elegantly simple and lightweight library for making iterator pipelines concurrent and amazingly fast, 12 | hence the name ppipe (parallel pipe). 13 | """ 14 | keywords = ["pipe", "pipeline", "iterator", "adaptor", "concurrent"] 15 | 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Michael J Welsh 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /examples/ppipe_example.rs: -------------------------------------------------------------------------------- 1 | /* The point of this simplistic example is to demonstrate how WITHOUT `ppipe`, every iteration moves 2 | * the corresponding item along the pipeline and then executes, in this case, the `for` loop's 3 | * body. The items are never pre-loaded into some buffer waiting for the iteration variable to take 4 | * ownership after being forced to move along the pipeline. This is almost never idealistic; why 5 | * limit yourself to serial processing when you have Rust's powerful parallel processing at hand? 6 | * This is what `ppipe` does. WITH `ppipe`, all previous iterator adaptors are ran regardless of what 7 | * iteration, in this case, the `for` loop is on, including any previous `ppipe` adaptors which are 8 | * busy doing their own thing. Every item that is processed is put in a buffer which can be 9 | * accessed as it is being added to, and if there are no items in the buffer, the iteration will 10 | * simply block until an item is available, or break if there are no more items being processed. 11 | * This means items can be added to the buffer as you are iterating over previous items in the 12 | * buffer, which ultimately reduces bottlenecking and GREATLY increases performance! 13 | */ 14 | 15 | 16 | extern crate ppipe; 17 | 18 | 19 | use ppipe::*; 20 | use std::thread::sleep; 21 | use std::time::Duration; 22 | 23 | 24 | #[allow(unused_variables)] 25 | fn main() { 26 | println!("Without ppipe usage:\n"); 27 | for v in (0..10).map(|x| { println!("not multitasking"); }) 28 | /* .filter..., .take..., etc... */ 29 | { 30 | sleep(Duration::from_millis(2000)); 31 | println!("Iteration completed"); 32 | } 33 | println!("\n"); 34 | 35 | println!("With ppipe usage:\n"); 36 | for v in (0..10).map(|x| { println!("multitasking"); }) 37 | .ppipe(None) 38 | /* .filter..., .take..., etc... */ 39 | { 40 | sleep(Duration::from_millis(2000)); 41 | println!("Iteration completed"); 42 | } 43 | println!("\n"); 44 | } 45 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! # ppipe 2 | //! 3 | //! An elegantly simple and lightweight library for making iterator pipelines concurrent and 4 | //! amazingly fast, hence the name "ppipe" (parallel pipe). 5 | 6 | 7 | #![deny(missing_docs)] 8 | 9 | 10 | use std::sync::mpsc; 11 | use std::thread; 12 | 13 | 14 | /// This trait does all the work for you so that generally every iterator you use has the ppipe 15 | /// method. Make sure to include this trait for it to take effect: 16 | /// 17 | /// ```no_run 18 | /// 19 | /// extern crate ppipe; 20 | /// use ppipe::*; 21 | /// 22 | /// ``` 23 | pub trait PPipe: Iterator { 24 | /// This method can be called on generally any iterator, making every previous task become part 25 | /// of a concurrent pipeline. 26 | /// 27 | /// `ppipe` takes an `Option` parameter which can be used to declare if you want 28 | /// back-pressure or not. `ppipe(Some(1000))` would mean that you want the concurrent receiver to 29 | /// hold no more than 1000 values and tell the sender to block until the receiver's buffer goes 30 | /// below 1000 over the course of, for example, a `for` loop. 31 | fn ppipe(self, back_pressure: Option) -> mpsc::IntoIter; 32 | } 33 | 34 | 35 | impl PPipe for T 36 | where T: Iterator + Send + 'static, 37 | T::Item: Send + 'static 38 | { 39 | fn ppipe(self, back_pressure: Option) -> mpsc::IntoIter { 40 | if back_pressure.is_some() { 41 | let (sender, receiver) = mpsc::sync_channel(back_pressure.unwrap()); 42 | 43 | thread::spawn(move || { 44 | for item in self { 45 | if sender.send(item).is_err() { 46 | break; 47 | } 48 | } 49 | }); 50 | 51 | receiver.into_iter() 52 | } else { 53 | let (sender, receiver) = mpsc::channel(); 54 | 55 | thread::spawn(move || { 56 | for item in self { 57 | if sender.send(item).is_err() { 58 | break; 59 | } 60 | } 61 | }); 62 | 63 | receiver.into_iter() 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ppipe 2 | An elegantly simple and lightweight Rust library for making iterator pipelines concurrent and amazingly fast, hence the name "ppipe" (parallel pipe). Find it here: https://crates.io/crates/ppipe 3 | 4 | 5 | ## Usage 6 | Put this in your Cargo.toml: 7 | ```toml 8 | [dependencies] 9 | 10 | ppipe = "0.2.0" 11 | ``` 12 | 13 | Then add this to your root module: 14 | ```rust 15 | extern crate ppipe; 16 | ``` 17 | 18 | And add this to whatever module you want to be able to use ppipe: 19 | ```rust 20 | use ppipe::*; 21 | ``` 22 | 23 | Now generally all iterators in scope should have the `ppipe` method! 24 | 25 | 26 | ## Overview 27 | If you followed the above steps, every iterator should have the `ppipe` method. The method takes an `Option` parameter which can be used to declare if you want back-pressure or not. `ppipe(Some(1000))` would mean that you want the concurrent receiver to hold no more than 1000 values and tell the sender to block until the receiver's buffer goes below 1000 over the course of, for example, a `for` loop. 28 | 29 | 30 | ## Example 31 | ```rust 32 | extern crate ppipe; 33 | use ppipe::*; 34 | 35 | // ... 36 | 37 | for item in excel_sheet.into_iter() 38 | .map(do_something) 39 | .write_out_err(some_err_handling_func) 40 | .ppipe(None) // create a thread for the above work 41 | .map(do_something_else) 42 | .ppipe(None) // create another thread for the the above work 43 | // ... 44 | { 45 | // ... 46 | } 47 | ``` 48 | 49 | ## How It Works Internally 50 | The significance of this little library is hard to put into words, so please refer to the **ppipe_example.rs** in the *examples* folder of this repository, which I will reference in this explanation. 51 | 52 | The point of this simplistic example is to demonstrate how WITHOUT `ppipe`, every iteration moves the corresponding item along the pipeline and then executes, in this case, the `for` loop's body. The items are never pre-loaded into some buffer waiting for the iteration variable to take ownership after being forced to move along the pipeline. This is almost never idealistic; why limit yourself to serial processing when you have Rust's powerful parallel processing at hand? This is what `ppipe` does. WITH `ppipe`, all previous iterator adaptors are ran regardless of what iteration, in this case, the `for` loop is on, including any previous `ppipe` adaptors which are busy doing their own thing. Every item that is processed is put in a buffer which can be accessed as it is being added to, and if there are no items in the buffer, the iteration will simply block until an item is available, or break if there are no more items being processed. This means items can be added to the buffer as you are iterating over previous items in the buffer, which ultimately reduces bottlenecking and GREATLY increases performance! 53 | 54 | --------------------------------------------------------------------------------