├── .envrc ├── .github ├── dependabot.yml └── workflows │ ├── coverage.yml │ └── rust.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── default.nix ├── flake.lock ├── flake.nix ├── shell.nix └── src ├── lib.rs └── tests.rs /.envrc: -------------------------------------------------------------------------------- 1 | use flake 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: cargo 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | open-pull-requests-limit: 10 8 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Coverage 2 | 3 | on: 4 | # Trigger the workflow on push or pull request, 5 | # but only for the master branch 6 | push: 7 | branches: 8 | - master 9 | pull_request: 10 | branches: 11 | - master 12 | release: 13 | types: 14 | - created 15 | 16 | jobs: 17 | coverage: 18 | runs-on: ubuntu-latest 19 | env: 20 | CARGO_TERM_COLOR: always 21 | steps: 22 | - uses: actions/checkout@v4 23 | - name: Install Rust 24 | run: rustup update stable 25 | - name: Install cargo-llvm-cov 26 | uses: taiki-e/install-action@cargo-llvm-cov 27 | - name: Generate code coverage 28 | run: cargo llvm-cov --all-features --workspace --codecov --output-path codecov.json 29 | - name: Upload coverage to Codecov 30 | uses: codecov/codecov-action@v3 31 | with: 32 | token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos 33 | files: codecov.json 34 | fail_ci_if_error: true 35 | -------------------------------------------------------------------------------- /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: Run fmt 17 | run: cargo fmt --all -- --check 18 | - name: Build 19 | run: cargo build --verbose --all-features 20 | - name: Run tests 21 | run: cargo test --verbose --all-features 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /.idea 3 | /.direnv 4 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 4 4 | 5 | [[package]] 6 | name = "crossbeam-deque" 7 | version = "0.8.6" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" 10 | dependencies = [ 11 | "crossbeam-epoch", 12 | "crossbeam-utils", 13 | ] 14 | 15 | [[package]] 16 | name = "crossbeam-epoch" 17 | version = "0.9.18" 18 | source = "registry+https://github.com/rust-lang/crates.io-index" 19 | checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" 20 | dependencies = [ 21 | "crossbeam-utils", 22 | ] 23 | 24 | [[package]] 25 | name = "crossbeam-queue" 26 | version = "0.3.12" 27 | source = "registry+https://github.com/rust-lang/crates.io-index" 28 | checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" 29 | dependencies = [ 30 | "crossbeam-utils", 31 | ] 32 | 33 | [[package]] 34 | name = "crossbeam-utils" 35 | version = "0.8.21" 36 | source = "registry+https://github.com/rust-lang/crates.io-index" 37 | checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" 38 | 39 | [[package]] 40 | name = "dynqueue" 41 | version = "0.3.1-alpha.0" 42 | dependencies = [ 43 | "crossbeam-queue", 44 | "rayon", 45 | ] 46 | 47 | [[package]] 48 | name = "either" 49 | version = "1.15.0" 50 | source = "registry+https://github.com/rust-lang/crates.io-index" 51 | checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" 52 | 53 | [[package]] 54 | name = "rayon" 55 | version = "1.10.0" 56 | source = "registry+https://github.com/rust-lang/crates.io-index" 57 | checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" 58 | dependencies = [ 59 | "either", 60 | "rayon-core", 61 | ] 62 | 63 | [[package]] 64 | name = "rayon-core" 65 | version = "1.12.1" 66 | source = "registry+https://github.com/rust-lang/crates.io-index" 67 | checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" 68 | dependencies = [ 69 | "crossbeam-deque", 70 | "crossbeam-utils", 71 | ] 72 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dynqueue" 3 | version = "0.3.1-alpha.0" 4 | authors = ["Harald Hoyer "] 5 | edition = "2018" 6 | 7 | license = "MIT" 8 | documentation = "https://docs.rs/dynqueue" 9 | homepage = "https://github.com/haraldh/dynqueue" 10 | repository = "https://github.com/haraldh/dynqueue" 11 | description = "Dynamically extendable Rayon parallel iterator" 12 | readme = "README.md" 13 | 14 | keywords = [ "parallel", "performance", "thread", "join", "concurrency"] 15 | categories = [ "concurrency" ] 16 | 17 | [dependencies] 18 | rayon = "1.3" 19 | crossbeam-queue = { version = "0.3", optional = true } 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Harald Hoyer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Rust](https://github.com/haraldh/dynqueue/workflows/Rust/badge.svg)](https://github.com/haraldh/dynqueue/actions) 2 | [![Coverage Status](https://codecov.io/gh/haraldh/dynqueue/graph/badge.svg?token=E2KO8O9W9O)](https://codecov.io/gh/haraldh/dynqueue) 3 | 4 | # DynQueue - dynamically extendable Rayon parallel iterator 5 | 6 | DynQueue is a Rust library that provides a specialized parallel iterator built on top of the Rayon parallel computing 7 | framework. Its key feature is the ability to dynamically add new items to the collection while it's being processed in 8 | parallel. 9 | 10 | This project fills a specific niche in the Rust parallel computing ecosystem by providing a way to work with dynamically 11 | growing workloads in a parallel context, something that's not directly supported by Rayon itself. 12 | 13 | ## Core Functionality: 14 | 15 | - Allows adding new elements to a collection while it's being iterated over in parallel 16 | - Works with multiple collection types: `Vec`, `VecDeque`, and `crossbeam_queue::SegQueue` (via feature flag) 17 | - Uses Rayon's parallel iteration capabilities for efficient parallel processing 18 | - Provides a clean API for dynamically expanding workloads 19 | 20 | ## Technical Implementation: 21 | 22 | - Implements Rayon's parallel iteration traits 23 | - Uses a handle-based approach where each iterator function receives a handle to insert new items 24 | - Thread-safe implementation using RwLock and Arc for standard collections 25 | - Optional integration with crossbeam's lock-free SegQueue 26 | 27 | ## Use Cases: 28 | 29 | The library is particularly useful for: 30 | 31 | - Tree/graph traversal algorithms where new nodes are discovered during processing 32 | - Work-stealing parallel algorithms where work can be dynamically generated 33 | - Any parallel processing task where the total workload is not known upfront 34 | 35 | ## Example Usage: 36 | 37 | ```rust 38 | use rayon::iter::IntoParallelIterator as _; 39 | use rayon::iter::ParallelIterator as _; 40 | use dynqueue::IntoDynQueue as _; 41 | 42 | fn main() { 43 | let mut result = vec![1, 2, 3] 44 | .into_dyn_queue() 45 | .into_par_iter() 46 | .map(|(handle, value)| { 47 | if value == 2 { 48 | handle.enqueue(4) 49 | }; 50 | value 51 | }) 52 | .collect::>(); 53 | result.sort(); 54 | 55 | assert_eq!(result, vec![1, 2, 3, 4]); 56 | } 57 | ``` 58 | 59 | ## TL;DR 60 | 61 | A `DynQueue` can be iterated with `into_par_iter` producing `(DynQueueHandle, T)` elements. 62 | With the `DynQueueHandle` a new `T` can be inserted in the `DynQueue`, 63 | which is currently iterated over. 64 | 65 | A `Vec`, `VecDeque` and `crossbeam_queue::SegQueue` (with `feature = "crossbeam-queue"`) 66 | can be turned into a `DynQueue` with `.into_dyn_queue()`. 67 | 68 | ## Features 69 | 70 | * `crossbeam-queue` : to use `crossbeam::queue::SegQueue` as the inner collection. 71 | 72 | ## Changelog 73 | 74 | ### 0.2.0 75 | 76 | - introduce `IntoDynQueue` 77 | - handle lockless collections 78 | 79 | ### 0.1.0 80 | 81 | - initial version 82 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , darwin 3 | , stdenv 4 | , openssl 5 | , pkg-config 6 | , rustPlatform 7 | , 8 | }: 9 | let 10 | cargoFile = (builtins.fromTOML (builtins.readFile ./Cargo.toml)).package; 11 | in 12 | rustPlatform.buildRustPackage { 13 | 14 | pname = cargoFile.name; # The name of the package 15 | version = cargoFile.version; # The version of the package 16 | 17 | # You can use lib here to make a more accurate source 18 | # this can be nice to reduce the amount of rebuilds 19 | # but thats out of scope for this post 20 | src = ./.; # The source of the package 21 | 22 | # The lock file of the package, this can be done in other ways 23 | # like cargoHash, we are not doing it in this case because this 24 | # is much simpler, especially if we have access to the lock file 25 | # in our source tree 26 | cargoLock.lockFile = ./Cargo.lock; 27 | 28 | # The runtime dependencies of the package 29 | buildInputs = 30 | [ openssl ] 31 | ++ lib.optionals stdenv.isDarwin ( 32 | with darwin.apple_sdk.frameworks; 33 | [ 34 | Security 35 | CoreFoundation 36 | SystemConfiguration 37 | ] 38 | ); 39 | 40 | # programs and libraries used at build-time that, if they are a compiler or 41 | # similar tool, produce code to run at run-time—i.e. tools used to build the new derivation 42 | nativeBuildInputs = [ pkg-config ]; 43 | 44 | meta = { 45 | license = lib.licenses.mit; 46 | mainProgram = cargoFile.name; 47 | }; 48 | } 49 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "nixpkgs": { 4 | "locked": { 5 | "lastModified": 1743315132, 6 | "narHash": "sha256-6hl6L/tRnwubHcA4pfUUtk542wn2Om+D4UnDhlDW9BE=", 7 | "owner": "nixos", 8 | "repo": "nixpkgs", 9 | "rev": "52faf482a3889b7619003c0daec593a1912fddc1", 10 | "type": "github" 11 | }, 12 | "original": { 13 | "owner": "nixos", 14 | "ref": "nixos-unstable", 15 | "repo": "nixpkgs", 16 | "type": "github" 17 | } 18 | }, 19 | "nixpkgs_2": { 20 | "locked": { 21 | "lastModified": 1736320768, 22 | "narHash": "sha256-nIYdTAiKIGnFNugbomgBJR+Xv5F1ZQU+HfaBqJKroC0=", 23 | "owner": "NixOS", 24 | "repo": "nixpkgs", 25 | "rev": "4bc9c909d9ac828a039f288cf872d16d38185db8", 26 | "type": "github" 27 | }, 28 | "original": { 29 | "owner": "NixOS", 30 | "ref": "nixpkgs-unstable", 31 | "repo": "nixpkgs", 32 | "type": "github" 33 | } 34 | }, 35 | "root": { 36 | "inputs": { 37 | "nixpkgs": "nixpkgs", 38 | "rust-overlay": "rust-overlay" 39 | } 40 | }, 41 | "rust-overlay": { 42 | "inputs": { 43 | "nixpkgs": "nixpkgs_2" 44 | }, 45 | "locked": { 46 | "lastModified": 1743388531, 47 | "narHash": "sha256-OBcNE+2/TD1AMgq8HKMotSQF8ZPJEFGZdRoBJ7t/HIc=", 48 | "owner": "oxalica", 49 | "repo": "rust-overlay", 50 | "rev": "011de3c895927300651d9c2cb8e062adf17aa665", 51 | "type": "github" 52 | }, 53 | "original": { 54 | "owner": "oxalica", 55 | "repo": "rust-overlay", 56 | "type": "github" 57 | } 58 | } 59 | }, 60 | "root": "root", 61 | "version": 7 62 | } 63 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | inputs = { 3 | nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; 4 | rust-overlay.url = "github:oxalica/rust-overlay"; 5 | }; 6 | outputs = { self, nixpkgs, rust-overlay, ... }: 7 | let 8 | namespace = "dynqueue"; 9 | overlays = [ 10 | rust-overlay.overlays.default 11 | ]; 12 | forEachSystem = systems: f: nixpkgs.lib.genAttrs systems f; 13 | forAllSystems = function: nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed ( 14 | system: function (import nixpkgs { inherit system overlays; }) 15 | ); 16 | in 17 | { 18 | packages = forAllSystems (pkgs: (self.overlays.default pkgs pkgs).${namespace}); 19 | cross = forAllSystems (pkgs: (forEachSystem (nixpkgs.lib.filter (sys: sys != pkgs.system) nixpkgs.lib.systems.flakeExposed) (crossSystem: 20 | let 21 | crossPkgs = import nixpkgs { localSystem = pkgs.system; inherit crossSystem; }; 22 | in 23 | (self.overlays.default crossPkgs crossPkgs).${namespace} 24 | ))); 25 | devShells = forAllSystems (pkgs: (self.overlays.default pkgs pkgs).devShells); 26 | formatter = forAllSystems (pkgs: pkgs.nixpkgs-fmt); 27 | overlays.default = final: prev: 28 | let pkgs = final; in { 29 | devShells.default = pkgs.callPackage ./shell.nix { }; 30 | ${namespace} = { 31 | dynqueue = pkgs.callPackage ./default.nix { }; 32 | default = pkgs.callPackage ./default.nix { }; 33 | }; 34 | }; 35 | }; 36 | } 37 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | { clippy 2 | , rustfmt 3 | , callPackage 4 | , rust-analyzer 5 | , 6 | }: 7 | let 8 | mainPkg = callPackage ./default.nix { }; 9 | in 10 | mainPkg.overrideAttrs (prev: { 11 | nativeBuildInputs = [ 12 | # Additional Rust tooling 13 | clippy 14 | rustfmt 15 | rust-analyzer 16 | ] ++ (prev.nativeBuildInputs or [ ]); 17 | }) 18 | 19 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! DynQueue - dynamically extendable Rayon parallel iterator 2 | //! 3 | //! A `DynQueue` can be iterated with `into_par_iter` producing `(DynQueueHandle, T)` elements. 4 | //! With the `DynQueueHandle` a new `T` can be inserted in the `DynQueue`, 5 | //! which is currently iterated over. 6 | //! 7 | //! # Example 8 | //! 9 | //! ``` 10 | //! use rayon::iter::IntoParallelIterator as _; 11 | //! use rayon::iter::ParallelIterator as _; 12 | //! 13 | //! use dynqueue::IntoDynQueue as _; 14 | //! 15 | //! let mut result = vec![1, 2, 3] 16 | //! .into_dyn_queue() 17 | //! .into_par_iter() 18 | //! .map(|(handle, value)| { 19 | //! if value == 2 { 20 | //! handle.enqueue(4) 21 | //! }; 22 | //! value 23 | //! }) 24 | //! .collect::>(); 25 | //! result.sort(); 26 | //! 27 | //! assert_eq!(result, vec![1, 2, 3, 4]); 28 | //! ``` 29 | //! 30 | //! # Panics 31 | //! 32 | //! The `DynQueueHandle` shall not outlive the `DynQueue` iterator 33 | //! 34 | //! ```should_panic 35 | //! use dynqueue::{DynQueue, DynQueueHandle, IntoDynQueue}; 36 | //! 37 | //! use rayon::iter::IntoParallelIterator as _; 38 | //! use rayon::iter::ParallelIterator as _; 39 | //! use std::sync::RwLock; 40 | //! 41 | //! static mut STALE_HANDLE: Option>>> = None; 42 | //! 43 | //! pub fn test_func() -> Vec { 44 | //! vec![1u8, 2u8, 3u8] 45 | //! .into_dyn_queue() 46 | //! .into_par_iter() 47 | //! .map(|(handle, value)| unsafe { 48 | //! STALE_HANDLE.replace(handle); 49 | //! value 50 | //! }) 51 | //! .collect::>() 52 | //! } 53 | //! // test_func() panics 54 | //! let result = test_func(); 55 | //! unsafe { 56 | //! STALE_HANDLE.as_ref().unwrap().enqueue(4); 57 | //! } 58 | //! ``` 59 | 60 | #![deny(clippy::all)] 61 | #![deny(missing_docs)] 62 | 63 | #[allow(unused)] 64 | macro_rules! doc_comment { 65 | ($x:expr) => { 66 | #[doc = $x] 67 | #[doc(hidden)] 68 | mod readme_tests {} 69 | }; 70 | } 71 | 72 | doc_comment!(include_str!("../README.md")); 73 | 74 | use rayon::iter::plumbing::{ 75 | bridge_unindexed, Consumer, Folder, UnindexedConsumer, UnindexedProducer, 76 | }; 77 | use std::collections::VecDeque; 78 | use std::marker::PhantomData; 79 | use std::sync::{Arc, RwLock}; 80 | 81 | #[cfg(test)] 82 | mod tests; 83 | 84 | /// Trait to produce a new DynQueue 85 | pub trait IntoDynQueue> { 86 | /// new 87 | fn into_dyn_queue<'a>(self) -> DynQueue<'a, T, U>; 88 | } 89 | 90 | /// Everything implementing `Queue` can be handled by DynQueue 91 | #[allow(clippy::len_without_is_empty)] 92 | pub trait Queue 93 | where 94 | Self: Sized, 95 | { 96 | /// push an element in the queue 97 | fn push(&self, v: T); 98 | 99 | /// pop an element from the queue 100 | fn pop(&self) -> Option; 101 | 102 | /// number of elements in the queue 103 | fn len(&self) -> usize; 104 | 105 | /// split off `size` elements 106 | fn split_off(&self, size: usize) -> Self; 107 | } 108 | 109 | impl IntoDynQueue>> for Vec { 110 | #[inline(always)] 111 | fn into_dyn_queue<'a>(self) -> DynQueue<'a, T, RwLock>> { 112 | DynQueue(Arc::new(DynQueueInner(RwLock::new(self), PhantomData))) 113 | } 114 | } 115 | 116 | impl IntoDynQueue>> for RwLock> { 117 | #[inline(always)] 118 | fn into_dyn_queue<'a>(self) -> DynQueue<'a, T, RwLock>> { 119 | DynQueue(Arc::new(DynQueueInner(self, PhantomData))) 120 | } 121 | } 122 | 123 | impl Queue for RwLock> { 124 | #[inline(always)] 125 | fn push(&self, v: T) { 126 | self.write().unwrap().push(v) 127 | } 128 | 129 | #[inline(always)] 130 | fn pop(&self) -> Option { 131 | self.write().unwrap().pop() 132 | } 133 | 134 | #[inline(always)] 135 | fn len(&self) -> usize { 136 | self.read().unwrap().len() 137 | } 138 | 139 | #[inline(always)] 140 | fn split_off(&self, size: usize) -> Self { 141 | RwLock::new(self.write().unwrap().split_off(size)) 142 | } 143 | } 144 | 145 | impl IntoDynQueue>> for VecDeque { 146 | #[inline(always)] 147 | fn into_dyn_queue<'a>(self) -> DynQueue<'a, T, RwLock>> { 148 | DynQueue(Arc::new(DynQueueInner(RwLock::new(self), PhantomData))) 149 | } 150 | } 151 | 152 | impl IntoDynQueue>> for RwLock> { 153 | #[inline(always)] 154 | fn into_dyn_queue<'a>(self) -> DynQueue<'a, T, RwLock>> { 155 | DynQueue(Arc::new(DynQueueInner(self, PhantomData))) 156 | } 157 | } 158 | 159 | impl Queue for RwLock> { 160 | #[inline(always)] 161 | fn push(&self, v: T) { 162 | self.write().unwrap().push_back(v) 163 | } 164 | 165 | #[inline(always)] 166 | fn pop(&self) -> Option { 167 | self.write().unwrap().pop_front() 168 | } 169 | 170 | #[inline(always)] 171 | fn len(&self) -> usize { 172 | self.read().unwrap().len() 173 | } 174 | 175 | #[inline(always)] 176 | fn split_off(&self, size: usize) -> Self { 177 | RwLock::new(self.write().unwrap().split_off(size)) 178 | } 179 | } 180 | 181 | #[cfg(feature = "crossbeam-queue")] 182 | use crossbeam_queue::SegQueue; 183 | 184 | #[cfg(feature = "crossbeam-queue")] 185 | impl IntoDynQueue> for SegQueue { 186 | #[inline(always)] 187 | fn into_dyn_queue<'a>(self) -> DynQueue<'a, T, Self> { 188 | DynQueue(Arc::new(DynQueueInner(self, PhantomData))) 189 | } 190 | } 191 | 192 | #[cfg(feature = "crossbeam-queue")] 193 | impl Queue for SegQueue { 194 | #[inline(always)] 195 | fn push(&self, v: T) { 196 | SegQueue::push(self, v); 197 | } 198 | 199 | #[inline(always)] 200 | fn pop(&self) -> Option { 201 | SegQueue::pop(self) 202 | } 203 | 204 | #[inline(always)] 205 | fn len(&self) -> usize { 206 | SegQueue::len(self) 207 | } 208 | 209 | #[inline(always)] 210 | fn split_off(&self, size: usize) -> Self { 211 | let q = SegQueue::new(); 212 | (0..size) 213 | .filter_map(|_| Queue::pop(self)) 214 | .for_each(|ele| q.push(ele)); 215 | q 216 | } 217 | } 218 | 219 | // PhantomData should prevent `DynQueueInner` to outlive the original `DynQueue` 220 | // but does not always. 221 | struct DynQueueInner<'a, T, U: Queue>(U, PhantomData<&'a T>); 222 | 223 | /// The `DynQueueHandle` returned by the iterator in addition to `T` 224 | pub struct DynQueueHandle<'a, T, U: Queue>(Arc>); 225 | 226 | impl> DynQueueHandle<'_, T, U> { 227 | /// Enqueue `T` in the `DynQueue`, which is currently iterated. 228 | #[inline] 229 | pub fn enqueue(&self, job: T) { 230 | (self.0).0.push(job) 231 | } 232 | } 233 | 234 | /// The `DynQueue` which can be parallel iterated over 235 | pub struct DynQueue<'a, T, U: Queue>(Arc>); 236 | 237 | impl<'a, T, U> UnindexedProducer for DynQueue<'a, T, U> 238 | where 239 | T: Send + Sync, 240 | U: IntoDynQueue + Queue + Send + Sync, 241 | { 242 | type Item = (DynQueueHandle<'a, T, U>, T); 243 | 244 | fn split(self) -> (Self, Option) { 245 | let len = (self.0).0.len(); 246 | 247 | if len >= 2 { 248 | let new_q = (self.0).0.split_off(len / 2); 249 | (self, Some(new_q.into_dyn_queue())) 250 | } else { 251 | (self, None) 252 | } 253 | } 254 | 255 | fn fold_with(self, folder: F) -> F 256 | where 257 | F: Folder, 258 | { 259 | let mut folder = folder; 260 | loop { 261 | let ret = (self.0).0.pop(); 262 | 263 | if let Some(v) = ret { 264 | folder = folder.consume((DynQueueHandle(self.0.clone()), v)); 265 | 266 | if folder.full() { 267 | break; 268 | } 269 | } else { 270 | // Self shall have the only reference 271 | assert_eq!(Arc::strong_count(&self.0), 1, "Stale Handle"); 272 | break; 273 | } 274 | } 275 | folder 276 | } 277 | } 278 | 279 | impl<'a, T, U> rayon::iter::ParallelIterator for DynQueue<'a, T, U> 280 | where 281 | T: Send + Sync, 282 | U: IntoDynQueue + Queue + Send + Sync, 283 | { 284 | type Item = (DynQueueHandle<'a, T, U>, T); 285 | 286 | fn drive_unindexed(self, consumer: C) -> >::Result 287 | where 288 | C: UnindexedConsumer, 289 | { 290 | bridge_unindexed(self, consumer) 291 | } 292 | } 293 | -------------------------------------------------------------------------------- /src/tests.rs: -------------------------------------------------------------------------------- 1 | use crate::{DynQueueHandle, IntoDynQueue, Queue}; 2 | use std::collections::VecDeque; 3 | 4 | const SLEEP_MS: u64 = 10; 5 | 6 | #[inline] 7 | fn handle_queue>(t: (DynQueueHandle, u64)) -> u64 { 8 | let (h, v) = t; 9 | 10 | if v % 2 == 0 { 11 | h.enqueue(11); 12 | } 13 | if v % 3 == 0 { 14 | h.enqueue(11); 15 | } 16 | if v % 4 == 0 { 17 | h.enqueue(11); 18 | } 19 | if v == 11 { 20 | h.enqueue(5); 21 | h.enqueue(17); 22 | } 23 | v 24 | } 25 | 26 | #[inline] 27 | fn get_input() -> Vec { 28 | vec![ 29 | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 30 | ] 31 | } 32 | 33 | #[inline] 34 | fn get_expected() -> Vec { 35 | vec![ 36 | 1, 2, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 37 | 8, 9, 10, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 38 | 11, 11, 11, 12, 13, 14, 15, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 39 | 17, 17, 17, 17, 17, 17, 17, 17, 17, 18, 19, 20, 21, 40 | ] 41 | } 42 | 43 | #[test] 44 | fn dynqueue_iter_test_const_sleep() { 45 | use rayon::iter::IntoParallelIterator as _; 46 | use rayon::iter::ParallelIterator as _; 47 | use std::time::Duration; 48 | let expected = get_expected(); 49 | 50 | let med = expected.iter().sum::() / expected.len() as u64; 51 | 52 | let jq = get_input().into_dyn_queue(); 53 | let now = std::time::Instant::now(); 54 | 55 | let mut res = jq 56 | .into_par_iter() 57 | .map(handle_queue) 58 | .map(|v| { 59 | std::thread::sleep(Duration::from_millis(SLEEP_MS * med)); 60 | v 61 | }) 62 | .collect::>(); 63 | eprintln!("elapsed = {:#?}", now.elapsed()); 64 | res.sort(); 65 | assert_eq!(res, expected); 66 | eprintln!( 67 | "instead of = {}ms", 68 | res.len() * med as usize * SLEEP_MS as usize 69 | ); 70 | } 71 | 72 | #[cfg(feature = "crossbeam-queue")] 73 | #[test] 74 | fn dynqueue_iter_test_const_sleep_segqueue() { 75 | use crossbeam_queue::SegQueue; 76 | use rayon::iter::IntoParallelIterator as _; 77 | use rayon::iter::ParallelIterator as _; 78 | use std::time::Duration; 79 | let expected = get_expected(); 80 | 81 | let med = expected.iter().sum::() / expected.iter().count() as u64; 82 | let jq = SegQueue::new(); 83 | get_input().drain(..).for_each(|ele| jq.push(ele)); 84 | 85 | let now = std::time::Instant::now(); 86 | 87 | let mut res = jq 88 | .into_dyn_queue() 89 | .into_par_iter() 90 | .map(handle_queue) 91 | .map(|v| { 92 | std::thread::sleep(Duration::from_millis(SLEEP_MS * med)); 93 | v 94 | }) 95 | .collect::>(); 96 | eprintln!("elapsed = {:#?}", now.elapsed()); 97 | res.sort(); 98 | assert_eq!(res, expected); 99 | eprintln!( 100 | "instead of = {}ms", 101 | res.iter().count() * med as usize * SLEEP_MS as usize 102 | ); 103 | } 104 | 105 | #[test] 106 | fn dynqueue_iter_test_const_sleep_vecdeque() { 107 | use rayon::iter::IntoParallelIterator as _; 108 | use rayon::iter::ParallelIterator as _; 109 | use std::time::Duration; 110 | let expected = get_expected(); 111 | 112 | let med = expected.iter().sum::() / expected.len() as u64; 113 | 114 | let jq = VecDeque::from(get_input()); 115 | let now = std::time::Instant::now(); 116 | 117 | let mut res = jq 118 | .into_dyn_queue() 119 | .into_par_iter() 120 | .map(handle_queue) 121 | .map(|v| { 122 | std::thread::sleep(Duration::from_millis(SLEEP_MS * med)); 123 | v 124 | }) 125 | .collect::>(); 126 | eprintln!("elapsed = {:#?}", now.elapsed()); 127 | res.sort(); 128 | assert_eq!(res, expected); 129 | eprintln!( 130 | "instead of = {}ms", 131 | res.len() * med as usize * SLEEP_MS as usize 132 | ); 133 | } 134 | 135 | #[test] 136 | fn dynqueue_iter_test_sleep_v() { 137 | use rayon::iter::IntoParallelIterator as _; 138 | use rayon::iter::ParallelIterator as _; 139 | use std::time::Duration; 140 | 141 | let jq = get_input(); 142 | 143 | let now = std::time::Instant::now(); 144 | 145 | let mut res = jq 146 | .into_dyn_queue() 147 | .into_par_iter() 148 | .map(handle_queue) 149 | .map(|v| { 150 | std::thread::sleep(Duration::from_millis(SLEEP_MS * v)); 151 | v 152 | }) 153 | .collect::>(); 154 | eprintln!("elapsed = {:#?}", now.elapsed()); 155 | res.sort(); 156 | assert_eq!(res, get_expected()); 157 | eprintln!("instead of = {}ms", res.iter().sum::() * SLEEP_MS); 158 | } 159 | 160 | #[test] 161 | fn dynqueue_iter_test_sleep_inv_v() { 162 | use rayon::iter::IntoParallelIterator as _; 163 | use rayon::iter::ParallelIterator as _; 164 | use std::time::Duration; 165 | 166 | let jq = get_input(); 167 | 168 | let now = std::time::Instant::now(); 169 | 170 | let mut res = jq 171 | .into_dyn_queue() 172 | .into_par_iter() 173 | .map(handle_queue) 174 | .map(|v| { 175 | std::thread::sleep(Duration::from_millis(SLEEP_MS * (22 - v))); 176 | v 177 | }) 178 | .collect::>(); 179 | eprintln!("elapsed = {:#?}", now.elapsed()); 180 | res.sort(); 181 | assert_eq!(res, get_expected()); 182 | eprintln!( 183 | "instead of = {}ms", 184 | (res.len() as u64 * 22 - res.iter().sum::()) * SLEEP_MS 185 | ); 186 | } 187 | 188 | #[test] 189 | fn par_iter_test() { 190 | use rayon::iter::IntoParallelIterator as _; 191 | use rayon::iter::ParallelIterator as _; 192 | use std::time::Duration; 193 | 194 | let now = std::time::Instant::now(); 195 | 196 | let res = get_expected() 197 | .into_par_iter() 198 | .map(|v| { 199 | std::thread::sleep(Duration::from_millis(SLEEP_MS * v)); 200 | v 201 | }) 202 | .collect::>(); 203 | eprintln!("elapsed = {:#?}", now.elapsed()); 204 | eprintln!("instead of = {}ms", res.iter().sum::() * SLEEP_MS); 205 | } 206 | --------------------------------------------------------------------------------