├── .github └── workflows │ └── main-checks.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── Makefile ├── README.md ├── images ├── animation.html └── process.gif └── src ├── deluge.rs ├── deluge_ext.rs ├── helpers ├── indexable_stream.rs ├── mod.rs └── preloaded_futures.rs ├── into_deluge.rs ├── iter.rs ├── lib.rs └── ops ├── all.rs ├── all_par.rs ├── any.rs ├── any_par.rs ├── chain.rs ├── collect.rs ├── collect_par.rs ├── count.rs ├── filter.rs ├── filter_map.rs ├── first.rs ├── fold.rs ├── fold_par.rs ├── last.rs ├── map.rs ├── mod.rs ├── take.rs └── zip.rs /.github/workflows/main-checks.yml: -------------------------------------------------------------------------------- 1 | name: Main checks 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - '.github/workflows/main-checks.yml' 7 | - 'src/**/*' 8 | - 'Cargo.toml' 9 | push: 10 | branches: ["main"] 11 | 12 | jobs: 13 | clippy: 14 | name: Clippy 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | profile: 20 | - dev 21 | - release 22 | feature: 23 | - tokio 24 | - async-std 25 | steps: 26 | - uses: actions/checkout@v2 27 | - uses: actions-rs/toolchain@v1 28 | with: 29 | toolchain: nightly 30 | override: true 31 | profile: minimal 32 | components: clippy 33 | - uses: Swatinem/rust-cache@v1 34 | 35 | - name: Run clippy 36 | uses: actions-rs/cargo@v1 37 | with: 38 | command: clippy 39 | args: --all-targets --no-default-features --features ${{ matrix.feature }} --profile ${{ matrix.profile }} -- -D warnings 40 | 41 | format: 42 | name: cargo fmt 43 | runs-on: ubuntu-latest 44 | steps: 45 | - uses: actions/checkout@v2 46 | - uses: actions-rs/toolchain@v1 47 | with: 48 | toolchain: nightly 49 | override: true 50 | profile: minimal 51 | components: rustfmt 52 | - uses: Swatinem/rust-cache@v1 53 | 54 | - name: Run fmt 55 | uses: actions-rs/cargo@v1 56 | with: 57 | command: fmt 58 | toolchain: nightly 59 | args: --all -- --check 60 | 61 | test: 62 | runs-on: ubuntu-latest 63 | 64 | strategy: 65 | fail-fast: false 66 | matrix: 67 | feature: 68 | - tokio 69 | - async-std 70 | steps: 71 | - uses: actions/checkout@v3 72 | - uses: actions-rs/toolchain@v1 73 | with: 74 | toolchain: nightly 75 | override: true 76 | profile: minimal 77 | - uses: Swatinem/rust-cache@v1 78 | - name: Run tests 79 | run: cargo test --no-default-features --features ${{ matrix.feature }} 80 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "addr2line" 7 | version = "0.21.0" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" 10 | dependencies = [ 11 | "gimli", 12 | ] 13 | 14 | [[package]] 15 | name = "adler" 16 | version = "1.0.2" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" 19 | 20 | [[package]] 21 | name = "async-attributes" 22 | version = "1.1.2" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" 25 | dependencies = [ 26 | "quote", 27 | "syn 1.0.109", 28 | ] 29 | 30 | [[package]] 31 | name = "async-channel" 32 | version = "1.9.0" 33 | source = "registry+https://github.com/rust-lang/crates.io-index" 34 | checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" 35 | dependencies = [ 36 | "concurrent-queue", 37 | "event-listener 2.5.3", 38 | "futures-core", 39 | ] 40 | 41 | [[package]] 42 | name = "async-channel" 43 | version = "2.1.0" 44 | source = "registry+https://github.com/rust-lang/crates.io-index" 45 | checksum = "d37875bd9915b7d67c2f117ea2c30a0989874d0b2cb694fe25403c85763c0c9e" 46 | dependencies = [ 47 | "concurrent-queue", 48 | "event-listener 3.1.0", 49 | "event-listener-strategy", 50 | "futures-core", 51 | "pin-project-lite", 52 | ] 53 | 54 | [[package]] 55 | name = "async-executor" 56 | version = "1.6.0" 57 | source = "registry+https://github.com/rust-lang/crates.io-index" 58 | checksum = "4b0c4a4f319e45986f347ee47fef8bf5e81c9abc3f6f58dc2391439f30df65f0" 59 | dependencies = [ 60 | "async-lock 2.8.0", 61 | "async-task", 62 | "concurrent-queue", 63 | "fastrand 2.0.1", 64 | "futures-lite 1.13.0", 65 | "slab", 66 | ] 67 | 68 | [[package]] 69 | name = "async-global-executor" 70 | version = "2.3.1" 71 | source = "registry+https://github.com/rust-lang/crates.io-index" 72 | checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" 73 | dependencies = [ 74 | "async-channel 1.9.0", 75 | "async-executor", 76 | "async-io", 77 | "async-lock 2.8.0", 78 | "blocking", 79 | "futures-lite 1.13.0", 80 | "once_cell", 81 | ] 82 | 83 | [[package]] 84 | name = "async-io" 85 | version = "1.13.0" 86 | source = "registry+https://github.com/rust-lang/crates.io-index" 87 | checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" 88 | dependencies = [ 89 | "async-lock 2.8.0", 90 | "autocfg", 91 | "cfg-if", 92 | "concurrent-queue", 93 | "futures-lite 1.13.0", 94 | "log", 95 | "parking", 96 | "polling", 97 | "rustix", 98 | "slab", 99 | "socket2 0.4.10", 100 | "waker-fn", 101 | ] 102 | 103 | [[package]] 104 | name = "async-lock" 105 | version = "2.8.0" 106 | source = "registry+https://github.com/rust-lang/crates.io-index" 107 | checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" 108 | dependencies = [ 109 | "event-listener 2.5.3", 110 | ] 111 | 112 | [[package]] 113 | name = "async-lock" 114 | version = "3.1.0" 115 | source = "registry+https://github.com/rust-lang/crates.io-index" 116 | checksum = "deb2ab2aa8a746e221ab826c73f48bc6ba41be6763f0855cb249eb6d154cf1d7" 117 | dependencies = [ 118 | "event-listener 3.1.0", 119 | "event-listener-strategy", 120 | "pin-project-lite", 121 | ] 122 | 123 | [[package]] 124 | name = "async-std" 125 | version = "1.12.0" 126 | source = "registry+https://github.com/rust-lang/crates.io-index" 127 | checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" 128 | dependencies = [ 129 | "async-attributes", 130 | "async-channel 1.9.0", 131 | "async-global-executor", 132 | "async-io", 133 | "async-lock 2.8.0", 134 | "crossbeam-utils", 135 | "futures-channel", 136 | "futures-core", 137 | "futures-io", 138 | "futures-lite 1.13.0", 139 | "gloo-timers", 140 | "kv-log-macro", 141 | "log", 142 | "memchr", 143 | "once_cell", 144 | "pin-project-lite", 145 | "pin-utils", 146 | "slab", 147 | "wasm-bindgen-futures", 148 | ] 149 | 150 | [[package]] 151 | name = "async-task" 152 | version = "4.5.0" 153 | source = "registry+https://github.com/rust-lang/crates.io-index" 154 | checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" 155 | 156 | [[package]] 157 | name = "atomic-waker" 158 | version = "1.1.2" 159 | source = "registry+https://github.com/rust-lang/crates.io-index" 160 | checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" 161 | 162 | [[package]] 163 | name = "autocfg" 164 | version = "1.1.0" 165 | source = "registry+https://github.com/rust-lang/crates.io-index" 166 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 167 | 168 | [[package]] 169 | name = "backtrace" 170 | version = "0.3.69" 171 | source = "registry+https://github.com/rust-lang/crates.io-index" 172 | checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" 173 | dependencies = [ 174 | "addr2line", 175 | "cc", 176 | "cfg-if", 177 | "libc", 178 | "miniz_oxide", 179 | "object", 180 | "rustc-demangle", 181 | ] 182 | 183 | [[package]] 184 | name = "bitflags" 185 | version = "1.3.2" 186 | source = "registry+https://github.com/rust-lang/crates.io-index" 187 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 188 | 189 | [[package]] 190 | name = "blocking" 191 | version = "1.5.1" 192 | source = "registry+https://github.com/rust-lang/crates.io-index" 193 | checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" 194 | dependencies = [ 195 | "async-channel 2.1.0", 196 | "async-lock 3.1.0", 197 | "async-task", 198 | "fastrand 2.0.1", 199 | "futures-io", 200 | "futures-lite 2.0.1", 201 | "piper", 202 | "tracing", 203 | ] 204 | 205 | [[package]] 206 | name = "bumpalo" 207 | version = "3.14.0" 208 | source = "registry+https://github.com/rust-lang/crates.io-index" 209 | checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" 210 | 211 | [[package]] 212 | name = "bytes" 213 | version = "1.5.0" 214 | source = "registry+https://github.com/rust-lang/crates.io-index" 215 | checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" 216 | 217 | [[package]] 218 | name = "cc" 219 | version = "1.0.83" 220 | source = "registry+https://github.com/rust-lang/crates.io-index" 221 | checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" 222 | dependencies = [ 223 | "libc", 224 | ] 225 | 226 | [[package]] 227 | name = "cfg-if" 228 | version = "1.0.0" 229 | source = "registry+https://github.com/rust-lang/crates.io-index" 230 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 231 | 232 | [[package]] 233 | name = "concurrent-queue" 234 | version = "2.3.0" 235 | source = "registry+https://github.com/rust-lang/crates.io-index" 236 | checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" 237 | dependencies = [ 238 | "crossbeam-utils", 239 | ] 240 | 241 | [[package]] 242 | name = "crossbeam-utils" 243 | version = "0.8.16" 244 | source = "registry+https://github.com/rust-lang/crates.io-index" 245 | checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" 246 | dependencies = [ 247 | "cfg-if", 248 | ] 249 | 250 | [[package]] 251 | name = "deluge" 252 | version = "0.2.0" 253 | dependencies = [ 254 | "async-std", 255 | "futures", 256 | "more-asserts", 257 | "num_cpus", 258 | "pin-project", 259 | "tokio", 260 | ] 261 | 262 | [[package]] 263 | name = "errno" 264 | version = "0.3.6" 265 | source = "registry+https://github.com/rust-lang/crates.io-index" 266 | checksum = "7c18ee0ed65a5f1f81cac6b1d213b69c35fa47d4252ad41f1486dbd8226fe36e" 267 | dependencies = [ 268 | "libc", 269 | "windows-sys", 270 | ] 271 | 272 | [[package]] 273 | name = "event-listener" 274 | version = "2.5.3" 275 | source = "registry+https://github.com/rust-lang/crates.io-index" 276 | checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" 277 | 278 | [[package]] 279 | name = "event-listener" 280 | version = "3.1.0" 281 | source = "registry+https://github.com/rust-lang/crates.io-index" 282 | checksum = "d93877bcde0eb80ca09131a08d23f0a5c18a620b01db137dba666d18cd9b30c2" 283 | dependencies = [ 284 | "concurrent-queue", 285 | "parking", 286 | "pin-project-lite", 287 | ] 288 | 289 | [[package]] 290 | name = "event-listener-strategy" 291 | version = "0.3.0" 292 | source = "registry+https://github.com/rust-lang/crates.io-index" 293 | checksum = "d96b852f1345da36d551b9473fa1e2b1eb5c5195585c6c018118bc92a8d91160" 294 | dependencies = [ 295 | "event-listener 3.1.0", 296 | "pin-project-lite", 297 | ] 298 | 299 | [[package]] 300 | name = "fastrand" 301 | version = "1.9.0" 302 | source = "registry+https://github.com/rust-lang/crates.io-index" 303 | checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" 304 | dependencies = [ 305 | "instant", 306 | ] 307 | 308 | [[package]] 309 | name = "fastrand" 310 | version = "2.0.1" 311 | source = "registry+https://github.com/rust-lang/crates.io-index" 312 | checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" 313 | 314 | [[package]] 315 | name = "futures" 316 | version = "0.3.29" 317 | source = "registry+https://github.com/rust-lang/crates.io-index" 318 | checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" 319 | dependencies = [ 320 | "futures-channel", 321 | "futures-core", 322 | "futures-executor", 323 | "futures-io", 324 | "futures-sink", 325 | "futures-task", 326 | "futures-util", 327 | ] 328 | 329 | [[package]] 330 | name = "futures-channel" 331 | version = "0.3.29" 332 | source = "registry+https://github.com/rust-lang/crates.io-index" 333 | checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" 334 | dependencies = [ 335 | "futures-core", 336 | "futures-sink", 337 | ] 338 | 339 | [[package]] 340 | name = "futures-core" 341 | version = "0.3.29" 342 | source = "registry+https://github.com/rust-lang/crates.io-index" 343 | checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" 344 | 345 | [[package]] 346 | name = "futures-executor" 347 | version = "0.3.29" 348 | source = "registry+https://github.com/rust-lang/crates.io-index" 349 | checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" 350 | dependencies = [ 351 | "futures-core", 352 | "futures-task", 353 | "futures-util", 354 | ] 355 | 356 | [[package]] 357 | name = "futures-io" 358 | version = "0.3.29" 359 | source = "registry+https://github.com/rust-lang/crates.io-index" 360 | checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" 361 | 362 | [[package]] 363 | name = "futures-lite" 364 | version = "1.13.0" 365 | source = "registry+https://github.com/rust-lang/crates.io-index" 366 | checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" 367 | dependencies = [ 368 | "fastrand 1.9.0", 369 | "futures-core", 370 | "futures-io", 371 | "memchr", 372 | "parking", 373 | "pin-project-lite", 374 | "waker-fn", 375 | ] 376 | 377 | [[package]] 378 | name = "futures-lite" 379 | version = "2.0.1" 380 | source = "registry+https://github.com/rust-lang/crates.io-index" 381 | checksum = "d3831c2651acb5177cbd83943f3d9c8912c5ad03c76afcc0e9511ba568ec5ebb" 382 | dependencies = [ 383 | "futures-core", 384 | "pin-project-lite", 385 | ] 386 | 387 | [[package]] 388 | name = "futures-macro" 389 | version = "0.3.29" 390 | source = "registry+https://github.com/rust-lang/crates.io-index" 391 | checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" 392 | dependencies = [ 393 | "proc-macro2", 394 | "quote", 395 | "syn 2.0.39", 396 | ] 397 | 398 | [[package]] 399 | name = "futures-sink" 400 | version = "0.3.29" 401 | source = "registry+https://github.com/rust-lang/crates.io-index" 402 | checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" 403 | 404 | [[package]] 405 | name = "futures-task" 406 | version = "0.3.29" 407 | source = "registry+https://github.com/rust-lang/crates.io-index" 408 | checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" 409 | 410 | [[package]] 411 | name = "futures-util" 412 | version = "0.3.29" 413 | source = "registry+https://github.com/rust-lang/crates.io-index" 414 | checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" 415 | dependencies = [ 416 | "futures-channel", 417 | "futures-core", 418 | "futures-io", 419 | "futures-macro", 420 | "futures-sink", 421 | "futures-task", 422 | "memchr", 423 | "pin-project-lite", 424 | "pin-utils", 425 | "slab", 426 | ] 427 | 428 | [[package]] 429 | name = "gimli" 430 | version = "0.28.0" 431 | source = "registry+https://github.com/rust-lang/crates.io-index" 432 | checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" 433 | 434 | [[package]] 435 | name = "gloo-timers" 436 | version = "0.2.6" 437 | source = "registry+https://github.com/rust-lang/crates.io-index" 438 | checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" 439 | dependencies = [ 440 | "futures-channel", 441 | "futures-core", 442 | "js-sys", 443 | "wasm-bindgen", 444 | ] 445 | 446 | [[package]] 447 | name = "hermit-abi" 448 | version = "0.3.3" 449 | source = "registry+https://github.com/rust-lang/crates.io-index" 450 | checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" 451 | 452 | [[package]] 453 | name = "instant" 454 | version = "0.1.12" 455 | source = "registry+https://github.com/rust-lang/crates.io-index" 456 | checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" 457 | dependencies = [ 458 | "cfg-if", 459 | ] 460 | 461 | [[package]] 462 | name = "io-lifetimes" 463 | version = "1.0.11" 464 | source = "registry+https://github.com/rust-lang/crates.io-index" 465 | checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" 466 | dependencies = [ 467 | "hermit-abi", 468 | "libc", 469 | "windows-sys", 470 | ] 471 | 472 | [[package]] 473 | name = "js-sys" 474 | version = "0.3.65" 475 | source = "registry+https://github.com/rust-lang/crates.io-index" 476 | checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" 477 | dependencies = [ 478 | "wasm-bindgen", 479 | ] 480 | 481 | [[package]] 482 | name = "kv-log-macro" 483 | version = "1.0.7" 484 | source = "registry+https://github.com/rust-lang/crates.io-index" 485 | checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" 486 | dependencies = [ 487 | "log", 488 | ] 489 | 490 | [[package]] 491 | name = "libc" 492 | version = "0.2.150" 493 | source = "registry+https://github.com/rust-lang/crates.io-index" 494 | checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" 495 | 496 | [[package]] 497 | name = "linux-raw-sys" 498 | version = "0.3.8" 499 | source = "registry+https://github.com/rust-lang/crates.io-index" 500 | checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" 501 | 502 | [[package]] 503 | name = "lock_api" 504 | version = "0.4.11" 505 | source = "registry+https://github.com/rust-lang/crates.io-index" 506 | checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" 507 | dependencies = [ 508 | "autocfg", 509 | "scopeguard", 510 | ] 511 | 512 | [[package]] 513 | name = "log" 514 | version = "0.4.20" 515 | source = "registry+https://github.com/rust-lang/crates.io-index" 516 | checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" 517 | dependencies = [ 518 | "value-bag", 519 | ] 520 | 521 | [[package]] 522 | name = "memchr" 523 | version = "2.6.4" 524 | source = "registry+https://github.com/rust-lang/crates.io-index" 525 | checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" 526 | 527 | [[package]] 528 | name = "miniz_oxide" 529 | version = "0.7.1" 530 | source = "registry+https://github.com/rust-lang/crates.io-index" 531 | checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" 532 | dependencies = [ 533 | "adler", 534 | ] 535 | 536 | [[package]] 537 | name = "mio" 538 | version = "0.8.9" 539 | source = "registry+https://github.com/rust-lang/crates.io-index" 540 | checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" 541 | dependencies = [ 542 | "libc", 543 | "wasi", 544 | "windows-sys", 545 | ] 546 | 547 | [[package]] 548 | name = "more-asserts" 549 | version = "0.3.1" 550 | source = "registry+https://github.com/rust-lang/crates.io-index" 551 | checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" 552 | 553 | [[package]] 554 | name = "num_cpus" 555 | version = "1.16.0" 556 | source = "registry+https://github.com/rust-lang/crates.io-index" 557 | checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" 558 | dependencies = [ 559 | "hermit-abi", 560 | "libc", 561 | ] 562 | 563 | [[package]] 564 | name = "object" 565 | version = "0.32.1" 566 | source = "registry+https://github.com/rust-lang/crates.io-index" 567 | checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" 568 | dependencies = [ 569 | "memchr", 570 | ] 571 | 572 | [[package]] 573 | name = "once_cell" 574 | version = "1.18.0" 575 | source = "registry+https://github.com/rust-lang/crates.io-index" 576 | checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" 577 | 578 | [[package]] 579 | name = "parking" 580 | version = "2.2.0" 581 | source = "registry+https://github.com/rust-lang/crates.io-index" 582 | checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" 583 | 584 | [[package]] 585 | name = "parking_lot" 586 | version = "0.12.1" 587 | source = "registry+https://github.com/rust-lang/crates.io-index" 588 | checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" 589 | dependencies = [ 590 | "lock_api", 591 | "parking_lot_core", 592 | ] 593 | 594 | [[package]] 595 | name = "parking_lot_core" 596 | version = "0.9.9" 597 | source = "registry+https://github.com/rust-lang/crates.io-index" 598 | checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" 599 | dependencies = [ 600 | "cfg-if", 601 | "libc", 602 | "redox_syscall", 603 | "smallvec", 604 | "windows-targets", 605 | ] 606 | 607 | [[package]] 608 | name = "pin-project" 609 | version = "1.1.3" 610 | source = "registry+https://github.com/rust-lang/crates.io-index" 611 | checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" 612 | dependencies = [ 613 | "pin-project-internal", 614 | ] 615 | 616 | [[package]] 617 | name = "pin-project-internal" 618 | version = "1.1.3" 619 | source = "registry+https://github.com/rust-lang/crates.io-index" 620 | checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" 621 | dependencies = [ 622 | "proc-macro2", 623 | "quote", 624 | "syn 2.0.39", 625 | ] 626 | 627 | [[package]] 628 | name = "pin-project-lite" 629 | version = "0.2.13" 630 | source = "registry+https://github.com/rust-lang/crates.io-index" 631 | checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" 632 | 633 | [[package]] 634 | name = "pin-utils" 635 | version = "0.1.0" 636 | source = "registry+https://github.com/rust-lang/crates.io-index" 637 | checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" 638 | 639 | [[package]] 640 | name = "piper" 641 | version = "0.2.1" 642 | source = "registry+https://github.com/rust-lang/crates.io-index" 643 | checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" 644 | dependencies = [ 645 | "atomic-waker", 646 | "fastrand 2.0.1", 647 | "futures-io", 648 | ] 649 | 650 | [[package]] 651 | name = "polling" 652 | version = "2.8.0" 653 | source = "registry+https://github.com/rust-lang/crates.io-index" 654 | checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" 655 | dependencies = [ 656 | "autocfg", 657 | "bitflags", 658 | "cfg-if", 659 | "concurrent-queue", 660 | "libc", 661 | "log", 662 | "pin-project-lite", 663 | "windows-sys", 664 | ] 665 | 666 | [[package]] 667 | name = "proc-macro2" 668 | version = "1.0.69" 669 | source = "registry+https://github.com/rust-lang/crates.io-index" 670 | checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" 671 | dependencies = [ 672 | "unicode-ident", 673 | ] 674 | 675 | [[package]] 676 | name = "quote" 677 | version = "1.0.33" 678 | source = "registry+https://github.com/rust-lang/crates.io-index" 679 | checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" 680 | dependencies = [ 681 | "proc-macro2", 682 | ] 683 | 684 | [[package]] 685 | name = "redox_syscall" 686 | version = "0.4.1" 687 | source = "registry+https://github.com/rust-lang/crates.io-index" 688 | checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" 689 | dependencies = [ 690 | "bitflags", 691 | ] 692 | 693 | [[package]] 694 | name = "rustc-demangle" 695 | version = "0.1.23" 696 | source = "registry+https://github.com/rust-lang/crates.io-index" 697 | checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" 698 | 699 | [[package]] 700 | name = "rustix" 701 | version = "0.37.27" 702 | source = "registry+https://github.com/rust-lang/crates.io-index" 703 | checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" 704 | dependencies = [ 705 | "bitflags", 706 | "errno", 707 | "io-lifetimes", 708 | "libc", 709 | "linux-raw-sys", 710 | "windows-sys", 711 | ] 712 | 713 | [[package]] 714 | name = "scopeguard" 715 | version = "1.2.0" 716 | source = "registry+https://github.com/rust-lang/crates.io-index" 717 | checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 718 | 719 | [[package]] 720 | name = "signal-hook-registry" 721 | version = "1.4.1" 722 | source = "registry+https://github.com/rust-lang/crates.io-index" 723 | checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" 724 | dependencies = [ 725 | "libc", 726 | ] 727 | 728 | [[package]] 729 | name = "slab" 730 | version = "0.4.9" 731 | source = "registry+https://github.com/rust-lang/crates.io-index" 732 | checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" 733 | dependencies = [ 734 | "autocfg", 735 | ] 736 | 737 | [[package]] 738 | name = "smallvec" 739 | version = "1.11.2" 740 | source = "registry+https://github.com/rust-lang/crates.io-index" 741 | checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" 742 | 743 | [[package]] 744 | name = "socket2" 745 | version = "0.4.10" 746 | source = "registry+https://github.com/rust-lang/crates.io-index" 747 | checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" 748 | dependencies = [ 749 | "libc", 750 | "winapi", 751 | ] 752 | 753 | [[package]] 754 | name = "socket2" 755 | version = "0.5.5" 756 | source = "registry+https://github.com/rust-lang/crates.io-index" 757 | checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" 758 | dependencies = [ 759 | "libc", 760 | "windows-sys", 761 | ] 762 | 763 | [[package]] 764 | name = "syn" 765 | version = "1.0.109" 766 | source = "registry+https://github.com/rust-lang/crates.io-index" 767 | checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" 768 | dependencies = [ 769 | "proc-macro2", 770 | "quote", 771 | "unicode-ident", 772 | ] 773 | 774 | [[package]] 775 | name = "syn" 776 | version = "2.0.39" 777 | source = "registry+https://github.com/rust-lang/crates.io-index" 778 | checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" 779 | dependencies = [ 780 | "proc-macro2", 781 | "quote", 782 | "unicode-ident", 783 | ] 784 | 785 | [[package]] 786 | name = "tokio" 787 | version = "1.34.0" 788 | source = "registry+https://github.com/rust-lang/crates.io-index" 789 | checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" 790 | dependencies = [ 791 | "backtrace", 792 | "bytes", 793 | "libc", 794 | "mio", 795 | "num_cpus", 796 | "parking_lot", 797 | "pin-project-lite", 798 | "signal-hook-registry", 799 | "socket2 0.5.5", 800 | "tokio-macros", 801 | "windows-sys", 802 | ] 803 | 804 | [[package]] 805 | name = "tokio-macros" 806 | version = "2.2.0" 807 | source = "registry+https://github.com/rust-lang/crates.io-index" 808 | checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" 809 | dependencies = [ 810 | "proc-macro2", 811 | "quote", 812 | "syn 2.0.39", 813 | ] 814 | 815 | [[package]] 816 | name = "tracing" 817 | version = "0.1.40" 818 | source = "registry+https://github.com/rust-lang/crates.io-index" 819 | checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" 820 | dependencies = [ 821 | "pin-project-lite", 822 | "tracing-core", 823 | ] 824 | 825 | [[package]] 826 | name = "tracing-core" 827 | version = "0.1.32" 828 | source = "registry+https://github.com/rust-lang/crates.io-index" 829 | checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" 830 | 831 | [[package]] 832 | name = "unicode-ident" 833 | version = "1.0.12" 834 | source = "registry+https://github.com/rust-lang/crates.io-index" 835 | checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" 836 | 837 | [[package]] 838 | name = "value-bag" 839 | version = "1.4.2" 840 | source = "registry+https://github.com/rust-lang/crates.io-index" 841 | checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" 842 | 843 | [[package]] 844 | name = "waker-fn" 845 | version = "1.1.1" 846 | source = "registry+https://github.com/rust-lang/crates.io-index" 847 | checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" 848 | 849 | [[package]] 850 | name = "wasi" 851 | version = "0.11.0+wasi-snapshot-preview1" 852 | source = "registry+https://github.com/rust-lang/crates.io-index" 853 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 854 | 855 | [[package]] 856 | name = "wasm-bindgen" 857 | version = "0.2.88" 858 | source = "registry+https://github.com/rust-lang/crates.io-index" 859 | checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" 860 | dependencies = [ 861 | "cfg-if", 862 | "wasm-bindgen-macro", 863 | ] 864 | 865 | [[package]] 866 | name = "wasm-bindgen-backend" 867 | version = "0.2.88" 868 | source = "registry+https://github.com/rust-lang/crates.io-index" 869 | checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" 870 | dependencies = [ 871 | "bumpalo", 872 | "log", 873 | "once_cell", 874 | "proc-macro2", 875 | "quote", 876 | "syn 2.0.39", 877 | "wasm-bindgen-shared", 878 | ] 879 | 880 | [[package]] 881 | name = "wasm-bindgen-futures" 882 | version = "0.4.38" 883 | source = "registry+https://github.com/rust-lang/crates.io-index" 884 | checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" 885 | dependencies = [ 886 | "cfg-if", 887 | "js-sys", 888 | "wasm-bindgen", 889 | "web-sys", 890 | ] 891 | 892 | [[package]] 893 | name = "wasm-bindgen-macro" 894 | version = "0.2.88" 895 | source = "registry+https://github.com/rust-lang/crates.io-index" 896 | checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" 897 | dependencies = [ 898 | "quote", 899 | "wasm-bindgen-macro-support", 900 | ] 901 | 902 | [[package]] 903 | name = "wasm-bindgen-macro-support" 904 | version = "0.2.88" 905 | source = "registry+https://github.com/rust-lang/crates.io-index" 906 | checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" 907 | dependencies = [ 908 | "proc-macro2", 909 | "quote", 910 | "syn 2.0.39", 911 | "wasm-bindgen-backend", 912 | "wasm-bindgen-shared", 913 | ] 914 | 915 | [[package]] 916 | name = "wasm-bindgen-shared" 917 | version = "0.2.88" 918 | source = "registry+https://github.com/rust-lang/crates.io-index" 919 | checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" 920 | 921 | [[package]] 922 | name = "web-sys" 923 | version = "0.3.65" 924 | source = "registry+https://github.com/rust-lang/crates.io-index" 925 | checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" 926 | dependencies = [ 927 | "js-sys", 928 | "wasm-bindgen", 929 | ] 930 | 931 | [[package]] 932 | name = "winapi" 933 | version = "0.3.9" 934 | source = "registry+https://github.com/rust-lang/crates.io-index" 935 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 936 | dependencies = [ 937 | "winapi-i686-pc-windows-gnu", 938 | "winapi-x86_64-pc-windows-gnu", 939 | ] 940 | 941 | [[package]] 942 | name = "winapi-i686-pc-windows-gnu" 943 | version = "0.4.0" 944 | source = "registry+https://github.com/rust-lang/crates.io-index" 945 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 946 | 947 | [[package]] 948 | name = "winapi-x86_64-pc-windows-gnu" 949 | version = "0.4.0" 950 | source = "registry+https://github.com/rust-lang/crates.io-index" 951 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 952 | 953 | [[package]] 954 | name = "windows-sys" 955 | version = "0.48.0" 956 | source = "registry+https://github.com/rust-lang/crates.io-index" 957 | checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" 958 | dependencies = [ 959 | "windows-targets", 960 | ] 961 | 962 | [[package]] 963 | name = "windows-targets" 964 | version = "0.48.5" 965 | source = "registry+https://github.com/rust-lang/crates.io-index" 966 | checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" 967 | dependencies = [ 968 | "windows_aarch64_gnullvm", 969 | "windows_aarch64_msvc", 970 | "windows_i686_gnu", 971 | "windows_i686_msvc", 972 | "windows_x86_64_gnu", 973 | "windows_x86_64_gnullvm", 974 | "windows_x86_64_msvc", 975 | ] 976 | 977 | [[package]] 978 | name = "windows_aarch64_gnullvm" 979 | version = "0.48.5" 980 | source = "registry+https://github.com/rust-lang/crates.io-index" 981 | checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" 982 | 983 | [[package]] 984 | name = "windows_aarch64_msvc" 985 | version = "0.48.5" 986 | source = "registry+https://github.com/rust-lang/crates.io-index" 987 | checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" 988 | 989 | [[package]] 990 | name = "windows_i686_gnu" 991 | version = "0.48.5" 992 | source = "registry+https://github.com/rust-lang/crates.io-index" 993 | checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" 994 | 995 | [[package]] 996 | name = "windows_i686_msvc" 997 | version = "0.48.5" 998 | source = "registry+https://github.com/rust-lang/crates.io-index" 999 | checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" 1000 | 1001 | [[package]] 1002 | name = "windows_x86_64_gnu" 1003 | version = "0.48.5" 1004 | source = "registry+https://github.com/rust-lang/crates.io-index" 1005 | checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" 1006 | 1007 | [[package]] 1008 | name = "windows_x86_64_gnullvm" 1009 | version = "0.48.5" 1010 | source = "registry+https://github.com/rust-lang/crates.io-index" 1011 | checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" 1012 | 1013 | [[package]] 1014 | name = "windows_x86_64_msvc" 1015 | version = "0.48.5" 1016 | source = "registry+https://github.com/rust-lang/crates.io-index" 1017 | checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" 1018 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "deluge" 3 | version = "0.2.1" 4 | authors = ["Michał Kawalec "] 5 | license = "MPL-2.0" 6 | description = """ 7 | A highly concurrent stream library driving the underlying futures 8 | either concurrently or in parallel to process streaming operations 9 | as quickly as possible. 10 | """ 11 | readme = "README.md" 12 | homepage = "https://github.com/mkawalec/deluge" 13 | repository = "https://github.com/mkawalec/deluge" 14 | documentation = "https://docs.rs/deluge/latest/deluge/" 15 | keywords = ["futures", "async", "deluge"] 16 | categories = ["asynchronous"] 17 | edition = "2021" 18 | exclude = [ 19 | ".github/*", 20 | ".gitignore", 21 | "Makefile", 22 | "images/*", 23 | ] 24 | 25 | [features] 26 | default = ["tokio"] 27 | async-runtime = [] 28 | tokio = ["dep:tokio", "async-runtime"] 29 | async-std = ["dep:async-std", "async-runtime"] 30 | 31 | [dependencies] 32 | async-std = { version = "1.12", features = ["attributes"], optional = true } 33 | futures = "0.3" 34 | num_cpus = "1.13" 35 | pin-project = "1.0" 36 | tokio = { version = "1.20", features = ["sync"], optional = true } 37 | 38 | [dev-dependencies] 39 | more-asserts = "0.3" 40 | tokio = { version = "1.20", features = ["full"] } 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | .PHONY=setup 3 | setup: 4 | cargo install cargo-sort 5 | rustup component add rustfmt clippy 6 | 7 | .PHONY=fmt 8 | fmt: setup 9 | cargo sort -w 10 | cargo fmt 11 | cargo clippy --all-targets --features tokio --profile dev -- -D warnings 12 | cargo clippy --all-targets --features tokio --profile release -- -D warnings 13 | cargo clippy --all-targets --no-default-features --features async-std --profile dev -- -D warnings 14 | cargo clippy --all-targets --no-default-features --features async-std --profile release -- -D warnings 15 | 16 | .PHONY=fmt-fix 17 | fmt-fix: setup 18 | cargo sort -w 19 | cargo fmt 20 | cargo clippy --fix --all-targets --features tokio --profile dev -- -D warnings 21 | cargo clippy --fix --all-targets --features tokio --profile release -- -D warnings 22 | cargo clippy --fix --all-targets --no-default-features --features async-std --profile dev -- -D warnings 23 | cargo clippy --fix --all-targets --no-default-features --features async-std --profile release -- -D warnings 24 | 25 | .PHONY=test 26 | test: 27 | cargo test --features tokio 28 | cargo test --no-default-features --features async-std -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deluge is (not) a Stream 2 | 3 |
4 |

5 | Crate Info 6 | API Docs 7 | Rustc Version 1.64.0+ 8 |

9 | 10 |
11 | 12 | `Deluge` builds on top of `Stream` to provide stream operations that are parallel or concurrent by default. 13 | It allows it's user to have an ordered stream of futures that are evaluated concurrently, with all the complexity hidden inside `Deluge` itself. 14 | We achieve this by working one level higher than a stream. 15 | Instead of returning values that might materialize at some point in the future, we immediately return an iterator of unevaluated futures which can then be evaluated by the collector. 16 | 17 | The animation below shows an example of mapping over a highly concurrent six element collection. 📘 indicates the time it takes for an underlying element to become available, while 📗 the time it takes to apply a mapped operation. 18 | 19 | ![Example of processing using Deluge and Streams](./images/process.gif) 20 | 21 | **This library is still experimental, use at your own risk** 22 | 23 | ### Design decisions 24 | 25 | This is an opinionated library that puts ease of use and external simplicity at the forefront. 26 | Operations that apply to individual elements like maps and filters **do not** allocate. 27 | They simply wrap each element in another future but they do not control the way these processed elements are evaluated. 28 | It is the collector that controls the evaluation strategy. 29 | At the moment there are two basic collectors supplied: a concurrent and a parallel one. 30 | Where there is a decision between performance and ease of use to be made, we are likely to fall on the side of ease of use. 31 | 32 | The concurrent collector accepts an optional concurrency limit. 33 | If it is specified, at most the number of futures equal to that limit will be evaluated at once. 34 | 35 | ```rust 36 | let result = [1, 2, 3, 4] 37 | .into_deluge() 38 | .map(|x| async move { x * 2 }) 39 | .collect::>(None) 40 | .await; 41 | 42 | assert_eq!(vec![2, 4, 6, 8], result); 43 | ``` 44 | 45 | The parallel collector spawns a number of workers. 46 | If a number of workers is not specified, it will default to the number of logical cpus, if the concurrency limit is not specified each worker will default to `total_futures_to_evaluate / number_of_workers`. 47 | Note that you need to enable either a `tokio` or `async-std` feature to support parallel collectors. 48 | 49 | ```rust 50 | let result = (0..150) 51 | .into_deluge() 52 | .map(|idx| async move { 53 | tokio::time::sleep(Duration::from_millis(50)).await; 54 | idx 55 | }) 56 | .collect_par::>(10, None) 57 | .await; 58 | 59 | assert_eq!(result.len(), 150); 60 | ``` 61 | 62 | Please take a look at [the tests](https://github.com/mkawalec/deluge/blob/main/src/deluge_ext.rs) for more examples of using the library. 63 | 64 | ### Converting to a Stream 65 | 66 | Both `collect` and `collect_par` implement a `Stream`. 67 | If you want to use any of the existing `Stream` extensions or functions that are missing from `Deluge`, `collect` and then use it as any other stream. 68 | Everything that is before collection in `Deluge` gets the benefits of running the futures in parallel. 69 | 70 | ### Questions 71 | 72 | #### I would want to add another operation to DelugeExt. Should I? 73 | 74 | By all means. 75 | Please do not allocate on the heap in operations that transform individual elements. 76 | Any operation you would find useful is fair game, contributions are welcome. 77 | 78 | #### I found a performance improvement, should I submit a PR? 79 | 80 | Absolutely! 81 | As long the API exposed to the users does not get more complex, the number of allocations does not go up and intermediate memory usage does not increase. 82 | Please open an issue first if you feel that breaking any of the above rules is absolutely neccessary and we will discuss. 83 | -------------------------------------------------------------------------------- /images/animation.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | My first three.js app 6 | 55 | 56 | 57 |
58 |

59 |
60 | 61 |
62 |

63 |
64 | 65 | 134 | 135 | -------------------------------------------------------------------------------- /images/process.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mkawalec/deluge/1d6a39937dbb07e75f3d94ecb9742b63c3805373/images/process.gif -------------------------------------------------------------------------------- /src/deluge.rs: -------------------------------------------------------------------------------- 1 | use std::future::Future; 2 | 3 | /// A stream of unevaluated futures eventually returning an element of the stream 4 | /// 5 | /// An executor such as `collect` or `collect_par` controls how these futures are evaluated. 6 | /// If a `None` is returned for a given element of a collection, it means that 7 | /// element was filtered out earlier in the processing chain and should be omitted. 8 | /// 9 | /// If `None` is returned from the call to `next`, the Deluge has ran out of items to provide. 10 | /// Calling `next` again will be unsafe and may lead to panics. 11 | pub trait Deluge { 12 | type Item: Send; 13 | type Output<'x>: Future> + 'x 14 | where 15 | Self: 'x; 16 | 17 | fn next(&self) -> Option>; 18 | } 19 | -------------------------------------------------------------------------------- /src/deluge_ext.rs: -------------------------------------------------------------------------------- 1 | use std::default::Default; 2 | use std::future::Future; 3 | 4 | use crate::deluge::Deluge; 5 | use crate::ops::*; 6 | 7 | impl DelugeExt for T where T: Deluge {} 8 | 9 | /// Exposes easy to use Deluge operations. **This should be your first step** 10 | pub trait DelugeExt: Deluge { 11 | /// Resolves to true if all of the calls to `F` return true 12 | /// for each element of the deluge. 13 | /// Evaluates the elements concurrently and short circuits evaluation 14 | /// on a first failure. 15 | /// 16 | /// # Examples 17 | /// 18 | /// ``` 19 | /// use deluge::*; 20 | /// 21 | /// # futures::executor::block_on(async { 22 | /// let result = [1, 2, 3, 4] 23 | /// .into_deluge() 24 | /// .all(None, |x| async move { x < 5 }) 25 | /// .await; 26 | /// 27 | /// assert!(result); 28 | /// 29 | /// let result = [1, 2, 3, 4] 30 | /// .into_deluge() 31 | /// .all(None, |x| async move { x < 3 }) 32 | /// .await; 33 | /// 34 | /// assert!(!result); 35 | /// # }); 36 | /// ``` 37 | fn all<'a, Fut, F>(self, concurrency: impl Into>, f: F) -> All<'a, Self, Fut, F> 38 | where 39 | F: Fn(Self::Item) -> Fut + Send + 'a, 40 | Fut: Future + Send, 41 | Self: Sized, 42 | { 43 | All::new(self, concurrency, f) 44 | } 45 | 46 | /// A parallel version of `DelugeExt::all`. 47 | /// Resolves to true if all of the calls to `F` return true 48 | /// for each element of the deluge. 49 | /// Evaluates the elements in parallel and short circuits evaluation 50 | /// on a first failure. 51 | /// 52 | /// # Examples 53 | /// 54 | /// ``` 55 | /// use deluge::*; 56 | /// 57 | /// # futures::executor::block_on(async { 58 | /// let result = [1, 2, 3, 4] 59 | /// .into_deluge() 60 | /// .all_par(None, None, |x| async move { x < 5 }) 61 | /// .await; 62 | /// 63 | /// assert!(result); 64 | /// 65 | /// let result = [1, 2, 3, 4] 66 | /// .into_deluge() 67 | /// .all_par(None, None, |x| async move { x < 3 }) 68 | /// .await; 69 | /// 70 | /// assert!(!result); 71 | /// # }); 72 | /// ``` 73 | fn all_par<'a, Fut, F>( 74 | self, 75 | worker_count: impl Into>, 76 | worker_concurrency: impl Into>, 77 | f: F, 78 | ) -> AllPar<'a, Self, Fut, F> 79 | where 80 | F: Fn(Self::Item) -> Fut + Send + 'a, 81 | Fut: Future + Send, 82 | Self: Sized, 83 | { 84 | AllPar::new(self, worker_count, worker_concurrency, f) 85 | } 86 | 87 | /// Resolves to true if any of the calls to `F` return true 88 | /// for any element of the deluge. 89 | /// Evaluates the elements concurrently and short circuits evaluation 90 | /// on a successful result. 91 | /// 92 | /// # Examples 93 | /// 94 | /// ``` 95 | /// use deluge::*; 96 | /// 97 | /// # futures::executor::block_on(async { 98 | /// let result = [1, 2, 3, 4] 99 | /// .into_deluge() 100 | /// .any(None, |x| async move { x == 4 }) 101 | /// .await; 102 | /// 103 | /// assert!(result); 104 | /// 105 | /// let result = [1, 2, 3, 4] 106 | /// .into_deluge() 107 | /// .any(None, |x| async move { x > 10 }) 108 | /// .await; 109 | /// 110 | /// assert!(!result); 111 | /// # }); 112 | /// ``` 113 | fn any<'a, Fut, F>(self, concurrency: impl Into>, f: F) -> Any<'a, Self, Fut, F> 114 | where 115 | F: Fn(Self::Item) -> Fut + Send + 'a, 116 | Fut: Future + Send, 117 | Self: Sized, 118 | { 119 | Any::new(self, concurrency, f) 120 | } 121 | 122 | /// A parallel version of `DelugeExt::any`. 123 | /// Resolves to true if any of the calls to `F` return true 124 | /// for any element of the deluge. 125 | /// Evaluates the elements in parallel and short circuits evaluation 126 | /// on a successful result. 127 | /// 128 | /// # Examples 129 | /// 130 | /// ``` 131 | /// use deluge::*; 132 | /// 133 | /// # futures::executor::block_on(async { 134 | /// let result = [1, 2, 3, 4] 135 | /// .into_deluge() 136 | /// .any_par(None, None, |x| async move { x == 4 }) 137 | /// .await; 138 | /// 139 | /// assert!(result); 140 | /// 141 | /// let result = [1, 2, 3, 4] 142 | /// .into_deluge() 143 | /// .any_par(None, None, |x| async move { x > 10 }) 144 | /// .await; 145 | /// 146 | /// assert!(!result); 147 | /// # }); 148 | /// ``` 149 | fn any_par<'a, Fut, F>( 150 | self, 151 | worker_count: impl Into>, 152 | worker_concurrency: impl Into>, 153 | f: F, 154 | ) -> AnyPar<'a, Self, Fut, F> 155 | where 156 | F: Fn(Self::Item) -> Fut + Send + 'a, 157 | Fut: Future + Send, 158 | Self: Sized, 159 | { 160 | AnyPar::new(self, worker_count, worker_concurrency, f) 161 | } 162 | 163 | /// Chains two deluges together 164 | /// 165 | /// # Examples 166 | /// 167 | /// ``` 168 | /// use deluge::*; 169 | /// 170 | /// # futures::executor::block_on(async { 171 | /// let result = [1, 2, 3, 4] 172 | /// .into_deluge() 173 | /// .chain([5, 6, 7, 8].into_deluge()) 174 | /// .collect::>(None) 175 | /// .await; 176 | /// 177 | /// assert_eq!(result.len(), 8); 178 | /// assert_eq!(result, vec![1, 2, 3, 4, 5, 6, 7, 8]); 179 | /// # }) 180 | /// ``` 181 | fn chain<'a, Del2>(self, deluge2: Del2) -> Chain<'a, Self, Del2> 182 | where 183 | Del2: for<'x> Deluge = Self::Output<'x>> + 'static, 184 | Self: Sized, 185 | { 186 | Chain::new(self, deluge2) 187 | } 188 | 189 | /// Consumes all the items in a deluge and returns 190 | /// the number of elements that were observed. 191 | /// 192 | /// # Examples 193 | /// 194 | /// ``` 195 | /// use deluge::*; 196 | /// 197 | /// # futures::executor::block_on(async { 198 | /// let result = [1, 2, 3, 4] 199 | /// .into_deluge() 200 | /// .count(); 201 | /// 202 | /// assert_eq!(result, 4); 203 | /// # }) 204 | /// ``` 205 | fn count(self) -> usize 206 | where 207 | Self: Sized, 208 | { 209 | count(self) 210 | } 211 | 212 | /// Transforms each element by applying an asynchronous function `f` to it 213 | /// 214 | /// # Examples 215 | /// 216 | /// ``` 217 | /// use deluge::*; 218 | /// 219 | /// # futures::executor::block_on(async { 220 | /// let result = [1, 2, 3, 4] 221 | /// .into_deluge() 222 | /// .map(|x| async move { x * 2 }) 223 | /// .collect::>(None) 224 | /// .await; 225 | /// 226 | /// assert_eq!(vec![2, 4, 6, 8], result); 227 | /// # }); 228 | /// ``` 229 | fn map(self, f: F) -> Map 230 | where 231 | F: Fn(Self::Item) -> Fut + Send, 232 | Fut: Future + Send, 233 | Self: Sized, 234 | { 235 | Map::new(self, f) 236 | } 237 | 238 | /// Leaves the elements for which `f` returns a promise evaluating to `true`. 239 | /// 240 | /// # WARNING 241 | /// 242 | /// Currently has [a breaking bug](https://github.com/mkawalec/deluge/issues/1). 243 | /// 244 | /// # Examples 245 | /// 246 | // ``` 247 | // use deluge::*; 248 | // 249 | // # futures::executor::block_on(async { 250 | // let result = (0..10).into_deluge() 251 | // .filter(|x| async move { x % 2 == 0 }) 252 | // .collect::>(None) 253 | // .await; 254 | // 255 | // assert_eq!(vec![0, 2, 4, 6, 8], result); 256 | // # }); 257 | // 258 | // ``` 259 | // 27.10.2022: Commented out filter until the issue#1 is solved 260 | /*fn filter<'a, F>(self, f: F) -> Filter<'a, Self, F> 261 | where 262 | for<'b> F: XFn<'b, Self::Item, bool> + Send + 'b, 263 | Self: Sized, 264 | { 265 | Filter::new(self, f) 266 | }*/ 267 | 268 | /// Filters out elements for which a function returns `None`, 269 | /// substitutes the elements for the ones there it returns `Some(new_value)`. 270 | /// 271 | /// # Examples 272 | /// 273 | /// ``` 274 | /// use deluge::*; 275 | /// 276 | /// # futures::executor::block_on(async { 277 | /// let result = (0..10).into_deluge() 278 | /// .filter_map(|x| async move { if x % 2 == 0 { 279 | /// Some(x.to_string()) 280 | /// } else { 281 | /// None 282 | /// } 283 | /// }) 284 | /// .collect::>(None) 285 | /// .await; 286 | /// 287 | /// assert_eq!(vec![0.to_string(), 2.to_string(), 4.to_string(), 6.to_string(), 8.to_string()], result); 288 | /// # }); 289 | /// 290 | /// ``` 291 | fn filter_map(self, f: F) -> FilterMap 292 | where 293 | F: Fn(Self::Item) -> Fut + Send, 294 | Fut: Future + Send, 295 | Self: Sized, 296 | { 297 | FilterMap::new(self, f) 298 | } 299 | 300 | /// Returns the first element of the input deluge and then finishes 301 | /// 302 | /// # Examples 303 | /// ``` 304 | /// use deluge::*; 305 | /// 306 | /// # futures::executor::block_on(async { 307 | /// let result = (0..10).into_deluge() 308 | /// .first() 309 | /// .collect::>(None) 310 | /// .await; 311 | /// 312 | /// assert_eq!(vec![0], result); 313 | /// # }); 314 | /// 315 | /// ``` 316 | fn first(self) -> First 317 | where 318 | Self: Sized, 319 | { 320 | First::new(self) 321 | } 322 | 323 | /// Concurrently accummulates values in the accummulator. The degree of concurrency 324 | /// can either be unlimited (the default) or limited depending on the requirements. 325 | /// 326 | /// # Examples 327 | /// 328 | /// Unlimited concurrency: 329 | /// 330 | /// ``` 331 | /// use deluge::*; 332 | /// 333 | /// # futures::executor::block_on(async { 334 | /// let result = (0..100).into_deluge() 335 | /// .fold(None, 0, |acc, x| async move { acc + x }) 336 | /// .await; 337 | /// 338 | /// assert_eq!(result, 4950); 339 | /// # }); 340 | /// ``` 341 | /// 342 | /// Concurrency limited to at most ten futures evaluated at once: 343 | /// 344 | /// ``` 345 | /// use deluge::*; 346 | /// 347 | /// # futures::executor::block_on(async { 348 | /// let result = (0..100).into_deluge() 349 | /// .fold(10, 0, |acc, x| async move { acc + x }) 350 | /// .await; 351 | /// 352 | /// assert_eq!(result, 4950); 353 | /// # }); 354 | /// ``` 355 | fn fold( 356 | self, 357 | concurrency: impl Into>, 358 | acc: Acc, 359 | f: F, 360 | ) -> Fold 361 | where 362 | F: FnMut(Acc, Self::Item) -> Fut + Send, 363 | Fut: Future + Send, 364 | Self: Sized, 365 | { 366 | Fold::new(self, concurrency, acc, f) 367 | } 368 | 369 | /// Accummulates values in an accummulator with futures evaluated in parallel. 370 | /// The number of workers spawned and concurrency for each worker can be controlled. 371 | /// By default the number of workers equals the number of logical cpus 372 | /// and concurrency for each worker is the total number of futures to evaluate 373 | /// divided by the number of available workers. 374 | /// 375 | /// # Examples 376 | /// 377 | /// ``` 378 | /// use deluge::*; 379 | /// 380 | /// # futures::executor::block_on(async { 381 | /// let result = (0..100).into_deluge() 382 | /// .fold_par(None, None, 0, |acc, x| async move { acc + x }) 383 | /// .await; 384 | /// 385 | /// assert_eq!(result, 4950); 386 | /// # }); 387 | /// ``` 388 | #[cfg(feature = "async-runtime")] 389 | fn fold_par<'a, Acc, F, Fut>( 390 | self, 391 | worker_count: impl Into>, 392 | worker_concurrency: impl Into>, 393 | acc: Acc, 394 | f: F, 395 | ) -> FoldPar<'a, Self, Acc, F, Fut> 396 | where 397 | F: FnMut(Acc, Self::Item) -> Fut + Send + 'a, 398 | Fut: Future + Send + 'a, 399 | Self: Sized, 400 | { 401 | FoldPar::new(self, worker_count, worker_concurrency, acc, f) 402 | } 403 | 404 | /// Returns the last element of the input deluge and then finishes 405 | /// 406 | /// # Examples 407 | /// ``` 408 | /// use deluge::*; 409 | /// 410 | /// # futures::executor::block_on(async { 411 | /// let result = (0..10).into_deluge() 412 | /// .last() 413 | /// .collect::>(None) 414 | /// .await; 415 | /// 416 | /// assert_eq!(vec![9], result); 417 | /// # }); 418 | /// 419 | /// ``` 420 | fn last(self) -> Last 421 | where 422 | Self: Sized, 423 | { 424 | Last::new(self) 425 | } 426 | 427 | /// Consumes at most `how_many` elements from the Deluge, ignoring the rest. 428 | /// 429 | /// # Examples 430 | /// 431 | /// ``` 432 | /// use deluge::*; 433 | /// 434 | /// # futures::executor::block_on(async { 435 | /// let result = (0..100).into_deluge() 436 | /// .take(1) 437 | /// .fold(None, 0, |acc, x| async move { acc + x }) 438 | /// .await; 439 | /// 440 | /// assert_eq!(0, result); 441 | /// # }); 442 | /// ``` 443 | fn take(self, how_many: usize) -> Take 444 | where 445 | Self: Sized, 446 | { 447 | Take::new(self, how_many) 448 | } 449 | 450 | /// Combines two Deluges into one with elements being 451 | /// tuples of subsequent elements from each 452 | /// 453 | /// # Examples 454 | /// 455 | /// ``` 456 | /// use deluge::*; 457 | /// 458 | /// # futures::executor::block_on(async { 459 | /// let result = (0..100).rev() 460 | /// .into_deluge() 461 | /// .zip((0..90).into_deluge(), None) 462 | /// .collect::>(None) 463 | /// .await; 464 | /// 465 | /// assert_eq!(result.len(), 90); 466 | /// assert_eq!(result[0], (99, 0)); 467 | /// assert_eq!(result[1], (98, 1)); 468 | /// # }); 469 | /// ``` 470 | fn zip<'a, Del2>( 471 | self, 472 | other: Del2, 473 | concurrency: impl Into>, 474 | ) -> Zip<'a, Self, Del2> 475 | where 476 | Del2: Deluge + 'a, 477 | Self: Sized, 478 | { 479 | Zip::new(self, other, concurrency) 480 | } 481 | 482 | /// Collects elements in the current `Deluge` into a collection with a desired concurrency 483 | /// 484 | /// # Examples 485 | /// 486 | /// ``` 487 | /// use deluge::*; 488 | /// 489 | /// # futures::executor::block_on(async { 490 | /// let result = (0..100).into_deluge() 491 | /// .collect::>(None) 492 | /// .await; 493 | /// 494 | /// assert_eq!(result.len(), 100); 495 | /// # }); 496 | /// ``` 497 | fn collect<'a, C>(self, concurrency: impl Into>) -> Collect<'a, Self, C> 498 | where 499 | C: Default + Extend, 500 | Self: Sized, 501 | { 502 | Collect::new(self, concurrency) 503 | } 504 | 505 | /// Collects elements in the current `Deluge` into a collection 506 | /// in parallel. Optionally accepts a degree of parallelism 507 | /// and concurrency for each worker. 508 | /// 509 | /// If the number of workers is not specified, we will default to the number of logical cpus. 510 | /// If concurrency per worker is not specified, we will default to the total number of 511 | /// items in a current deluge divided by the number of workers. 512 | /// 513 | /// # Examples 514 | /// 515 | /// ``` 516 | /// use deluge::*; 517 | /// 518 | /// # futures::executor::block_on(async { 519 | /// let result = (0..100).into_deluge() 520 | /// .collect_par::>(None, None) 521 | /// .await; 522 | /// 523 | /// assert_eq!(result.len(), 100); 524 | /// # }); 525 | /// ``` 526 | #[cfg(feature = "async-runtime")] 527 | fn collect_par<'a, C>( 528 | self, 529 | worker_count: impl Into>, 530 | worker_concurrency: impl Into>, 531 | ) -> CollectPar<'a, Self, C> 532 | where 533 | C: Default + Extend, 534 | Self: Sized, 535 | { 536 | CollectPar::new(self, worker_count, worker_concurrency) 537 | } 538 | } 539 | 540 | #[cfg(test)] 541 | mod tests { 542 | use super::*; 543 | use crate::into_deluge::IntoDeluge; 544 | use crate::iter::iter; 545 | use more_asserts::{assert_gt, assert_lt}; 546 | use std::sync::Arc; 547 | use std::time::{Duration, Instant}; 548 | use tokio::sync::Mutex; 549 | 550 | #[tokio::test] 551 | async fn map_can_be_created() { 552 | [1, 2, 3, 4].into_deluge().map(|x| async move { x * 2 }); 553 | assert_eq!(2, 2); 554 | } 555 | 556 | #[tokio::test] 557 | async fn we_can_collect() { 558 | let result = [1, 2, 3, 4].into_deluge().collect::>(None).await; 559 | 560 | assert_eq!(vec![1, 2, 3, 4], result); 561 | } 562 | 563 | #[tokio::test] 564 | async fn any_works() { 565 | let result = [1, 2, 3, 4] 566 | .into_deluge() 567 | .any(None, |x| async move { x == 4 }) 568 | .await; 569 | 570 | assert!(result); 571 | } 572 | 573 | #[tokio::test] 574 | async fn any_short_circuits() { 575 | let evaluated = Arc::new(Mutex::new(Vec::new())); 576 | 577 | let result = [1, 2, 3, 4, 5, 6, 7] 578 | .into_deluge() 579 | .any(1, |x| { 580 | let evaluated = evaluated.clone(); 581 | async move { 582 | { 583 | let mut evaluated = evaluated.lock().await; 584 | evaluated.push(x); 585 | } 586 | tokio::time::sleep(Duration::from_millis(100 - 10 * x)).await; 587 | x == 2 588 | } 589 | }) 590 | .await; 591 | 592 | assert!(result); 593 | // We might evaluate a little bit more than we should have, but not much more 594 | assert_lt!(Arc::try_unwrap(evaluated).unwrap().into_inner().len(), 4); 595 | } 596 | 597 | #[tokio::test] 598 | async fn any_par_works() { 599 | let result = [1, 2, 3, 4] 600 | .into_deluge() 601 | .any_par(None, None, |x| async move { x == 4 }) 602 | .await; 603 | 604 | assert!(result); 605 | } 606 | 607 | #[tokio::test] 608 | async fn any_par_short_circuits() { 609 | let evaluated = Arc::new(Mutex::new(Vec::new())); 610 | 611 | let result = [1, 2, 3, 4, 5, 6, 7] 612 | .into_deluge() 613 | .any_par(2, 1, |x| { 614 | let evaluated = evaluated.clone(); 615 | async move { 616 | { 617 | let mut evaluated = evaluated.lock().await; 618 | evaluated.push(x); 619 | } 620 | tokio::time::sleep(Duration::from_millis(100 - 10 * x)).await; 621 | x == 2 622 | } 623 | }) 624 | .await; 625 | 626 | assert!(result); 627 | // We might evaluate a little bit more than we should have, but not much more 628 | assert_lt!(Arc::try_unwrap(evaluated).unwrap().into_inner().len(), 5); 629 | } 630 | 631 | #[tokio::test] 632 | async fn all_works() { 633 | let result = [1, 2, 3, 4] 634 | .into_deluge() 635 | .all(None, |x| async move { x < 5 }).await; 636 | 637 | assert!(result); 638 | } 639 | 640 | #[tokio::test] 641 | async fn all_short_circuits() { 642 | let evaluated = Arc::new(Mutex::new(Vec::new())); 643 | 644 | let result = [1, 2, 3, 4, 5, 6, 7] 645 | .into_deluge() 646 | .all(1, |x| { 647 | let evaluated = evaluated.clone(); 648 | async move { 649 | { 650 | let mut evaluated = evaluated.lock().await; 651 | evaluated.push(x); 652 | } 653 | tokio::time::sleep(Duration::from_millis(100 - 10 * x)).await; 654 | x < 3 655 | } 656 | }) 657 | .await; 658 | 659 | assert!(!result); 660 | // We might evaluate a little bit more than we should have, but not much more 661 | assert_lt!(Arc::try_unwrap(evaluated).unwrap().into_inner().len(), 5); 662 | } 663 | 664 | #[tokio::test] 665 | async fn all_par_works() { 666 | let result = [1, 2, 3, 4] 667 | .into_deluge() 668 | .all_par(None, None, |x| async move { x < 5 }) 669 | .await; 670 | 671 | assert!(result); 672 | } 673 | 674 | #[tokio::test] 675 | async fn all_par_short_circuits() { 676 | let evaluated = Arc::new(Mutex::new(Vec::new())); 677 | 678 | let result = [1, 2, 3, 4, 5, 6, 7] 679 | .into_deluge() 680 | .all_par(2, 1, |x| { 681 | let evaluated = evaluated.clone(); 682 | async move { 683 | { 684 | let mut evaluated = evaluated.lock().await; 685 | evaluated.push(x); 686 | } 687 | tokio::time::sleep(Duration::from_millis(100 - 10 * x)).await; 688 | x < 3 689 | } 690 | }) 691 | .await; 692 | 693 | assert!(!result); 694 | // We might evaluate a little bit more than we should have, but not much more 695 | assert_lt!(Arc::try_unwrap(evaluated).unwrap().into_inner().len(), 7); 696 | } 697 | 698 | #[tokio::test] 699 | async fn chain_works() { 700 | let result = [1, 2, 3, 4] 701 | .into_deluge() 702 | .chain([5, 6, 7, 8].into_deluge()) 703 | .collect::>(None) 704 | .await; 705 | 706 | assert_eq!(result.len(), 8); 707 | assert_eq!(result, vec![1, 2, 3, 4, 5, 6, 7, 8]); 708 | } 709 | 710 | #[tokio::test] 711 | async fn count_works() { 712 | let result = [1, 2, 3, 4].into_deluge().count(); 713 | 714 | assert_eq!(result, 4); 715 | } 716 | 717 | #[tokio::test] 718 | async fn we_can_mult() { 719 | let result = [1, 2, 3, 4] 720 | .into_deluge() 721 | .map(|x| async move { x * 2 }) 722 | .collect::>(None) 723 | .await; 724 | 725 | assert_eq!(vec![2, 4, 6, 8], result); 726 | } 727 | 728 | #[tokio::test] 729 | async fn filter_map_works() { 730 | let result = [1, 2, 3, 4] 731 | .into_deluge() 732 | .filter_map(|x| async move { 733 | if x % 2 == 0 { 734 | Some("yes") 735 | } else { 736 | None 737 | } 738 | }) 739 | .collect::>(None) 740 | .await; 741 | 742 | assert_eq!(vec!["yes", "yes"], result); 743 | } 744 | 745 | #[tokio::test] 746 | async fn we_can_go_between_values_and_deluges() { 747 | let result = [1, 2, 3, 4] 748 | .into_deluge() 749 | .map(|x| async move { x * 2 }) 750 | .collect::>(None) 751 | .await 752 | .into_deluge() 753 | .map(|x| async move { x * 2 }) 754 | .fold(None, 0, |acc, x| async move { acc + x }) 755 | .await; 756 | 757 | assert_eq!(result, 40); 758 | } 759 | 760 | #[tokio::test] 761 | async fn we_wait_cuncurrently() { 762 | let start = Instant::now(); 763 | let result = (0..100) 764 | .into_deluge() 765 | .map(|idx| async move { 766 | tokio::time::sleep(Duration::from_millis(100 - (idx as u64))).await; 767 | idx 768 | }) 769 | .collect::>(None) 770 | .await; 771 | 772 | let iteration_took = Instant::now() - start; 773 | assert_lt!(iteration_took.as_millis(), 200); 774 | 775 | assert_eq!(result.len(), 100); 776 | 777 | result 778 | .into_iter() 779 | .enumerate() 780 | .for_each(|(idx, elem)| assert_eq!(idx, elem)); 781 | } 782 | 783 | #[tokio::test] 784 | async fn concurrency_limit() { 785 | let start = Instant::now(); 786 | let result = (0..15) 787 | .into_deluge() 788 | .map(|idx| async move { 789 | tokio::time::sleep(Duration::from_millis(50)).await; 790 | idx 791 | }) 792 | .collect::>(5) 793 | .await; 794 | 795 | let iteration_took = Instant::now() - start; 796 | assert_gt!(iteration_took.as_millis(), 150); 797 | assert_lt!(iteration_took.as_millis(), 200); 798 | 799 | assert_eq!(result.len(), 15); 800 | } 801 | 802 | #[tokio::test] 803 | async fn take_until_a_limit() { 804 | let result = (0..100) 805 | .into_deluge() 806 | .take(10) 807 | .fold(None, 0, |acc, idx| async move { acc + idx }) 808 | .await; 809 | 810 | assert_eq!(result, 45); 811 | } 812 | 813 | #[tokio::test] 814 | async fn first_works() { 815 | let result = (0..100) 816 | .into_deluge() 817 | .first() 818 | .collect::>(None) 819 | .await; 820 | 821 | assert_eq!(result, vec![0]); 822 | } 823 | 824 | #[tokio::test] 825 | async fn last_works() { 826 | let result = (0..100) 827 | .into_deluge() 828 | .last() 829 | .collect::>(None) 830 | .await; 831 | 832 | assert_eq!(result, vec![99]); 833 | } 834 | 835 | #[cfg(feature = "tokio")] 836 | #[tokio::test] 837 | async fn concurrent_fold() { 838 | let start = Instant::now(); 839 | let result = (0..100) 840 | .into_deluge() 841 | .map(|idx| async move { 842 | tokio::time::sleep(Duration::from_millis(100)).await; 843 | idx 844 | }) 845 | .fold(None, 0, |acc, idx| async move { acc + idx }) 846 | .await; 847 | 848 | let iteration_took = Instant::now() - start; 849 | assert_lt!(iteration_took.as_millis(), 200); 850 | 851 | assert_eq!(result, 4950); 852 | } 853 | 854 | #[cfg(feature = "tokio")] 855 | #[tokio::test] 856 | async fn parallel_test() { 857 | let start = Instant::now(); 858 | let result = (0..150) 859 | .into_deluge() 860 | .map(|idx| async move { 861 | tokio::time::sleep(Duration::from_millis(50)).await; 862 | idx 863 | }) 864 | .collect_par::>(10, 5) 865 | .await; 866 | 867 | let iteration_took = Instant::now() - start; 868 | assert_gt!(iteration_took.as_millis(), 150); 869 | assert_lt!(iteration_took.as_millis(), 200); 870 | 871 | assert_eq!(result.len(), 150); 872 | } 873 | 874 | #[cfg(feature = "async-std")] 875 | #[async_std::test] 876 | async fn parallel_test() { 877 | let start = Instant::now(); 878 | let result = (0..150) 879 | .into_deluge() 880 | .map(|idx| async move { 881 | async_std::task::sleep(Duration::from_millis(50)).await; 882 | idx 883 | }) 884 | .collect_par::>(10, 5) 885 | .await; 886 | 887 | let iteration_took = Instant::now() - start; 888 | assert_gt!(iteration_took.as_millis(), 150); 889 | assert_lt!(iteration_took.as_millis(), 200); 890 | 891 | assert_eq!(result.len(), 150); 892 | } 893 | 894 | #[cfg(feature = "tokio")] 895 | #[tokio::test] 896 | async fn parallel_fold() { 897 | let start = Instant::now(); 898 | let result = (0..150) 899 | .into_deluge() 900 | .map(|idx| async move { 901 | tokio::time::sleep(Duration::from_millis(50)).await; 902 | idx 903 | }) 904 | .fold_par(10, 5, 0, |acc, x| async move { acc + x }) 905 | .await; 906 | 907 | let iteration_took = Instant::now() - start; 908 | assert_gt!(iteration_took.as_millis(), 150); 909 | assert_lt!(iteration_took.as_millis(), 200); 910 | 911 | assert_eq!(result, 11175); 912 | } 913 | 914 | #[cfg(feature = "async-std")] 915 | #[async_std::test] 916 | async fn parallel_fold() { 917 | let start = Instant::now(); 918 | let result = (0..150) 919 | .into_deluge() 920 | .map(|idx| async move { 921 | async_std::task::sleep(Duration::from_millis(50)).await; 922 | idx 923 | }) 924 | .fold_par(10, 5, 0, |acc, x| async move { acc + x }) 925 | .await; 926 | 927 | let iteration_took = Instant::now() - start; 928 | assert_gt!(iteration_took.as_millis(), 150); 929 | assert_lt!(iteration_took.as_millis(), 200); 930 | 931 | assert_eq!(result, 11175); 932 | } 933 | 934 | #[cfg(feature = "async-runtime")] 935 | #[tokio::test] 936 | async fn zips_work() { 937 | let result = (0..100) 938 | .into_deluge() 939 | .zip((10..90).into_deluge(), None) 940 | .collect::>(None) 941 | .await; 942 | 943 | assert_eq!(result.len(), 80); 944 | } 945 | 946 | #[cfg(feature = "async-runtime")] 947 | #[tokio::test] 948 | async fn zips_inverted_waits() { 949 | let other_deluge = (0..90).into_deluge().map(|idx| async move { 950 | // We sleep here so first element from this Deluge 951 | // only becomes available with the last element from the next one 952 | tokio::time::sleep(Duration::from_millis(idx)).await; 953 | idx 954 | }); 955 | 956 | let result = (0..100).rev() 957 | .into_deluge() 958 | .map(|idx| async move { 959 | tokio::time::sleep(Duration::from_millis(idx)).await; 960 | idx 961 | }) 962 | .zip(other_deluge, None) 963 | .collect::>(None) 964 | .await; 965 | 966 | assert_eq!(result.len(), 90); 967 | for (idx, (fst, snd)) in result.into_iter().enumerate() { 968 | assert_eq!(idx as u64, 99 - fst); 969 | assert_eq!(idx as u64, snd); 970 | } 971 | } 972 | 973 | // Filter doesn't want to build, I have no idea why. 974 | // Let's move to augmenting the collector first 975 | /* 976 | #[tokio::test] 977 | async fn filter_works() { 978 | let result = (0..100) 979 | .into_deluge() 980 | .filter(|idx| async move { 981 | idx % 2 == 0 982 | }) 983 | .collect::>().await; 984 | 985 | assert_eq!(result.len(), 50); 986 | result.into_iter() 987 | .enumerate() 988 | .for_each(|(idx, elem)| assert_eq!(idx * 2, elem)); 989 | } 990 | */ 991 | } 992 | -------------------------------------------------------------------------------- /src/helpers/indexable_stream.rs: -------------------------------------------------------------------------------- 1 | use futures::stream::{StreamExt, StreamFuture}; 2 | use futures::Stream; 3 | use pin_project::pin_project; 4 | use std::collections::{BTreeMap, HashMap}; 5 | use std::future::Future; 6 | use std::marker::PhantomData; 7 | use std::pin::Pin; 8 | use std::sync::Arc; 9 | use std::task::{Context, Poll, Waker}; 10 | 11 | #[cfg(feature = "tokio")] 12 | type Mutex = tokio::sync::Mutex; 13 | #[cfg(feature = "async-std")] 14 | type Mutex = async_std::sync::Mutex; 15 | 16 | #[cfg(feature = "tokio")] 17 | type OwnedMutexGuard = tokio::sync::OwnedMutexGuard; 18 | #[cfg(feature = "async-std")] 19 | type OwnedMutexGuard = async_std::sync::MutexGuardArc; 20 | 21 | /// Stream wrapper allowing it's users to subscribe to an element at a specific index 22 | /// through `IndexableStream::get_nth`. 23 | pub struct IndexableStream<'a, S: Stream + 'a> { 24 | wakers: std::sync::Mutex>, 25 | inner: Arc>>, 26 | _lifetime: PhantomData<&'a ()>, 27 | } 28 | 29 | struct InnerIndexableStream { 30 | stream: Option>>, 31 | items: HashMap, 32 | next_promise: Option>>>, 33 | current_index: usize, 34 | exhausted: bool, 35 | } 36 | 37 | impl<'a, S: Stream + 'a> IndexableStream<'a, S> { 38 | pub fn new(stream: S) -> Self { 39 | Self { 40 | wakers: std::sync::Mutex::new(BTreeMap::new()), 41 | inner: Arc::new(Mutex::new(InnerIndexableStream { 42 | stream: Some(Box::pin(stream)), 43 | items: HashMap::new(), 44 | next_promise: None, 45 | current_index: 0, 46 | exhausted: false, 47 | })), 48 | _lifetime: PhantomData, 49 | } 50 | } 51 | 52 | pub fn get_nth(self: Arc, idx: usize) -> GetNthElement<'a, S> { 53 | GetNthElement { 54 | indexable: self, 55 | idx, 56 | mutex_guard: None, 57 | } 58 | } 59 | } 60 | 61 | type BoxFuture<'a, T> = Pin + 'a>>; 62 | 63 | #[pin_project] 64 | pub struct GetNthElement<'a, S: Stream + 'a> { 65 | indexable: Arc>, 66 | idx: usize, 67 | mutex_guard: Option>>>, 68 | } 69 | 70 | impl<'a, S: Stream + 'a> Future for GetNthElement<'a, S> { 71 | type Output = Option; 72 | 73 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 74 | let this = self.as_mut().project(); 75 | { 76 | let mut wakers = this.indexable.wakers.lock().unwrap(); 77 | wakers.insert(*this.idx, cx.waker().clone()); 78 | } 79 | 80 | if this.mutex_guard.is_none() { 81 | #[cfg(feature = "tokio")] 82 | { 83 | *this.mutex_guard = Some(Box::pin(this.indexable.inner.clone().lock_owned())); 84 | } 85 | #[cfg(feature = "async-std")] 86 | { 87 | *this.mutex_guard = Some(Box::pin(this.indexable.inner.lock_arc())); 88 | } 89 | } 90 | 91 | let mut mutex_guard = this.mutex_guard.take().unwrap(); 92 | match Pin::new(&mut mutex_guard).poll(cx) { 93 | Poll::Pending => { 94 | *this.mutex_guard = Some(mutex_guard); 95 | Poll::Pending 96 | } 97 | Poll::Ready(mut inner) => { 98 | if inner.exhausted { 99 | let mut wakers = this.indexable.wakers.lock().unwrap(); 100 | wakers.remove(&*this.idx); 101 | if let Some(v) = inner.items.remove(this.idx) { 102 | return Poll::Ready(Some(v)); 103 | } else { 104 | return Poll::Ready(None); 105 | } 106 | } 107 | 108 | if let Some(el) = inner.items.remove(this.idx) { 109 | let mut wakers = this.indexable.wakers.lock().unwrap(); 110 | wakers.remove(&*this.idx); 111 | return Poll::Ready(Some(el)); 112 | } 113 | 114 | let is_first_entry = { 115 | let mut wakers = this.indexable.wakers.lock().unwrap(); 116 | if let Some(first_entry) = wakers.first_entry() { 117 | *first_entry.key() == *this.idx 118 | } else { 119 | false 120 | } 121 | }; 122 | 123 | if is_first_entry { 124 | if inner.next_promise.is_none() { 125 | inner.next_promise = Some(inner.stream.take().unwrap().into_future()); 126 | } 127 | let mut next_promise = inner.next_promise.take().unwrap(); 128 | 129 | match Pin::new(&mut next_promise).poll(cx) { 130 | Poll::Pending => { 131 | inner.next_promise = Some(next_promise); 132 | Poll::Pending 133 | } 134 | Poll::Ready((None, _)) => { 135 | inner.exhausted = true; 136 | let wakers = this.indexable.wakers.lock().unwrap(); 137 | wakers.values().for_each(|waker| waker.wake_by_ref()); 138 | 139 | Poll::Ready(None) 140 | } 141 | Poll::Ready((Some(v), stream)) => { 142 | inner.stream = Some(stream); 143 | 144 | if *this.idx == inner.current_index { 145 | let mut wakers = this.indexable.wakers.lock().unwrap(); 146 | wakers.remove(this.idx); 147 | 148 | inner.current_index += 1; 149 | Poll::Ready(Some(v)) 150 | } else { 151 | let current_index = inner.current_index; 152 | inner.items.insert(current_index, v); 153 | 154 | let mut wakers = this.indexable.wakers.lock().unwrap(); 155 | if let Some(entry) = wakers.first_entry() { 156 | entry.get().wake_by_ref(); 157 | } 158 | 159 | inner.current_index += 1; 160 | Poll::Pending 161 | } 162 | } 163 | } 164 | } else { 165 | Poll::Pending 166 | } 167 | } 168 | } 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /src/helpers/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod indexable_stream; 2 | pub(crate) mod preloaded_futures; 3 | -------------------------------------------------------------------------------- /src/helpers/preloaded_futures.rs: -------------------------------------------------------------------------------- 1 | use crate::deluge::Deluge; 2 | 3 | use std::cell::RefCell; 4 | use std::collections::VecDeque; 5 | use std::future::Future; 6 | use std::marker::PhantomData; 7 | use std::pin::Pin; 8 | 9 | pub struct PreloadedFutures<'a, Del> 10 | where 11 | Del: Deluge + 'a, 12 | { 13 | storage: RefCell>>>>, 14 | _del: PhantomData, 15 | } 16 | 17 | impl<'a, Del> PreloadedFutures<'a, Del> 18 | where 19 | Del: Deluge + 'a, 20 | { 21 | pub fn new(deluge: Del) -> Self { 22 | let mut storage = VecDeque::new(); 23 | let deluge_borrow: &'a Del = unsafe { std::mem::transmute(&deluge) }; 24 | while let Some(v) = deluge_borrow.next() { 25 | storage.push_back(Box::pin(v)); 26 | } 27 | 28 | Self { 29 | storage: RefCell::new(storage), 30 | _del: PhantomData, 31 | } 32 | } 33 | 34 | pub fn len(&self) -> usize { 35 | self.storage.borrow().len() 36 | } 37 | } 38 | 39 | impl<'a, Del> Deluge for PreloadedFutures<'a, Del> 40 | where 41 | Del: Deluge + 'a, 42 | { 43 | type Item = Del::Item; 44 | type Output<'x> = impl Future> + 'x where Self: 'x; 45 | 46 | fn next(&self) -> Option> { 47 | let next_item = { self.storage.borrow_mut().pop_front() }; 48 | 49 | next_item.map(|el| async move { el.await }) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/into_deluge.rs: -------------------------------------------------------------------------------- 1 | use crate::iter::{iter, Iter}; 2 | use crate::Deluge; 3 | 4 | /// Allows converting any type that implements `IntoDeluge` into a `Deluge`. 5 | /// Specifically anything that implements `IntoIterator` or is a `Deluge` itself 6 | /// can be converted into `Deluge`. 7 | pub trait IntoDeluge 8 | where 9 | T: Deluge 10 | { 11 | fn into_deluge(self) -> T 12 | where 13 | Self: Sized; 14 | } 15 | 16 | impl IntoDeluge> for T 17 | where 18 | T: IntoIterator, 19 | ::IntoIter: Send + 'static, 20 | ::Item: Send, 21 | { 22 | fn into_deluge(self) -> Iter { 23 | iter(self) 24 | } 25 | } 26 | 27 | impl IntoDeluge for T 28 | where 29 | T: Deluge 30 | { 31 | fn into_deluge(self) -> Self { 32 | self 33 | } 34 | } 35 | 36 | /* 37 | impl IntoDeluge for futures::Stream 38 | { 39 | fn into_deluge 40 | } 41 | */ 42 | 43 | #[cfg(test)] 44 | mod tests { 45 | use super::*; 46 | 47 | #[test] 48 | fn we_can_convert_to_deluge() { 49 | [1, 2, 3].into_deluge(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/iter.rs: -------------------------------------------------------------------------------- 1 | use crate::deluge::Deluge; 2 | use std::cell::RefCell; 3 | use std::future::{self, Future}; 4 | 5 | pub struct Iter { 6 | iter: RefCell, 7 | } 8 | 9 | impl Unpin for Iter {} 10 | 11 | /// Converts any iterator into a `Deluge` 12 | pub(crate) fn iter(i: I) -> Iter 13 | where 14 | I: IntoIterator, 15 | { 16 | Iter { 17 | iter: RefCell::new(i.into_iter()), 18 | } 19 | } 20 | 21 | impl Deluge for Iter 22 | where 23 | I: Iterator + Send + 'static, 24 | ::Item: Send, 25 | { 26 | type Item = I::Item; 27 | type Output<'a> = impl Future> + 'a; 28 | 29 | fn next(&self) -> Option> { 30 | let item = { self.iter.borrow_mut().next() }; 31 | item.map(|item| future::ready(Some(item))) 32 | } 33 | } 34 | 35 | #[cfg(test)] 36 | mod tests { 37 | use super::*; 38 | 39 | #[tokio::test] 40 | async fn we_can_create_iter() { 41 | let _del = iter([1, 2, 3]); 42 | assert_eq!(2, 2); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![feature(associated_type_defaults)] 2 | #![feature(type_alias_impl_trait)] 3 | #![feature(const_trait_impl)] 4 | #![feature(let_chains)] 5 | #![feature(stmt_expr_attributes)] 6 | #![feature(extend_one)] 7 | #![feature(impl_trait_in_assoc_type)] 8 | 9 | //! # Deluge is (not) a Stream 10 | //! 11 | //! Deluge implements parallel and concurrent stream operations while driving the underlying futures concurrently. 12 | //! This is in contrast to standard streams which evaluate each future sequentially, leading to large delays on highly concurrent operations. 13 | //! 14 | //! ```toml 15 | //! deluge = "0.1" 16 | //! ``` 17 | //! 18 | //! **This library is still experimental, use at your own risk** 19 | //! 20 | //! ### Available features 21 | //! 22 | //! By default the library does not build the parallel collectors and folds. 23 | //! In order to enable them, please enable either the `tokio` or `async-std` feature. 24 | //! 25 | //! ```toml 26 | //! deluge = { version = "0.1", features = ["tokio"] } 27 | //! ``` 28 | //! 29 | //! or 30 | //! 31 | //! ```toml 32 | //! deluge = { version = "0.1", features = ["async-std"] } 33 | //! ``` 34 | //! 35 | //! ### Design decisions 36 | //! 37 | //! This is an opinionated library that puts ease of use and external simplicity at the forefront. 38 | //! Operations that apply to individual elements like maps and filters **do not** allocate. 39 | //! They simply wrap each element in another future but they do not control the way these processed elements are evaluated. 40 | //! It is the collector that controls the evaluation strategy. 41 | //! At the moment there are two basic collectors supplied: a concurrent and a parallel one. 42 | //! 43 | //! The concurrent collector accepts an optional concurrency limit. 44 | //! If it is specified, at most the number of futures equal to that limit will be evaluated at once. 45 | //! 46 | //! ``` 47 | //! use deluge::*; 48 | //! 49 | //! # futures::executor::block_on(async { 50 | //! let result = [1, 2, 3, 4] 51 | //! .into_deluge() 52 | //! .map(|x| async move { x * 2 }) 53 | //! .collect::>(None) 54 | //! .await; 55 | //! 56 | //! assert_eq!(vec![2, 4, 6, 8], result); 57 | //! # }); 58 | //! ``` 59 | //! 60 | //! The parallel collector spawns a number of workers. 61 | //! If a number of workers is not specified, it will default to the number of cpus, if the concurrency limit is not specified each worker will default to `total_futures_to_evaluate / number_of_workers`. 62 | //! Note that you need to enable either a `tokio` or `async-std` feature to support parallel collectors. 63 | //! 64 | //! ``` 65 | //! use deluge::*; 66 | //! # use std::time::Duration; 67 | //! 68 | //! # let rt = tokio::runtime::Runtime::new().unwrap(); 69 | //! # #[cfg(feature = "parallel")] 70 | //! # rt.handle().block_on(async { 71 | //! let result = (0..150) 72 | //! .into_deluge() 73 | //! .map(|idx| async move { 74 | //! tokio::time::sleep(Duration::from_millis(50)).await; 75 | //! idx 76 | //! }) 77 | //! .collect_par::>(10, None) 78 | //! .await; 79 | //! 80 | //! assert_eq!(result.len(), 150); 81 | //! # }); 82 | //! ``` 83 | 84 | mod deluge; 85 | mod deluge_ext; 86 | mod helpers; 87 | mod into_deluge; 88 | mod iter; 89 | mod ops; 90 | 91 | pub use self::deluge::*; 92 | pub use deluge_ext::*; 93 | pub use into_deluge::*; 94 | pub use iter::*; 95 | -------------------------------------------------------------------------------- /src/ops/all.rs: -------------------------------------------------------------------------------- 1 | use pin_project::pin_project; 2 | 3 | use super::collect::Collect; 4 | use super::map::Map; 5 | use crate::deluge::Deluge; 6 | use futures::task::{Context, Poll}; 7 | use futures::Stream; 8 | use std::future::Future; 9 | use std::pin::Pin; 10 | 11 | #[pin_project] 12 | pub struct All<'a, Del, Fut, F> 13 | where 14 | Del: Deluge + 'a, 15 | F: Fn(Del::Item) -> Fut + Send + 'a, 16 | Fut: Future + Send, 17 | { 18 | #[pin] 19 | stream: Collect<'a, Map, ()>, 20 | } 21 | 22 | impl<'a, Del, Fut, F> All<'a, Del, Fut, F> 23 | where 24 | Del: Deluge + 'a, 25 | F: Fn(Del::Item) -> Fut + Send + 'a, 26 | Fut: Future + Send, 27 | { 28 | pub(crate) fn new(deluge: Del, concurrency: impl Into>, f: F) -> Self { 29 | Self { 30 | stream: Collect::new(Map::new(deluge, f), concurrency), 31 | } 32 | } 33 | } 34 | 35 | impl<'a, Del, Fut, F> Future for All<'a, Del, Fut, F> 36 | where 37 | Del: Deluge + 'a, 38 | F: Fn(Del::Item) -> Fut + Send + 'a, 39 | Fut: Future + Send, 40 | { 41 | type Output = bool; 42 | 43 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 44 | let this = self.as_mut().project(); 45 | match this.stream.poll_next(cx) { 46 | Poll::Ready(Some(true)) => { 47 | cx.waker().wake_by_ref(); 48 | Poll::Pending 49 | } 50 | Poll::Ready(Some(false)) => Poll::Ready(false), 51 | Poll::Ready(None) => Poll::Ready(true), 52 | Poll::Pending => Poll::Pending, 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/ops/all_par.rs: -------------------------------------------------------------------------------- 1 | use pin_project::pin_project; 2 | 3 | use super::collect_par::CollectPar; 4 | use super::map::Map; 5 | use crate::deluge::Deluge; 6 | use futures::task::{Context, Poll}; 7 | use futures::Stream; 8 | use std::future::Future; 9 | use std::pin::Pin; 10 | 11 | #[pin_project] 12 | pub struct AllPar<'a, Del, Fut, F> 13 | where 14 | Del: Deluge + 'a, 15 | F: Fn(Del::Item) -> Fut + Send + 'a, 16 | Fut: Future + Send, 17 | { 18 | #[pin] 19 | stream: CollectPar<'a, Map, ()>, 20 | } 21 | 22 | impl<'a, Del, Fut, F> AllPar<'a, Del, Fut, F> 23 | where 24 | Del: Deluge + 'a, 25 | F: Fn(Del::Item) -> Fut + Send + 'a, 26 | Fut: Future + Send, 27 | { 28 | pub(crate) fn new( 29 | deluge: Del, 30 | worker_count: impl Into>, 31 | worker_concurrency: impl Into>, 32 | f: F, 33 | ) -> Self { 34 | Self { 35 | stream: CollectPar::new(Map::new(deluge, f), worker_count, worker_concurrency), 36 | } 37 | } 38 | } 39 | 40 | impl<'a, Del, Fut, F> Future for AllPar<'a, Del, Fut, F> 41 | where 42 | Del: Deluge + 'a, 43 | F: Fn(Del::Item) -> Fut + Send + 'a, 44 | Fut: Future + Send, 45 | { 46 | type Output = bool; 47 | 48 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 49 | let this = self.as_mut().project(); 50 | match this.stream.poll_next(cx) { 51 | Poll::Ready(Some(true)) => { 52 | cx.waker().wake_by_ref(); 53 | Poll::Pending 54 | } 55 | Poll::Ready(Some(false)) => Poll::Ready(false), 56 | Poll::Ready(None) => Poll::Ready(true), 57 | Poll::Pending => Poll::Pending, 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/ops/any.rs: -------------------------------------------------------------------------------- 1 | use pin_project::pin_project; 2 | 3 | use super::collect::Collect; 4 | use super::map::Map; 5 | use crate::deluge::Deluge; 6 | use futures::task::{Context, Poll}; 7 | use futures::Stream; 8 | use std::future::Future; 9 | use std::pin::Pin; 10 | 11 | #[pin_project] 12 | pub struct Any<'a, Del, Fut, F> 13 | where 14 | Del: Deluge + 'a, 15 | F: Fn(Del::Item) -> Fut + Send + 'a, 16 | Fut: Future + Send, 17 | { 18 | #[pin] 19 | stream: Collect<'a, Map, ()>, 20 | } 21 | 22 | impl<'a, Del, Fut, F> Any<'a, Del, Fut, F> 23 | where 24 | Del: Deluge + 'a, 25 | F: Fn(Del::Item) -> Fut + Send + 'a, 26 | Fut: Future + Send, 27 | { 28 | pub(crate) fn new(deluge: Del, concurrency: impl Into>, f: F) -> Self { 29 | Self { 30 | stream: Collect::new(Map::new(deluge, f), concurrency), 31 | } 32 | } 33 | } 34 | 35 | impl<'a, Del, Fut, F> Future for Any<'a, Del, Fut, F> 36 | where 37 | Del: Deluge + 'a, 38 | F: Fn(Del::Item) -> Fut + Send + 'a, 39 | Fut: Future + Send, 40 | { 41 | type Output = bool; 42 | 43 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 44 | let this = self.as_mut().project(); 45 | match this.stream.poll_next(cx) { 46 | Poll::Ready(Some(true)) => Poll::Ready(true), 47 | Poll::Ready(Some(false)) => { 48 | cx.waker().wake_by_ref(); 49 | Poll::Pending 50 | } 51 | Poll::Ready(None) => Poll::Ready(false), 52 | Poll::Pending => Poll::Pending, 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/ops/any_par.rs: -------------------------------------------------------------------------------- 1 | use pin_project::pin_project; 2 | 3 | use super::collect_par::CollectPar; 4 | use super::map::Map; 5 | use crate::deluge::Deluge; 6 | use futures::task::{Context, Poll}; 7 | use futures::Stream; 8 | use std::future::Future; 9 | use std::pin::Pin; 10 | 11 | #[pin_project] 12 | pub struct AnyPar<'a, Del, Fut, F> 13 | where 14 | Del: Deluge + 'a, 15 | F: Fn(Del::Item) -> Fut + Send + 'a, 16 | Fut: Future + Send, 17 | { 18 | #[pin] 19 | stream: CollectPar<'a, Map, ()>, 20 | } 21 | 22 | impl<'a, Del, Fut, F> AnyPar<'a, Del, Fut, F> 23 | where 24 | Del: Deluge + 'a, 25 | F: Fn(Del::Item) -> Fut + Send + 'a, 26 | Fut: Future + Send, 27 | { 28 | pub(crate) fn new( 29 | deluge: Del, 30 | worker_count: impl Into>, 31 | worker_concurrency: impl Into>, 32 | f: F, 33 | ) -> Self { 34 | Self { 35 | stream: CollectPar::new(Map::new(deluge, f), worker_count, worker_concurrency), 36 | } 37 | } 38 | } 39 | 40 | impl<'a, Del, Fut, F> Future for AnyPar<'a, Del, Fut, F> 41 | where 42 | Del: Deluge + 'a, 43 | F: Fn(Del::Item) -> Fut + Send + 'a, 44 | Fut: Future + Send, 45 | { 46 | type Output = bool; 47 | 48 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 49 | let this = self.as_mut().project(); 50 | match this.stream.poll_next(cx) { 51 | Poll::Ready(Some(true)) => Poll::Ready(true), 52 | Poll::Ready(Some(false)) => { 53 | cx.waker().wake_by_ref(); 54 | Poll::Pending 55 | } 56 | Poll::Ready(None) => Poll::Ready(false), 57 | Poll::Pending => Poll::Pending, 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/ops/chain.rs: -------------------------------------------------------------------------------- 1 | use std::marker::PhantomData; 2 | use std::sync::Mutex; 3 | 4 | use crate::deluge::Deluge; 5 | use futures::Future; 6 | 7 | pub struct Chain<'a, Del1, Del2> { 8 | deluge1: Del1, 9 | deluge2: Del2, 10 | first_exhausted: Mutex, 11 | _lifetime: PhantomData<&'a Del1>, 12 | } 13 | 14 | impl<'a, Del1, Del2> Chain<'a, Del1, Del2> 15 | where 16 | Del1: Deluge, 17 | Del2: for<'x> Deluge = Del1::Output<'x>>, 18 | { 19 | pub(crate) fn new(deluge1: Del1, deluge2: Del2) -> Self { 20 | Self { 21 | deluge1, 22 | deluge2, 23 | first_exhausted: Mutex::new(false), 24 | _lifetime: PhantomData, 25 | } 26 | } 27 | } 28 | 29 | impl<'a, Del1, Del2> Deluge for Chain<'a, Del1, Del2> 30 | where 31 | Del1: Deluge + 'static, 32 | Del2: for<'x> Deluge = Del1::Output<'x>> + 'static, 33 | { 34 | type Item = Del1::Item; 35 | type Output<'x> = impl Future> + 'x where Self: 'x; 36 | 37 | fn next(&self) -> Option> { 38 | let mut first_exhausted = self.first_exhausted.lock().unwrap(); 39 | 40 | let result = if *first_exhausted { 41 | self.deluge2.next() 42 | } else { 43 | match self.deluge1.next() { 44 | None => { 45 | *first_exhausted = true; 46 | self.deluge2.next() 47 | } 48 | otherwise => otherwise, 49 | } 50 | }; 51 | 52 | result.map(|v| async move { v.await }) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/ops/collect.rs: -------------------------------------------------------------------------------- 1 | use crate::deluge::Deluge; 2 | use core::pin::Pin; 3 | use futures::stream::Stream; 4 | use futures::task::{Context, Poll}; 5 | use pin_project::pin_project; 6 | use std::boxed::Box; 7 | use std::collections::{BTreeMap, HashMap}; 8 | use std::default::Default; 9 | use std::future::Future; 10 | use std::num::NonZeroUsize; 11 | 12 | type DelOutput<'a, Del> = dyn Future::Item>> + 'a; 13 | 14 | #[pin_project] 15 | pub struct Collect<'a, Del, C> 16 | where 17 | Del: Deluge, 18 | { 19 | deluge: Del, 20 | deluge_exhausted: bool, 21 | 22 | insert_idx: usize, 23 | concurrency: Option, 24 | 25 | polled_futures: HashMap>>>, 26 | completed_items: BTreeMap>, 27 | 28 | last_provided_idx: Option, 29 | collection: Option, 30 | } 31 | 32 | impl<'a, Del: Deluge, C: Default> Collect<'a, Del, C> { 33 | pub(crate) fn new(deluge: Del, concurrency: impl Into>) -> Self { 34 | Self { 35 | deluge, 36 | deluge_exhausted: false, 37 | 38 | insert_idx: 0, 39 | concurrency: concurrency.into().and_then(NonZeroUsize::new), 40 | 41 | polled_futures: HashMap::new(), 42 | completed_items: BTreeMap::new(), 43 | last_provided_idx: None, 44 | 45 | collection: Some(C::default()), 46 | } 47 | } 48 | } 49 | 50 | impl<'a, Del, C> Stream for Collect<'a, Del, C> 51 | where 52 | Del: Deluge + 'a, 53 | { 54 | type Item = Del::Item; 55 | 56 | fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 57 | let this = self.as_mut().project(); 58 | 59 | loop { 60 | while !*this.deluge_exhausted { 61 | let concurrency_limit = if let Some(limit) = this.concurrency { 62 | limit.get() 63 | } else { 64 | usize::MAX 65 | }; 66 | 67 | if this.polled_futures.len() < concurrency_limit { 68 | // We **know** that a reference to deluge lives for 'a, 69 | // so it should be safe to force the type system to acknowledge that 70 | let deluge: &'a Del = unsafe { std::mem::transmute(&mut *this.deluge) }; 71 | let next = deluge.next(); 72 | if let Some(future) = next { 73 | this.polled_futures 74 | .insert(*this.insert_idx, Box::pin(future)); 75 | *this.insert_idx += 1; 76 | } else { 77 | *this.deluge_exhausted = true; 78 | } 79 | } else { 80 | // We would exceed the concurrency limit by loading more elements 81 | break; 82 | } 83 | } 84 | 85 | // Drive all available futures 86 | if !this.polled_futures.is_empty() { 87 | this.polled_futures.retain(|idx, fut| { 88 | match Pin::new(fut).poll(cx) { 89 | Poll::Ready(v) => { 90 | // Drop the items that should be ignored on the floor. 91 | // The indexes in the `completed_items` map don't need 92 | // to be contignous, it's enough for them to be monotonic 93 | this.completed_items.insert(*idx, v); 94 | false 95 | } 96 | _ => true, 97 | } 98 | }); 99 | } 100 | 101 | // If all the polled futures were immediately evaluated 102 | // and we can still evaluate something more, load more items to evaluate 103 | // 104 | // Otherwise if these features need more time to evaluate 105 | // they will re-enter self::poll through the waker 106 | if !this.polled_futures.is_empty() || *this.deluge_exhausted { 107 | break; 108 | } 109 | } 110 | 111 | loop { 112 | let idx_to_provide = this.last_provided_idx.map(|x| x + 1).unwrap_or(0); 113 | if let Some(val) = this.completed_items.get_mut(&idx_to_provide) { 114 | *this.last_provided_idx = Some(idx_to_provide); 115 | // If we saw an item, always instruct the waker to try again 116 | cx.waker().wake_by_ref(); 117 | 118 | if val.is_some() { 119 | return Poll::Ready(Some(val.take().unwrap())); 120 | } 121 | } else if this.polled_futures.is_empty() { 122 | // Our input has been exhausted, nothing more to see 123 | return Poll::Ready(None); 124 | } else { 125 | return Poll::Pending; 126 | } 127 | } 128 | } 129 | } 130 | 131 | impl<'a, Del, C> Future for Collect<'a, Del, C> 132 | where 133 | Del: Deluge + 'a, 134 | C: Default + Extend, 135 | { 136 | type Output = C; 137 | 138 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 139 | match self.as_mut().poll_next(cx) { 140 | Poll::Ready(Some(v)) => { 141 | self.collection.as_mut().unwrap().extend_one(v); 142 | Poll::Pending 143 | } 144 | Poll::Ready(None) => Poll::Ready(self.collection.take().unwrap()), 145 | Poll::Pending => Poll::Pending, 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/ops/collect_par.rs: -------------------------------------------------------------------------------- 1 | use crate::deluge::Deluge; 2 | use core::pin::Pin; 3 | use futures::stream::{FuturesUnordered, StreamExt}; 4 | use futures::task::{Context, Poll}; 5 | use futures::Stream; 6 | use pin_project::pin_project; 7 | use std::boxed::Box; 8 | use std::collections::BTreeMap; 9 | use std::default::Default; 10 | use std::future::Future; 11 | use std::num::NonZeroUsize; 12 | use std::sync::Arc; 13 | 14 | #[cfg(feature = "tokio")] 15 | type Mutex = tokio::sync::Mutex; 16 | #[cfg(feature = "tokio")] 17 | use tokio::sync::mpsc; 18 | #[cfg(feature = "tokio")] 19 | type SendError = tokio::sync::mpsc::error::SendError; 20 | 21 | #[cfg(feature = "async-std")] 22 | type Mutex = async_std::sync::Mutex; 23 | #[cfg(feature = "async-std")] 24 | use async_std::channel as mpsc; 25 | #[cfg(feature = "async-std")] 26 | type SendError = mpsc::SendError; 27 | 28 | type OutstandingFutures<'a, Del> = 29 | Arc::Output<'a>>>>>>; 30 | type CompletedItem = (usize, Option<::Item>); 31 | type Worker<'a> = Pin + 'a>>; 32 | 33 | #[pin_project] 34 | pub struct CollectPar<'a, Del, C> 35 | where 36 | Del: Deluge + 'a, 37 | { 38 | deluge: Del, 39 | deluge_exhausted: bool, 40 | worker_count: usize, 41 | worker_concurrency: Option, 42 | 43 | workers: Vec>, 44 | outstanding_futures: Option>, 45 | completed_items: BTreeMap>, 46 | #[allow(clippy::type_complexity)] 47 | completed_channel: Option<( 48 | mpsc::Sender>, 49 | mpsc::Receiver>, 50 | )>, 51 | 52 | last_provided_idx: Option, 53 | collection: Option, 54 | } 55 | 56 | impl<'a, Del: Deluge, C: Default> CollectPar<'a, Del, C> { 57 | pub(crate) fn new( 58 | deluge: Del, 59 | worker_count: impl Into>, 60 | worker_concurrency: impl Into>, 61 | ) -> Self { 62 | let worker_count = worker_count.into().unwrap_or_else(num_cpus::get); 63 | let mut workers = Vec::new(); 64 | workers.reserve_exact(worker_count); 65 | 66 | Self { 67 | deluge, 68 | deluge_exhausted: false, 69 | worker_count, 70 | worker_concurrency: worker_concurrency.into().and_then(NonZeroUsize::new), 71 | 72 | workers, 73 | outstanding_futures: None, 74 | completed_items: BTreeMap::new(), 75 | // Only spawn the channel after we know how many items we have to eval 76 | completed_channel: None, 77 | 78 | last_provided_idx: None, 79 | collection: Some(C::default()), 80 | } 81 | } 82 | } 83 | 84 | // Approach 85 | // 1. Central mutexed container for jobs to be stolen from 86 | // 2. Each worker starts with worker_concurrency futures 87 | // and steals from the central place as needed 88 | 89 | fn create_worker<'a, Del: Deluge + 'a>( 90 | outstanding_futures: OutstandingFutures<'a, Del>, 91 | completed_channel: mpsc::Sender>, 92 | concurrency: NonZeroUsize, 93 | ) -> Pin + 'a>> { 94 | Box::pin(async move { 95 | let mut evaluated_futures = FuturesUnordered::new(); 96 | 97 | let run_worker = make_fn_once(|| async { 98 | let mut more_work_available = true; 99 | loop { 100 | // Load up on work if we aren't full and we expect more work to show up 101 | if evaluated_futures.len() < concurrency.get() && more_work_available { 102 | let mut outstanding_futures = outstanding_futures.lock().await; 103 | while evaluated_futures.len() < concurrency.get() 104 | && !outstanding_futures.is_empty() 105 | { 106 | if let Some((idx, fut)) = outstanding_futures.pop_first() { 107 | evaluated_futures.push(IndexedFuture::new(idx, fut)); 108 | } 109 | } 110 | 111 | if outstanding_futures.is_empty() { 112 | more_work_available = false; 113 | } 114 | } 115 | 116 | if let Some(result) = evaluated_futures.next().await { 117 | completed_channel.send(result).await?; 118 | } else { 119 | // If there is no more results to fetch, double check if nothing 120 | // was returned into `outstanding_futures` by another crashing worker. 121 | // If it is still empty, terminate. 122 | let outstanding_futures = outstanding_futures.lock().await; 123 | if !outstanding_futures.is_empty() { 124 | more_work_available = true; 125 | continue; 126 | } else { 127 | break; 128 | } 129 | } 130 | } 131 | 132 | Ok::<(), SendError>>(()) 133 | }); 134 | 135 | if let Err(_e) = run_worker().await { 136 | // Return unevaluated futures into the pile 137 | let mut outstanding_futures = outstanding_futures.lock().await; 138 | evaluated_futures.into_iter().for_each(|fut| { 139 | outstanding_futures.insert(fut.index(), fut.into_future()); 140 | }); 141 | } 142 | }) 143 | } 144 | 145 | impl<'a, Del, C> Stream for CollectPar<'a, Del, C> 146 | where 147 | Del: Deluge + 'a, 148 | { 149 | type Item = Del::Item; 150 | 151 | fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 152 | let this = self.as_mut().project(); 153 | 154 | if !*this.deluge_exhausted { 155 | let mut outstanding_futures = BTreeMap::new(); 156 | let mut insert_idx = 0; 157 | 158 | // Load up all the futures 159 | loop { 160 | // We **know** that a reference to deluge lives for 'a, 161 | // so it should be safe to force the dilesystem to acknowledge that 162 | let deluge: &'a Del = unsafe { std::mem::transmute(&mut *this.deluge) }; 163 | 164 | let next = deluge.next(); 165 | if let Some(future) = next { 166 | outstanding_futures.insert(insert_idx, Box::pin(future)); 167 | insert_idx += 1; 168 | } else { 169 | *this.deluge_exhausted = true; 170 | break; 171 | } 172 | } 173 | 174 | let total_futures = outstanding_futures.len(); 175 | 176 | #[cfg(feature = "tokio")] 177 | { 178 | *this.completed_channel = Some(mpsc::channel(total_futures)); 179 | } 180 | #[cfg(feature = "async-std")] 181 | { 182 | *this.completed_channel = Some(mpsc::bounded(total_futures)); 183 | } 184 | 185 | *this.outstanding_futures = Some(Arc::new(Mutex::new(outstanding_futures))); 186 | 187 | // Spawn workers 188 | if this.workers.is_empty() { 189 | let worker_concurrency = this 190 | .worker_concurrency 191 | .or_else(|| NonZeroUsize::new(total_futures / *this.worker_count)) 192 | .unwrap_or_else(|| unsafe { NonZeroUsize::new_unchecked(1) }); 193 | 194 | for _ in 0..(*this.worker_count) { 195 | this.workers.push(create_worker::<'a, Del>( 196 | this.outstanding_futures.as_ref().unwrap().clone(), 197 | this.completed_channel.as_ref().unwrap().0.clone(), 198 | worker_concurrency, 199 | )); 200 | } 201 | } 202 | } 203 | 204 | // Drive the workers 205 | this.workers 206 | .retain_mut(|worker| !matches!(Pin::new(worker).poll(cx), Poll::Ready(_))); 207 | 208 | // Drain the compelted channel 209 | while let Ok((idx, v)) = this.completed_channel.as_mut().unwrap().1.try_recv() { 210 | this.completed_items.insert(idx, v); 211 | } 212 | 213 | loop { 214 | let idx_to_provide = this.last_provided_idx.map(|x| x + 1).unwrap_or(0); 215 | if let Some(val) = this.completed_items.get_mut(&idx_to_provide) { 216 | *this.last_provided_idx = Some(idx_to_provide); 217 | // If we saw an item, always instruct the waker to try again 218 | cx.waker().wake_by_ref(); 219 | 220 | if val.is_some() { 221 | return Poll::Ready(Some(val.take().unwrap())); 222 | } 223 | } else if this.workers.is_empty() { 224 | // Our input has been exhausted, nothing more to see 225 | return Poll::Ready(None); 226 | } else { 227 | return Poll::Pending; 228 | } 229 | } 230 | } 231 | } 232 | 233 | impl<'a, Del, C> Future for CollectPar<'a, Del, C> 234 | where 235 | Del: Deluge + 'a, 236 | C: Default + Extend, 237 | { 238 | type Output = C; 239 | 240 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 241 | match self.as_mut().poll_next(cx) { 242 | Poll::Ready(Some(v)) => { 243 | self.collection.as_mut().unwrap().extend_one(v); 244 | Poll::Pending 245 | } 246 | Poll::Ready(None) => Poll::Ready(self.collection.take().unwrap()), 247 | Poll::Pending => Poll::Pending, 248 | } 249 | } 250 | } 251 | 252 | // A helper type allowing us to have a future with a synchronously available index on it 253 | #[pin_project] 254 | pub struct IndexedFuture { 255 | future: Pin>, 256 | index: usize, 257 | } 258 | 259 | impl IndexedFuture { 260 | fn new(index: usize, future: Pin>) -> Self { 261 | IndexedFuture { future, index } 262 | } 263 | 264 | fn index(&self) -> usize { 265 | self.index 266 | } 267 | 268 | fn into_future(self) -> Pin> { 269 | self.future 270 | } 271 | } 272 | 273 | impl Future for IndexedFuture 274 | where 275 | Fut: Future, 276 | { 277 | type Output = (usize, Fut::Output); 278 | 279 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 280 | let this = self.as_mut().project(); 281 | 282 | match Pin::new(this.future).poll(cx) { 283 | Poll::Ready(result) => Poll::Ready((*this.index, result)), 284 | _ => Poll::Pending, 285 | } 286 | } 287 | } 288 | 289 | // Useful for when we need to prove to the compiler that our worker body is a `FnOnce` 290 | fn make_fn_once T>(f: F) -> F { 291 | f 292 | } 293 | -------------------------------------------------------------------------------- /src/ops/count.rs: -------------------------------------------------------------------------------- 1 | use crate::deluge::Deluge; 2 | 3 | pub fn count(deluge: Del) -> usize { 4 | let mut count = 0; 5 | while deluge.next().is_some() { 6 | count += 1; 7 | } 8 | 9 | count 10 | } 11 | -------------------------------------------------------------------------------- /src/ops/filter.rs: -------------------------------------------------------------------------------- 1 | use crate::deluge::Deluge; 2 | use std::{future::Future, marker::PhantomData}; 3 | 4 | pub struct Filter<'x, Del, F> { 5 | deluge: Del, 6 | f: F, 7 | _del: PhantomData<&'x Del>, 8 | } 9 | 10 | impl<'x, Del, F> Filter<'x, Del, F> { 11 | pub(crate) fn new(deluge: Del, f: F) -> Self { 12 | Self { 13 | deluge, 14 | f, 15 | _del: PhantomData, 16 | } 17 | } 18 | } 19 | 20 | /// An internal helper trait allowing us to bind the lifetime 21 | /// of an output future with a lifetime of a parameter to a callback function 22 | pub trait XFn<'a, I: 'a, O> { 23 | type Output: Future + 'a; 24 | fn call(&'a self, x: &'a I) -> Self::Output; 25 | } 26 | 27 | impl<'a, I: 'a, O, F, Fut> XFn<'a, I, O> for F 28 | where 29 | F: Fn(&'a I) -> Fut, 30 | Fut: Future + 'a, 31 | { 32 | type Output = Fut; 33 | fn call(&'a self, x: &'a I) -> Fut { 34 | self(x) 35 | } 36 | } 37 | 38 | impl<'a, InputDel, F> Deluge for Filter<'a, InputDel, F> 39 | where 40 | InputDel: Deluge + 'a, 41 | for<'b> F: XFn<'b, InputDel::Item, bool> + Send + 'b, 42 | { 43 | type Item = InputDel::Item; 44 | type Output<'x> = impl Future> + 'x where Self: 'x; 45 | 46 | fn next(&self) -> Option> { 47 | self.deluge.next().map(|item| async { 48 | let item = item.await; 49 | if let Some(item) = item { 50 | if self.f.call(&item).await { 51 | Some(item) 52 | } else { 53 | None 54 | } 55 | } else { 56 | None 57 | } 58 | }) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/ops/filter_map.rs: -------------------------------------------------------------------------------- 1 | use crate::deluge::Deluge; 2 | use std::future::Future; 3 | 4 | pub struct FilterMap { 5 | deluge: Del, 6 | f: F, 7 | } 8 | 9 | impl FilterMap { 10 | pub(crate) fn new(deluge: Del, f: F) -> Self { 11 | Self { deluge, f } 12 | } 13 | } 14 | 15 | impl Deluge for FilterMap 16 | where 17 | InputDel: Deluge, 18 | F: Fn(InputDel::Item) -> Fut + Send, 19 | Fut: Future> + Send, 20 | FOutput: Send, 21 | { 22 | type Item = FOutput; 23 | type Output<'x> = impl Future> + 'x where Self: 'x; 24 | 25 | fn next(&self) -> Option> { 26 | self.deluge.next().map(|item| async { 27 | let item = item.await; 28 | if let Some(item) = item { 29 | (self.f)(item).await 30 | } else { 31 | None 32 | } 33 | }) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/ops/first.rs: -------------------------------------------------------------------------------- 1 | use crate::deluge::Deluge; 2 | use std::sync::Mutex; 3 | 4 | pub struct First { 5 | deluge: Del, 6 | item_provided: Mutex, 7 | } 8 | 9 | impl First { 10 | pub(crate) fn new(deluge: Del) -> Self { 11 | Self { 12 | deluge, 13 | item_provided: Mutex::new(false), 14 | } 15 | } 16 | } 17 | 18 | impl Deluge for First 19 | where 20 | Del: Deluge, 21 | { 22 | type Item = Del::Item; 23 | type Output<'a> = Del::Output<'a> where Self: 'a; 24 | 25 | fn next(&self) -> Option> { 26 | let mut item_provided = self.item_provided.lock().unwrap(); 27 | 28 | if !*item_provided { 29 | *item_provided = true; 30 | self.deluge.next() 31 | } else { 32 | None 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/ops/fold.rs: -------------------------------------------------------------------------------- 1 | use crate::deluge::Deluge; 2 | use core::pin::Pin; 3 | use futures::task::{Context, Poll}; 4 | use pin_project::pin_project; 5 | use std::boxed::Box; 6 | use std::future::Future; 7 | use std::marker::PhantomData; 8 | 9 | use super::collect::Collect; 10 | 11 | #[pin_project] 12 | pub struct Fold 13 | where 14 | Del: Deluge, 15 | F: FnMut(Acc, Del::Item) -> Fut + Send, 16 | Fut: Future + Send, 17 | { 18 | deluge: Option, 19 | #[allow(clippy::type_complexity)] 20 | collect_future: Option>>>>, 21 | collected_result: Option>, 22 | current_el_future: Option, 23 | 24 | concurrency: Option, 25 | 26 | acc: Option, 27 | f: F, 28 | _acc_fut: PhantomData, 29 | } 30 | 31 | impl Fold 32 | where 33 | Del: Deluge, 34 | F: FnMut(Acc, Del::Item) -> Fut + Send, 35 | Fut: Future + Send, 36 | { 37 | pub(crate) fn new(deluge: Del, concurrency: impl Into>, acc: Acc, f: F) -> Self { 38 | Self { 39 | deluge: Some(deluge), 40 | collect_future: None, 41 | collected_result: None, 42 | current_el_future: None, 43 | 44 | concurrency: concurrency.into(), 45 | 46 | acc: Some(acc), 47 | f, 48 | _acc_fut: PhantomData, 49 | } 50 | } 51 | } 52 | 53 | impl Future for Fold 54 | where 55 | InputDel: Deluge + 'static, 56 | F: FnMut(Acc, InputDel::Item) -> Fut + Send, 57 | Fut: Future + Send, 58 | { 59 | type Output = Acc; 60 | 61 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 62 | let this = self.as_mut().project(); 63 | if this.deluge.is_some() && this.collect_future.is_none() { 64 | let collect_future = Collect::new(this.deluge.take().unwrap(), *this.concurrency); 65 | *this.collect_future = Some(Box::pin(collect_future)); 66 | } 67 | 68 | if let Some(collect_future) = this.collect_future.as_mut() { 69 | match Pin::new(collect_future).poll(cx) { 70 | Poll::Ready(v) => { 71 | *this.collected_result = Some(v.into_iter()); 72 | *this.collect_future = None; 73 | } 74 | _ => return Poll::Pending, 75 | } 76 | } 77 | 78 | loop { 79 | if let Some(collected_result) = this.collected_result.as_mut() && this.current_el_future.is_none() { 80 | if let Some(el) = collected_result.next() { 81 | *this.current_el_future = Some((this.f)(this.acc.take().unwrap(), el)); 82 | } else { 83 | *this.collected_result = None; 84 | break; 85 | } 86 | } 87 | 88 | if let Some(current_el_future) = this.current_el_future.as_mut() { 89 | // We're manually pin-projecting since we need to project into an Option 90 | match unsafe { Pin::new_unchecked(current_el_future) }.poll(cx) { 91 | Poll::Ready(v) => { 92 | *this.current_el_future = None; 93 | *this.acc = Some(v); 94 | } 95 | _ => return Poll::Pending, 96 | } 97 | }; 98 | } 99 | 100 | Poll::Ready(this.acc.take().unwrap()) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/ops/fold_par.rs: -------------------------------------------------------------------------------- 1 | use crate::deluge::Deluge; 2 | use core::pin::Pin; 3 | use futures::task::{Context, Poll}; 4 | use pin_project::pin_project; 5 | use std::boxed::Box; 6 | use std::future::Future; 7 | use std::marker::PhantomData; 8 | 9 | use super::collect_par::CollectPar; 10 | 11 | #[pin_project] 12 | pub struct FoldPar<'a, Del, Acc, F, Fut> 13 | where 14 | Del: Deluge, 15 | F: FnMut(Acc, Del::Item) -> Fut + Send + 'a, 16 | Fut: Future + Send + 'a, 17 | { 18 | deluge: Option, 19 | #[allow(clippy::type_complexity)] 20 | collect_future: Option> + 'a>>>, 21 | collected_result: Option>, 22 | current_el_future: Option, 23 | 24 | worker_count: Option, 25 | worker_concurrency: Option, 26 | 27 | acc: Option, 28 | f: F, 29 | _acc_fut: PhantomData, 30 | } 31 | 32 | impl<'a, Del, Acc, F, Fut> FoldPar<'a, Del, Acc, F, Fut> 33 | where 34 | Del: Deluge, 35 | F: FnMut(Acc, Del::Item) -> Fut + Send + 'a, 36 | Fut: Future + Send + 'a, 37 | { 38 | pub(crate) fn new( 39 | deluge: Del, 40 | worker_count: impl Into>, 41 | worker_concurrency: impl Into>, 42 | acc: Acc, 43 | f: F, 44 | ) -> Self { 45 | Self { 46 | deluge: Some(deluge), 47 | collect_future: None, 48 | collected_result: None, 49 | current_el_future: None, 50 | 51 | worker_count: worker_count.into(), 52 | worker_concurrency: worker_concurrency.into(), 53 | 54 | acc: Some(acc), 55 | f, 56 | _acc_fut: PhantomData, 57 | } 58 | } 59 | } 60 | 61 | impl<'a, InputDel, Acc, F, Fut> Future for FoldPar<'a, InputDel, Acc, F, Fut> 62 | where 63 | InputDel: Deluge + 'a, 64 | F: FnMut(Acc, InputDel::Item) -> Fut + Send + 'a, 65 | Fut: Future + Send + 'a, 66 | { 67 | type Output = Acc; 68 | 69 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 70 | let this = self.as_mut().project(); 71 | if this.deluge.is_some() && this.collect_future.is_none() { 72 | let collect_future = CollectPar::new( 73 | this.deluge.take().unwrap(), 74 | *this.worker_count, 75 | *this.worker_concurrency, 76 | ); 77 | *this.collect_future = Some(Box::pin(collect_future)); 78 | } 79 | 80 | if let Some(collect_future) = this.collect_future.as_mut() { 81 | match Pin::new(collect_future).poll(cx) { 82 | Poll::Ready(v) => { 83 | *this.collected_result = Some(v.into_iter()); 84 | *this.collect_future = None; 85 | } 86 | _ => return Poll::Pending, 87 | } 88 | } 89 | 90 | loop { 91 | if let Some(collected_result) = this.collected_result.as_mut() && this.current_el_future.is_none() { 92 | if let Some(el) = collected_result.next() { 93 | *this.current_el_future = Some((this.f)(this.acc.take().unwrap(), el)); 94 | } else { 95 | *this.collected_result = None; 96 | break; 97 | } 98 | } 99 | 100 | if let Some(current_el_future) = this.current_el_future.as_mut() { 101 | // We're manually pin-projecting since we need to project into an Option 102 | match unsafe { Pin::new_unchecked(current_el_future) }.poll(cx) { 103 | Poll::Ready(v) => { 104 | *this.current_el_future = None; 105 | *this.acc = Some(v); 106 | } 107 | _ => return Poll::Pending, 108 | } 109 | }; 110 | } 111 | 112 | Poll::Ready(this.acc.take().unwrap()) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/ops/last.rs: -------------------------------------------------------------------------------- 1 | use crate::deluge::Deluge; 2 | 3 | pub struct Last { 4 | deluge: Del, 5 | } 6 | 7 | impl Last { 8 | pub(crate) fn new(deluge: Del) -> Self { 9 | Self { deluge } 10 | } 11 | } 12 | 13 | impl Deluge for Last 14 | where 15 | Del: Deluge, 16 | { 17 | type Item = Del::Item; 18 | type Output<'a> = Del::Output<'a> where Self: 'a; 19 | 20 | fn next(&self) -> Option> { 21 | let mut previous_value = None; 22 | while let Some(v) = self.deluge.next() { 23 | previous_value = Some(v); 24 | } 25 | 26 | previous_value 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/ops/map.rs: -------------------------------------------------------------------------------- 1 | use crate::deluge::Deluge; 2 | use std::future::Future; 3 | 4 | pub struct Map { 5 | deluge: Del, 6 | f: F, 7 | } 8 | 9 | impl Map { 10 | pub(crate) fn new(deluge: Del, f: F) -> Self { 11 | Self { deluge, f } 12 | } 13 | } 14 | 15 | impl Deluge for Map 16 | where 17 | InputDel: Deluge, 18 | F: Fn(InputDel::Item) -> Fut + Send, 19 | Fut: Future + Send, 20 | ::Output: Send, 21 | { 22 | type Item = Fut::Output; 23 | type Output<'a> = impl Future> + 'a where Self: 'a; 24 | 25 | fn next(&self) -> Option> { 26 | self.deluge.next().map(|item| async { 27 | let item = item.await; 28 | if let Some(item) = item { 29 | Some((self.f)(item).await) 30 | } else { 31 | None 32 | } 33 | }) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/ops/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod all; 2 | #[cfg(feature = "async-runtime")] 3 | pub mod all_par; 4 | pub mod any; 5 | #[cfg(feature = "async-runtime")] 6 | pub mod any_par; 7 | pub mod chain; 8 | pub mod collect; 9 | #[cfg(feature = "async-runtime")] 10 | pub mod collect_par; 11 | pub mod count; 12 | //pub mod filter; 13 | pub mod filter_map; 14 | pub mod first; 15 | pub mod fold; 16 | #[cfg(feature = "async-runtime")] 17 | pub mod fold_par; 18 | pub mod last; 19 | pub mod map; 20 | pub mod take; 21 | #[cfg(feature = "async-runtime")] 22 | pub mod zip; 23 | 24 | pub(crate) use all::*; 25 | #[cfg(feature = "async-runtime")] 26 | pub(crate) use all_par::*; 27 | pub(crate) use any::*; 28 | #[cfg(feature = "async-runtime")] 29 | pub(crate) use any_par::*; 30 | pub(crate) use chain::*; 31 | pub(crate) use collect::*; 32 | #[cfg(feature = "async-runtime")] 33 | pub(crate) use collect_par::*; 34 | pub(crate) use count::*; 35 | //pub(crate) use filter::*; 36 | pub(crate) use filter_map::*; 37 | pub(crate) use first::*; 38 | pub(crate) use fold::*; 39 | #[cfg(feature = "async-runtime")] 40 | pub(crate) use fold_par::*; 41 | pub(crate) use last::*; 42 | pub(crate) use map::*; 43 | pub(crate) use take::*; 44 | #[cfg(feature = "async-runtime")] 45 | pub(crate) use zip::*; 46 | -------------------------------------------------------------------------------- /src/ops/take.rs: -------------------------------------------------------------------------------- 1 | use crate::deluge::Deluge; 2 | use futures::Future; 3 | use std::cell::RefCell; 4 | 5 | pub struct Take { 6 | deluge: Del, 7 | how_many: usize, 8 | how_many_provided: RefCell, 9 | } 10 | 11 | impl Take { 12 | pub(crate) fn new(deluge: Del, how_many: usize) -> Self { 13 | Self { 14 | deluge, 15 | how_many, 16 | how_many_provided: RefCell::new(0), 17 | } 18 | } 19 | } 20 | impl Deluge for Take 21 | where 22 | Del: Deluge + 'static, 23 | { 24 | type Item = Del::Item; 25 | type Output<'a> = impl Future> + 'a; 26 | 27 | fn next(&self) -> Option> { 28 | let mut how_many_provided = self.how_many_provided.borrow_mut(); 29 | if *how_many_provided < self.how_many { 30 | *how_many_provided += 1; 31 | self.deluge.next() 32 | } else { 33 | None 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/ops/zip.rs: -------------------------------------------------------------------------------- 1 | use super::collect::Collect; 2 | use crate::deluge::Deluge; 3 | use crate::helpers::indexable_stream::*; 4 | use crate::helpers::preloaded_futures::*; 5 | 6 | use pin_project::pin_project; 7 | use std::cell::RefCell; 8 | use std::future::Future; 9 | use std::pin::Pin; 10 | use std::sync::Arc; 11 | 12 | #[cfg(feature = "tokio")] 13 | type Mutex = tokio::sync::Mutex; 14 | #[cfg(feature = "async-std")] 15 | type Mutex = async_std::sync::Mutex; 16 | 17 | #[pin_project] 18 | pub struct Zip<'a, Del1, Del2> 19 | where 20 | Del1: Deluge + 'a, 21 | Del2: Deluge + 'a, 22 | { 23 | streams: Mutex>, 24 | 25 | provided_elems: RefCell, 26 | elems_to_provide: usize, 27 | } 28 | 29 | struct Streams<'a, Del1, Del2> 30 | where 31 | Del1: Deluge + 'a, 32 | Del2: Deluge + 'a, 33 | { 34 | first: Arc, ()>>>, 35 | second: Arc, ()>>>, 36 | } 37 | 38 | impl<'a, Del1, Del2> Zip<'a, Del1, Del2> 39 | where 40 | Del1: Deluge + 'a, 41 | Del2: Deluge + 'a, 42 | { 43 | pub(crate) fn new(first: Del1, second: Del2, concurrency: impl Into>) -> Self { 44 | let concurrency = concurrency.into(); 45 | 46 | // Preload the futures from each 47 | let preloaded1 = PreloadedFutures::new(first); 48 | let preloaded2 = PreloadedFutures::new(second); 49 | 50 | let elems_to_provide = std::cmp::min(preloaded1.len(), preloaded2.len()); 51 | 52 | Self { 53 | streams: Mutex::new(Streams { 54 | first: Arc::new(IndexableStream::new(Collect::new(preloaded1, concurrency))), 55 | second: Arc::new(IndexableStream::new(Collect::new(preloaded2, concurrency))), 56 | }), 57 | 58 | provided_elems: RefCell::new(0), 59 | elems_to_provide, 60 | } 61 | } 62 | } 63 | 64 | impl<'a, Del1, Del2> Deluge for Zip<'a, Del1, Del2> 65 | where 66 | Del1: Deluge + 'a, 67 | Del2: Deluge + 'a, 68 | { 69 | type Item = (Del1::Item, Del2::Item); 70 | type Output<'x> = impl Future> + 'x where Self: 'x; 71 | 72 | fn next(&self) -> Option> { 73 | let mut provided_elems = self.provided_elems.borrow_mut(); 74 | if *provided_elems >= self.elems_to_provide { 75 | None 76 | } else { 77 | let current_index = *provided_elems; 78 | 79 | *provided_elems += 1; 80 | Some(async move { 81 | let this = Pin::new(self).project_ref(); 82 | 83 | let (first_el, second_el) = { 84 | let streams = this.streams.lock().await; 85 | ( 86 | streams.first.clone().get_nth(current_index).await, 87 | streams.second.clone().get_nth(current_index).await, 88 | ) 89 | }; 90 | 91 | match (first_el, second_el) { 92 | (Some(first), Some(second)) => Some((first, second)), 93 | _ => None, 94 | } 95 | }) 96 | } 97 | } 98 | } 99 | --------------------------------------------------------------------------------