├── .gitignore ├── .github └── workflows │ └── ci.yml ├── Cargo.toml ├── src ├── item.rs └── lib.rs └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | repository_dispatch: 5 | workflow_dispatch: 6 | push: 7 | branches: 8 | - master 9 | tags: 10 | - "*.*.*" 11 | pull_request: 12 | types: 13 | - opened 14 | - synchronize 15 | 16 | env: 17 | CARGO_TERM_COLOR: always 18 | CARGO_INCREMENTAL: 0 19 | RUSTFLAGS: -D warnings 20 | 21 | jobs: 22 | test: 23 | runs-on: ubuntu-latest 24 | steps: 25 | - uses: actions/checkout@v4 26 | - uses: dtolnay/rust-toolchain@stable 27 | - run: cargo test 28 | 29 | lint: 30 | runs-on: ubuntu-latest 31 | steps: 32 | - uses: actions/checkout@v4 33 | - uses: dtolnay/rust-toolchain@stable 34 | with: 35 | toolchain: stable 36 | components: clippy, rustfmt 37 | - name: Run cargo fmt (check if all code is rustfmt-ed) 38 | run: cargo fmt --all -- --check 39 | - name: Run cargo clippy (deny warnings) 40 | run: cargo clippy -- -D warnings 41 | 42 | publish-check: 43 | name: Publish Check 44 | runs-on: ubuntu-latest 45 | steps: 46 | - uses: actions/checkout@v4 47 | - uses: dtolnay/rust-toolchain@stable 48 | - run: cargo fetch 49 | - run: cargo publish --dry-run 50 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "process-stream" 3 | version = "0.5.0" 4 | edition = "2024" 5 | description = "Thin wrapper around [`tokio::process`] to make it streamable" 6 | authors = ["kkharji", "Matthias Endler"] 7 | license = "MIT" 8 | readme = "README.md" 9 | homepage = "https://github.com/kkharji/process-stream" 10 | repository = "https://github.com/kkharji/process-stream" 11 | categories = ["asynchronous"] 12 | keywords = ["tokio", "stream", "async-stream", "process"] 13 | 14 | [dependencies] 15 | tap = "1.0.1" 16 | futures = "0.3" 17 | tokio = { version = "1", features = ["rt-multi-thread", "macros", "process"] } 18 | tokio-stream = { version = "0.1", features = ["io-util"] } 19 | async-stream = "0.3" 20 | serde = { version = "1.0", features = ["derive"], optional = true } 21 | 22 | [features] 23 | default = [] 24 | serde = ["dep:serde"] 25 | 26 | [lints.clippy] 27 | all = "deny" 28 | pedantic = "deny" 29 | nursery = "deny" 30 | match_same_arms = { level = "allow", priority = 1 } 31 | 32 | [lints.rust] 33 | # Groups 34 | warnings = "deny" 35 | future-incompatible = "deny" 36 | nonstandard-style = "deny" 37 | rust-2018-idioms = "deny" 38 | rust-2021-compatibility = "deny" 39 | rust-2024-compatibility = "deny" 40 | unused = "deny" 41 | 42 | # Individual lints that deserve special attention 43 | invalid-html-tags = "deny" 44 | absolute-paths-not-starting-with-crate = "deny" 45 | anonymous-parameters = "deny" 46 | macro-use-extern-crate = "deny" 47 | missing-copy-implementations = "deny" 48 | missing-debug-implementations = "deny" 49 | missing-docs = "deny" 50 | semicolon-in-expressions-from-macros = "deny" 51 | unreachable-pub = "deny" 52 | variant-size-differences = "deny" 53 | unsafe-code = "deny" 54 | invalid-value = "deny" 55 | missing-abi = "deny" 56 | large-assignments = "deny" 57 | explicit-outlives-requirements = "deny" 58 | trivial-casts = "deny" 59 | trivial-numeric-casts = "deny" 60 | -------------------------------------------------------------------------------- /src/item.rs: -------------------------------------------------------------------------------- 1 | use std::{fmt, io, ops::Deref}; 2 | 3 | /// [`crate::Process`] stream output 4 | #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] 5 | #[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] 6 | pub enum ProcessItem { 7 | /// A stdout chunk printed by the process. 8 | Output(String), 9 | /// A stderr chunk printed by the process or internal error message 10 | Error(String), 11 | /// Indication that the process exit successful 12 | Exit(String), 13 | } 14 | 15 | impl Deref for ProcessItem { 16 | type Target = str; 17 | 18 | fn deref(&self) -> &Self::Target { 19 | match self { 20 | Self::Output(s) => s, 21 | Self::Error(s) => s, 22 | Self::Exit(s) => s, 23 | } 24 | } 25 | } 26 | 27 | impl fmt::Display for ProcessItem { 28 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 29 | self.deref().fmt(f) 30 | } 31 | } 32 | 33 | impl fmt::Debug for ProcessItem { 34 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 35 | match self { 36 | Self::Output(out) => write!(f, "[Output] {out}"), 37 | Self::Error(err) => write!(f, "[Error] {err}"), 38 | Self::Exit(code) => write!(f, "[Exit] {code}"), 39 | } 40 | } 41 | } 42 | impl From<(bool, io::Result)> for ProcessItem { 43 | fn from((is_stdout, line): (bool, io::Result)) -> Self { 44 | match line { 45 | Ok(line) if is_stdout => Self::Output(line), 46 | Ok(line) => Self::Error(line), 47 | Err(e) => Self::Error(e.to_string()), 48 | } 49 | } 50 | } 51 | 52 | impl ProcessItem { 53 | /// Returns `true` if the process item is [`Output`]. 54 | /// 55 | /// [`Output`]: ProcessItem::Output 56 | #[must_use] 57 | pub const fn is_output(&self) -> bool { 58 | matches!(self, Self::Output(..)) 59 | } 60 | 61 | /// Returns `true` if the process item is [`Error`]. 62 | /// 63 | /// [`Error`]: ProcessItem::Error 64 | #[must_use] 65 | pub const fn is_error(&self) -> bool { 66 | matches!(self, Self::Error(..)) 67 | } 68 | 69 | /// Returns `true` if the process item is [`Exit`]. 70 | /// 71 | /// [`Exit`]: ProcessItem::Exit 72 | #[must_use] 73 | pub const fn is_exit(&self) -> bool { 74 | matches!(self, Self::Exit(..)) 75 | } 76 | 77 | /// Returns Some(`true`) if the process item is [`Exit`] and returned 0 78 | /// 79 | /// [`Exit`]: ProcessItem::Exit 80 | #[must_use] 81 | pub fn is_success(&self) -> Option { 82 | self.as_exit().map(|s| s.trim() == "0") 83 | } 84 | 85 | /// Return exit code if [`ProcessItem`] is [`ProcessItem::Exit`] 86 | #[must_use] 87 | pub const fn as_exit(&self) -> Option<&String> { 88 | if let Self::Exit(v) = self { 89 | Some(v) 90 | } else { 91 | None 92 | } 93 | } 94 | 95 | /// Return inner reference [`String`] value if [`ProcessItem`] is [`ProcessItem::Error`] 96 | #[must_use] 97 | pub const fn as_error(&self) -> Option<&String> { 98 | if let Self::Error(v) = self { 99 | Some(v) 100 | } else { 101 | None 102 | } 103 | } 104 | 105 | /// Return inner reference [`String`] value if [`ProcessItem`] is [`ProcessItem::Output`] 106 | #[must_use] 107 | pub const fn as_output(&self) -> Option<&String> { 108 | if let Self::Output(v) = self { 109 | Some(v) 110 | } else { 111 | None 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # process-stream 2 | 3 | Wraps `tokio::process::Command` to `future::stream`. 4 | 5 | This library provide ProcessExt to create your own custom process 6 | 7 | ## Install 8 | 9 | ```toml 10 | process-stream = "0.3.1" 11 | ``` 12 | 13 | ## Example usage: 14 | 15 | ### From `Vec` or `Vec<&str>` 16 | 17 | ```rust 18 | use process_stream::{Process, ProcessExt, StreamExt}; 19 | use std::io; 20 | 21 | #[tokio::main] 22 | async fn main() -> io::Result<()> { 23 | let mut ls_home: Process = vec!["/bin/ls", "."].into(); 24 | 25 | let mut stream = ls_home.spawn_and_stream()?; 26 | 27 | while let Some(output) = stream.next().await { 28 | println!("{output}") 29 | } 30 | 31 | Ok(()) 32 | } 33 | ``` 34 | 35 | ### From `Path/PathBuf/str` 36 | 37 | ```rust 38 | use process_stream::{Process, ProcessExt, StreamExt}; 39 | use std::io; 40 | 41 | #[tokio::main] 42 | async fn main() -> io::Result<()> { 43 | let mut process: Process = "/bin/ls".into(); 44 | 45 | // block until process completes 46 | let outputs = process.spawn_and_stream()?.collect::>().await; 47 | 48 | println!("{outputs:#?}"); 49 | 50 | Ok(()) 51 | } 52 | ``` 53 | 54 | ### New 55 | 56 | ```rust 57 | use process_stream::{Process, ProcessExt, StreamExt}; 58 | use std::io; 59 | 60 | #[tokio::main] 61 | async fn main() -> io::Result<()> { 62 | let mut ls_home = Process::new("/bin/ls"); 63 | ls_home.arg("~/"); 64 | 65 | let mut stream = ls_home.spawn_and_stream()?; 66 | 67 | while let Some(output) = stream.next().await { 68 | println!("{output}") 69 | } 70 | 71 | Ok(()) 72 | } 73 | ``` 74 | 75 | ### Kill 76 | 77 | ```rust 78 | use process_stream::{Process, ProcessExt, StreamExt}; 79 | use std::io; 80 | 81 | #[tokio::main] 82 | async fn main() -> io::Result<()> { 83 | let mut long_process = Process::new("cat"); 84 | 85 | let mut stream = long_process.spawn_and_stream()?; 86 | 87 | tokio::spawn(async move { 88 | while let Some(output) = stream.next().await { 89 | println!("{output}") 90 | } 91 | }); 92 | 93 | // process some outputs 94 | tokio::time::sleep(std::time::Duration::from_secs(2)).await; 95 | 96 | // close the process 97 | long_process.abort(); 98 | 99 | Ok(()) 100 | } 101 | ``` 102 | 103 | ### Communicate with running process 104 | ```rust 105 | use process_stream::{Process, ProcessExt, StreamExt}; 106 | use tokio::io::AsyncWriteExt; 107 | use std::process::Stdio; 108 | 109 | #[tokio::main] 110 | async fn main() -> std::io::Result<()> { 111 | let mut process: Process = Process::new("sort"); 112 | 113 | // Set stdin (by default is set to null) 114 | process.stdin(Stdio::piped()); 115 | 116 | // Get Stream; 117 | let mut stream = process.spawn_and_stream().unwrap(); 118 | 119 | // Get writer from stdin; 120 | let mut writer = process.take_stdin().unwrap(); 121 | 122 | // Spawn new async task and move stream to it 123 | let reader_thread = tokio::spawn(async move { 124 | while let Some(output) = stream.next().await { 125 | if output.is_exit() { 126 | println!("DONE") 127 | } else { 128 | println!("{output}") 129 | } 130 | } 131 | }); 132 | 133 | // Spawn new async task and move writer to it 134 | let writer_thread = tokio::spawn(async move { 135 | writer.write(b"b\nc\na\n").await.unwrap(); 136 | writer.write(b"f\ne\nd\n").await.unwrap(); 137 | }); 138 | 139 | // Wait till all threads finish 140 | writer_thread.await.unwrap(); 141 | reader_thread.await.unwrap(); 142 | 143 | // Result 144 | // a 145 | // b 146 | // c 147 | // d 148 | // e 149 | // f 150 | // DONE 151 | Ok(()) 152 | } 153 | ``` 154 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! process-stream is a thin wrapper around [`tokio::process`] to make it streamable 2 | #![deny(future_incompatible)] 3 | #![deny(nonstandard_style)] 4 | #![deny(missing_docs)] 5 | #![deny(rustdoc::broken_intra_doc_links)] 6 | #![doc = include_str!("../README.md")] 7 | 8 | /// Alias for a stream of process items 9 | pub type ProcessStream = Pin + Send>>; 10 | 11 | pub use async_stream::stream; 12 | use io::Result; 13 | use std::{ 14 | ffi::OsStr, 15 | io, 16 | ops::{Deref, DerefMut}, 17 | path::{Path, PathBuf}, 18 | pin::Pin, 19 | process::Stdio, 20 | sync::Arc, 21 | }; 22 | use tap::Pipe; 23 | use { 24 | tokio::{ 25 | io::{AsyncBufReadExt, AsyncRead, BufReader}, 26 | process::{ChildStdin, Command}, 27 | sync::Notify, 28 | }, 29 | tokio_stream::wrappers::LinesStream, 30 | }; 31 | 32 | mod item; 33 | pub use futures::Stream; 34 | pub use futures::StreamExt; 35 | pub use futures::TryStreamExt; 36 | pub use item::ProcessItem; 37 | pub use tokio_stream; 38 | 39 | /// `ProcessExt` trait that needs to be implemented to make something streamable 40 | pub trait ProcessExt { 41 | /// Get command that will be used to create a child process from 42 | fn get_command(&mut self) -> &mut Command; 43 | 44 | /// Get command after settings the required pipes; 45 | fn command(&mut self) -> &mut Command { 46 | let stdin = self.get_stdin().unwrap(); 47 | let stdout = self.get_stdout().unwrap(); 48 | let stderr = self.get_stderr().unwrap(); 49 | let command = self.get_command(); 50 | 51 | #[cfg(windows)] 52 | command.creation_flags(0x08000000); 53 | 54 | command.stdin(stdin); 55 | command.stdout(stdout); 56 | command.stderr(stderr); 57 | command 58 | } 59 | 60 | /// Spawn and stream process 61 | /// 62 | /// # Errors 63 | /// 64 | /// Will return an error if the process fails to spawn 65 | fn spawn_and_stream(&mut self) -> Result { 66 | self.spawn_and_stream_inner() 67 | } 68 | 69 | /// # Errors 70 | /// 71 | /// Will return an error if the process fails to spawn. 72 | /// 73 | /// # Panics 74 | /// 75 | /// Panics if: 76 | /// - `stdout` or `stderr` is not available in the spawned child process (should not happen if 77 | /// proper pipes are set up with `get_stdout()` and `get_stderr()`) 78 | #[allow(tail_expr_drop_order)] 79 | fn spawn_and_stream_inner(&mut self) -> Result { 80 | let abort = Arc::new(Notify::new()); 81 | 82 | let mut child = self.command().spawn()?; 83 | 84 | let stdout = child.stdout.take().unwrap(); 85 | let stderr = child.stderr.take().unwrap(); 86 | 87 | self.set_child_stdin(child.stdin.take()); 88 | self.set_aborter(Some(abort.clone())); 89 | 90 | let stdout_stream = into_stream(stdout, true); 91 | let stderr_stream = into_stream(stderr, false); 92 | let mut std_stream = tokio_stream::StreamExt::merge(stdout_stream, stderr_stream); 93 | let stream = stream! { 94 | loop { 95 | use ProcessItem::{Error, Exit}; 96 | tokio::select! { 97 | Some(output) = std_stream.next() => yield output, 98 | status = child.wait() => { 99 | // Drain the stream before exiting 100 | while let Some(output) = std_stream.next().await { 101 | yield output 102 | } 103 | match status { 104 | Err(err) => yield Error(err.to_string()), 105 | Ok(status) => { 106 | match status.code() { 107 | Some(code) => yield Exit(format!("{code}")), 108 | None => yield Error("Unable to get exit code".into()), 109 | } 110 | } 111 | } 112 | break; 113 | }, 114 | () = abort.notified() => { 115 | match child.start_kill() { 116 | Ok(()) => yield Exit("0".into()), 117 | Err(err) => yield Error(format!("abort Process Error: {err}")), 118 | } 119 | break; 120 | } 121 | } 122 | } 123 | }; 124 | 125 | Ok(stream.boxed()) 126 | } 127 | /// Get a notifier that can be used to abort the process 128 | fn aborter(&self) -> Option>; 129 | /// Set the notifier that should be used to abort the process 130 | fn set_aborter(&mut self, aborter: Option>); 131 | /// Get process stdin 132 | fn take_stdin(&mut self) -> Option { 133 | None 134 | } 135 | /// Set process stdin 136 | fn set_child_stdin(&mut self, _child_stdin: Option) {} 137 | /// Get process stdin pipe 138 | fn get_stdin(&mut self) -> Option { 139 | Some(Stdio::null()) 140 | } 141 | /// get process stdout pipe 142 | fn get_stdout(&mut self) -> Option { 143 | Some(Stdio::piped()) 144 | } 145 | /// get process stderr pipe 146 | fn get_stderr(&mut self) -> Option { 147 | Some(Stdio::piped()) 148 | } 149 | } 150 | 151 | /// Thin Wrapper around [`Command`] to make it streamable 152 | #[derive(Debug)] 153 | pub struct Process { 154 | inner: Command, 155 | stdin: Option, 156 | set_stdin: Option, 157 | set_stdout: Option, 158 | set_stderr: Option, 159 | abort: Option>, 160 | } 161 | 162 | impl ProcessExt for Process { 163 | fn get_command(&mut self) -> &mut Command { 164 | &mut self.inner 165 | } 166 | 167 | fn aborter(&self) -> Option> { 168 | self.abort.clone() 169 | } 170 | 171 | fn set_aborter(&mut self, aborter: Option>) { 172 | self.abort = aborter; 173 | } 174 | 175 | fn take_stdin(&mut self) -> Option { 176 | self.stdin.take() 177 | } 178 | 179 | fn set_child_stdin(&mut self, child_stdin: Option) { 180 | self.stdin = child_stdin; 181 | } 182 | 183 | fn get_stdin(&mut self) -> Option { 184 | self.set_stdin.take() 185 | } 186 | 187 | fn get_stdout(&mut self) -> Option { 188 | self.set_stdout.take() 189 | } 190 | 191 | fn get_stderr(&mut self) -> Option { 192 | self.set_stderr.take() 193 | } 194 | } 195 | 196 | impl Process { 197 | /// Create new process with a program 198 | pub fn new>(program: S) -> Self { 199 | Self { 200 | inner: Command::new(program), 201 | set_stdin: Some(Stdio::null()), 202 | set_stdout: Some(Stdio::piped()), 203 | set_stderr: Some(Stdio::piped()), 204 | stdin: None, 205 | abort: None, 206 | } 207 | } 208 | 209 | /// Set the process's stdin. 210 | pub fn stdin(&mut self, stdin: Stdio) { 211 | self.set_stdin = stdin.into(); 212 | } 213 | 214 | /// Set the process's stdout. 215 | pub fn stdout(&mut self, stdout: Stdio) { 216 | self.set_stdout = stdout.into(); 217 | } 218 | 219 | /// Set the process's stderr. 220 | pub fn stderr(&mut self, stderr: Stdio) { 221 | self.set_stderr = stderr.into(); 222 | } 223 | 224 | /// Abort the process 225 | pub fn abort(&self) { 226 | if let Some(aborter) = self.aborter() { 227 | aborter.notify_waiters(); 228 | } 229 | } 230 | } 231 | 232 | impl Deref for Process { 233 | type Target = Command; 234 | 235 | fn deref(&self) -> &Self::Target { 236 | &self.inner 237 | } 238 | } 239 | 240 | impl DerefMut for Process { 241 | fn deref_mut(&mut self) -> &mut Self::Target { 242 | &mut self.inner 243 | } 244 | } 245 | 246 | impl From for Process { 247 | fn from(command: Command) -> Self { 248 | Self { 249 | inner: command, 250 | stdin: None, 251 | set_stdin: Some(Stdio::null()), 252 | set_stdout: Some(Stdio::piped()), 253 | set_stderr: Some(Stdio::piped()), 254 | abort: None, 255 | } 256 | } 257 | } 258 | 259 | impl> From> for Process { 260 | fn from(mut command_args: Vec) -> Self { 261 | let command = command_args.remove(0); 262 | let mut inner = Command::new(command); 263 | inner.args(command_args); 264 | 265 | Self::from(inner) 266 | } 267 | } 268 | 269 | impl From<&Path> for Process { 270 | fn from(path: &Path) -> Self { 271 | let command = Command::new(path); 272 | Self::from(command) 273 | } 274 | } 275 | 276 | impl From<&str> for Process { 277 | fn from(path: &str) -> Self { 278 | let command = Command::new(path); 279 | Self::from(command) 280 | } 281 | } 282 | 283 | impl From<&PathBuf> for Process { 284 | fn from(path: &PathBuf) -> Self { 285 | let command = Command::new(path); 286 | Self::from(command) 287 | } 288 | } 289 | 290 | /// Convert `std_stream` to a stream of T 291 | pub fn into_stream(std: R, is_stdout: bool) -> impl Stream 292 | where 293 | T: From<(bool, Result)>, 294 | R: AsyncRead, 295 | { 296 | std.pipe(BufReader::new) 297 | .lines() 298 | .pipe(LinesStream::new) 299 | .map(move |line| T::from((is_stdout, line))) 300 | } 301 | 302 | #[cfg(test)] 303 | mod tests { 304 | use tokio::io::AsyncWriteExt; 305 | 306 | use crate::*; 307 | use std::io::Result; 308 | 309 | #[tokio::test] 310 | async fn test_from_path() -> Result<()> { 311 | let mut process: Process = "/bin/ls".into(); 312 | 313 | let outputs = process.spawn_and_stream()?.collect::>().await; 314 | println!("{outputs:#?}"); 315 | Ok(()) 316 | } 317 | 318 | #[tokio::test] 319 | async fn test_dref_item_as_str() { 320 | use ProcessItem::*; 321 | let items = vec![ 322 | Output("Hello".into()), 323 | Error("XXXXXXXXXX".into()), 324 | Exit("0".into()), 325 | ]; 326 | for item in items { 327 | println!("{:?}", item.as_bytes()) 328 | } 329 | } 330 | 331 | #[tokio::test] 332 | async fn communicate_with_running_process() -> Result<()> { 333 | let mut process: Process = Process::new("sort"); 334 | 335 | // Set stdin (by default is set to null) 336 | process.stdin(Stdio::piped()); 337 | 338 | // Get Stream; 339 | let mut stream = process.spawn_and_stream().unwrap(); 340 | 341 | // Get writer from stdin; 342 | let mut writer = process.take_stdin().unwrap(); 343 | 344 | // Start new running process 345 | let reader_thread = tokio::spawn(async move { 346 | while let Some(output) = stream.next().await { 347 | if output.is_exit() { 348 | println!("DONE") 349 | } else { 350 | println!("{output}") 351 | } 352 | } 353 | }); 354 | 355 | let writer_thread = tokio::spawn(async move { 356 | writer.write(b"b\nc\na\n").await.unwrap(); 357 | writer.write(b"f\ne\nd\n").await.unwrap(); 358 | }); 359 | 360 | writer_thread.await?; 361 | reader_thread.await?; 362 | 363 | Ok(()) 364 | } 365 | } 366 | --------------------------------------------------------------------------------