├── .rustfmt.toml ├── .gitignore ├── Cargo.toml ├── LICENSE-MIT ├── src ├── local.rs ├── main.rs ├── remote.rs └── lib.rs ├── README.md ├── LICENSE-APACHE ├── Cargo.lock └── LICENSE-3RD-PARTY /.rustfmt.toml: -------------------------------------------------------------------------------- 1 | max_width = 79 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "shepherd" 3 | version = "0.1.5" 4 | authors = ["Martin Disch "] 5 | edition = "2018" 6 | description = """ 7 | A distributed video encoder that splits files into chunks to encode them on 8 | multiple machines in parallel. 9 | """ 10 | repository = "https://github.com/martindisch/shepherd" 11 | readme = "README.md" 12 | keywords = ["video", "encoder", "split", "parallel", "distributed"] 13 | categories = ["command-line-utilities", "multimedia::video", "encoding"] 14 | license = "MIT OR Apache-2.0" 15 | 16 | [dependencies] 17 | clap = "2.33.4" 18 | dirs = "4.0.0" 19 | crossbeam = "0.8.2" 20 | log = "0.4.17" 21 | simplelog = "0.12.0" 22 | ctrlc = { version = "3.2.3", features = ["termination"] } 23 | 24 | [badges] 25 | maintenance = { status = "passively-maintained" } 26 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Martin Disch 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/local.rs: -------------------------------------------------------------------------------- 1 | //! Functions for operations on the local host. 2 | 3 | use std::{ 4 | fs, 5 | path::Path, 6 | process::Command, 7 | sync::atomic::{AtomicBool, Ordering}, 8 | sync::Arc, 9 | time::Duration, 10 | }; 11 | 12 | use super::Result; 13 | 14 | /// Uses `ffmpeg` to locally extract and encode the audio. 15 | pub fn extract_audio( 16 | input: &Path, 17 | output: &Path, 18 | running: &Arc, 19 | ) -> Result<()> { 20 | // Convert input and output to &str 21 | let input = input.to_str().ok_or("Input invalid Unicode")?; 22 | let output = output.to_str().ok_or("Output invalid Unicode")?; 23 | // Do the extraction 24 | let output = Command::new("ffmpeg") 25 | .args(&[ 26 | "-y", "-i", input, "-vn", "-c:a", "aac", "-b:a", "192k", output, 27 | ]) 28 | .output()?; 29 | if !output.status.success() && running.load(Ordering::SeqCst) { 30 | return Err("Failed extracting audio".into()); 31 | } 32 | 33 | Ok(()) 34 | } 35 | 36 | /// Uses `ffmpeg` to locally split the video into chunks. 37 | pub fn split_video( 38 | input: &Path, 39 | output_dir: &Path, 40 | segment_length: Duration, 41 | running: &Arc, 42 | ) -> Result<()> { 43 | // Isolate file extension, since we want the chunks to have the same 44 | let extension = input 45 | .extension() 46 | .ok_or("Unable to find extension")? 47 | .to_str() 48 | .ok_or("Unable to convert OsString extension")?; 49 | // Convert input and output to &str 50 | let input = input.to_str().ok_or("Input invalid Unicode")?; 51 | let mut output_dir = output_dir.to_path_buf(); 52 | output_dir.push(format!("chunk_%03d.{}", extension)); 53 | let output = output_dir.to_str().ok_or("Output invalid Unicode")?; 54 | // Do the chunking 55 | let output = Command::new("ffmpeg") 56 | .args(&[ 57 | "-y", 58 | "-i", 59 | input, 60 | "-an", 61 | "-c", 62 | "copy", 63 | "-f", 64 | "segment", 65 | "-segment_time", 66 | &segment_length.as_secs().to_string(), 67 | output, 68 | ]) 69 | .output()?; 70 | if !output.status.success() && running.load(Ordering::SeqCst) { 71 | return Err("Failed splitting video".into()); 72 | } 73 | 74 | Ok(()) 75 | } 76 | 77 | /// Uses `ffmpeg` to locally combine the encoded chunks and audio. 78 | pub fn combine( 79 | encoded_dir: &Path, 80 | audio: &Path, 81 | output: &Path, 82 | running: &Arc, 83 | ) -> Result<()> { 84 | // Create list of encoded chunks 85 | let mut chunks = fs::read_dir(&encoded_dir)? 86 | .map(|res| res.and_then(|readdir| Ok(readdir.path()))) 87 | .map(|res| res.map_err(|e| e.into())) 88 | .map(|res| res.and_then(|path| Ok(path.into_os_string()))) 89 | .map(|res| { 90 | res.and_then(|os_string| { 91 | os_string 92 | .into_string() 93 | .map_err(|_| "Failed OsString conversion".into()) 94 | }) 95 | }) 96 | .map(|res| res.and_then(|file| Ok(format!("file '{}'\n", file)))) 97 | .collect::>>()?; 98 | // Sort them so we have the right order 99 | chunks.sort(); 100 | // And finally join them 101 | let chunks = chunks.join(""); 102 | // Now write that to a file 103 | let mut file_list = encoded_dir.to_path_buf(); 104 | file_list.push("files.txt"); 105 | fs::write(&file_list, chunks)?; 106 | 107 | // Convert paths to &str 108 | let audio = audio.to_str().ok_or("Audio invalid Unicode")?; 109 | let file_list = file_list.to_str().ok_or("File list invalid Unicode")?; 110 | let output = output.to_str().ok_or("Output invalid Unicode")?; 111 | // Combine everything 112 | let output = Command::new("ffmpeg") 113 | .args(&[ 114 | "-y", 115 | "-f", 116 | "concat", 117 | "-safe", 118 | "0", 119 | "-i", 120 | file_list, 121 | "-i", 122 | audio, 123 | "-c", 124 | "copy", 125 | "-movflags", 126 | "+faststart", 127 | output, 128 | ]) 129 | .output()?; 130 | if !output.status.success() && running.load(Ordering::SeqCst) { 131 | return Err("Failed combining video".into()); 132 | } 133 | 134 | Ok(()) 135 | } 136 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use clap::{App, AppSettings, Arg}; 2 | use log::error; 3 | use simplelog::{ 4 | ColorChoice, ConfigBuilder, LevelFilter, TermLogger, TerminalMode, 5 | }; 6 | use std::process; 7 | 8 | use shepherd; 9 | 10 | fn main() { 11 | let matches = App::new(clap::crate_name!()) 12 | .version(clap::crate_version!()) 13 | .about( 14 | "A distributed video encoder that splits files into chunks \ 15 | for multiple machines.", 16 | ) 17 | .author(clap::crate_authors!()) 18 | .setting(AppSettings::TrailingVarArg) 19 | .arg( 20 | Arg::with_name("clients") 21 | .short("c") 22 | .long("clients") 23 | .value_name("hostnames") 24 | .use_delimiter(true) 25 | .takes_value(true) 26 | .required(true) 27 | .help("Comma-separated list of encoding hosts"), 28 | ) 29 | .arg( 30 | Arg::with_name("length") 31 | .short("l") 32 | .long("length") 33 | .value_name("seconds") 34 | .takes_value(true) 35 | .help("The length of video chunks in seconds"), 36 | ) 37 | .arg( 38 | Arg::with_name("tmp") 39 | .short("t") 40 | .long("tmp") 41 | .value_name("path") 42 | .takes_value(true) 43 | .help("The path to the local temporary directory"), 44 | ) 45 | .arg( 46 | Arg::with_name("keep") 47 | .short("k") 48 | .long("keep") 49 | .help("Don't clean up temporary files"), 50 | ) 51 | .arg( 52 | Arg::with_name("IN") 53 | .help("The original video file") 54 | .required(true), 55 | ) 56 | .arg( 57 | Arg::with_name("OUT") 58 | .help("The output video file") 59 | .required(true), 60 | ) 61 | .arg( 62 | Arg::with_name("ffmpeg") 63 | .value_name("FFMPEG OPTIONS") 64 | .multiple(true) 65 | .help( 66 | "Options/flags for ffmpeg encoding of chunks. The\n\ 67 | chunks are video only, so don't pass in anything\n\ 68 | concerning audio. Input/output file names are added\n\ 69 | by the application, so there is no need for that\n\ 70 | either. This is the last positional argument and\n\ 71 | needs to be preceeded by double hypens (--) as in:\n\ 72 | shepherd -c c1,c2 in.mp4 out.mp4 -- -c:v libx264\n\ 73 | -crf 26 -preset veryslow -profile:v high -level 4.2\n\ 74 | -pix_fmt yuv420p\n\ 75 | This is also the default that is used if no options\n\ 76 | are provided.", 77 | ), 78 | ) 79 | .get_matches(); 80 | // If we get here, unwrap is safe on mandatory arguments 81 | let input = matches.value_of("IN").unwrap(); 82 | let output = matches.value_of("OUT").unwrap(); 83 | let hosts = matches.values_of("clients").unwrap().collect(); 84 | let seconds = matches.value_of("length"); 85 | let tmp = matches.value_of("tmp"); 86 | let keep = matches.is_present("keep"); 87 | // Take the given arguments for ffmpeg or use the defaults 88 | let args: Vec<&str> = matches 89 | .values_of("ffmpeg") 90 | .map(|a| a.collect()) 91 | .unwrap_or_else(|| { 92 | vec![ 93 | "-c:v", 94 | "libx264", 95 | "-crf", 96 | "26", 97 | "-preset", 98 | "veryslow", 99 | "-profile:v", 100 | "high", 101 | "-level", 102 | "4.2", 103 | "-pix_fmt", 104 | "yuv420p", 105 | ] 106 | }); 107 | 108 | TermLogger::init( 109 | LevelFilter::Info, 110 | ConfigBuilder::new() 111 | .set_time_offset_to_local() 112 | .expect("Unable to determine time offset") 113 | .build(), 114 | TerminalMode::Mixed, 115 | ColorChoice::Auto, 116 | ) 117 | .expect("Failed initializing logger"); 118 | 119 | if cfg!(debug_assertions) { 120 | shepherd::run(input, output, &args, hosts, seconds, tmp, keep) 121 | .unwrap(); 122 | } else if let Err(e) = 123 | shepherd::run(input, output, &args, hosts, seconds, tmp, keep) 124 | { 125 | error!("{}", e); 126 | process::exit(1); 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/remote.rs: -------------------------------------------------------------------------------- 1 | //! Functions for operations on remote hosts. 2 | 3 | use crossbeam::channel::{self, Receiver}; 4 | use log::{debug, info}; 5 | use std::{ 6 | path::PathBuf, 7 | process::Command, 8 | sync::atomic::{AtomicBool, Ordering}, 9 | sync::Arc, 10 | thread, 11 | }; 12 | 13 | /// The name of the temporary directory in the home directory of remote hosts. 14 | pub static TMP_DIR: &str = "shepherd_tmp_remote"; 15 | 16 | /// The parent thread managing the operations for a host. 17 | pub fn host_thread( 18 | host: String, 19 | global_receiver: Receiver, 20 | encoded_dir: PathBuf, 21 | out_ext: String, 22 | args: Arc>, 23 | running: Arc, 24 | ) { 25 | debug!("Spawned host thread {}", host); 26 | 27 | // Clean up temporary directory on host. This is necessary, because it's 28 | // possible that the user ran with the --keep flag before. While our 29 | // application wouldn't get confused by old chunks lying around (they're 30 | // overwritten and those that aren't are disregarded, because it keeps 31 | // track of its chunks), we don't want the user to get the wrong idea. 32 | // Also, we don't care if this fails, because if it did then the directory 33 | // didn't exist anyway. 34 | Command::new("ssh") 35 | .args(&[&host, "rm", "-r", crate::remote::TMP_DIR]) 36 | .output() 37 | .expect("Failed executing ssh command"); 38 | 39 | // Create temporary directory on host 40 | let output = Command::new("ssh") 41 | .args(&[&host, "mkdir", TMP_DIR]) 42 | .output() 43 | .expect("Failed executing ssh command"); 44 | assert!( 45 | output.status.success() || !running.load(Ordering::SeqCst), 46 | "Failed creating remote temporary directory" 47 | ); 48 | 49 | // Create a channel holding a single chunk at a time for the encoder thread 50 | let (sender, receiver) = channel::bounded(0); 51 | // Create copy of host for thread 52 | let host_cpy = host.clone(); 53 | // Increase reference counts for Arcs 54 | let r = Arc::clone(&running); 55 | let a = Arc::clone(&args); 56 | // Start the encoder thread 57 | let handle = thread::Builder::new() 58 | .name(format!("{}-encoder", host)) 59 | .spawn(move || encoder_thread(host_cpy, out_ext, a, receiver, r)) 60 | .expect("Failed spawning thread"); 61 | 62 | // Try to fetch a chunk from the global channel 63 | while let Ok(chunk) = global_receiver.recv() { 64 | debug!("Host thread {} received chunk {:?}", host, chunk); 65 | // Transfer chunk to host 66 | let output = Command::new("scp") 67 | .args(&[ 68 | chunk.to_str().expect("Invalid Unicode"), 69 | &format!("{}:{}", host, TMP_DIR), 70 | ]) 71 | .output() 72 | .expect("Failed executing scp command"); 73 | assert!( 74 | output.status.success() || !running.load(Ordering::SeqCst), 75 | "Failed transferring chunk" 76 | ); 77 | 78 | // Pass the chunk to the encoder thread (blocks until encoder is ready 79 | // to receive and fails if it terminated prematurely) 80 | if sender.send(chunk).is_err() { 81 | // Encoder stopped, so quit early 82 | break; 83 | } 84 | } 85 | // Since the global channel is empty, drop our sender to disconnect the 86 | // local channel 87 | drop(sender); 88 | debug!("Host thread {} waiting for encoder to finish", host); 89 | 90 | // Wait for the encoder 91 | let encoded = handle.join().expect("Encoder thread panicked"); 92 | // Abort early if signal was sent 93 | if !running.load(Ordering::SeqCst) { 94 | info!("{} exiting", host); 95 | return; 96 | } 97 | debug!("Host thread {} got encoded chunks {:?}", host, encoded); 98 | 99 | // Get a &str from encoded_dir PathBuf 100 | let encoded_dir = encoded_dir.to_str().expect("Invalid Unicode"); 101 | // Transfer the encoded chunks back 102 | for chunk in &encoded { 103 | let output = Command::new("scp") 104 | .args(&[&format!("{}:{}", host, chunk), encoded_dir]) 105 | .output() 106 | .expect("Failed executing scp command"); 107 | assert!( 108 | output.status.success() || !running.load(Ordering::SeqCst), 109 | "Failed transferring encoded chunk" 110 | ); 111 | info!("{} returned encoded chunk {}", host, chunk); 112 | } 113 | 114 | debug!("Host thread {} exiting", host); 115 | } 116 | 117 | /// Encodes chunks on a host and returns the encoded remote file names. 118 | fn encoder_thread( 119 | host: String, 120 | out_ext: String, 121 | args: Arc>, 122 | receiver: Receiver, 123 | running: Arc, 124 | ) -> Vec { 125 | // We'll use this to store the encoded chunks' remote file names. 126 | let mut encoded = Vec::new(); 127 | 128 | while let Ok(chunk) = receiver.recv() { 129 | // Abort early if signal was sent 130 | if !running.load(Ordering::SeqCst) { 131 | break; 132 | } 133 | 134 | debug!("Encoder thread {} received chunk {:?}", host, chunk); 135 | // Construct the chunk's remote file name 136 | let chunk_name = format!( 137 | "{}/{}", 138 | TMP_DIR, 139 | chunk 140 | .file_name() 141 | .expect("No normal file") 142 | .to_str() 143 | .expect("Invalid Unicode") 144 | ); 145 | // Construct the encoded chunk's remote file name 146 | let enc_name = format!( 147 | "{}/enc_{}.{}", 148 | TMP_DIR, 149 | chunk 150 | .file_stem() 151 | .expect("No normal file") 152 | .to_str() 153 | .expect("Invalid Unicode"), 154 | out_ext 155 | ); 156 | 157 | // Build the ffmpeg command 158 | let mut command: Vec<&str> = 159 | vec![&host, "ffmpeg", "-y", "-i", &chunk_name]; 160 | command.extend(args.iter().map(|s| s.as_str())); 161 | command.push(&enc_name); 162 | 163 | // Encode the chunk remotely 164 | info!("{} starts encoding chunk {:?}", host, chunk); 165 | let output = Command::new("ssh") 166 | .args(&command) 167 | .output() 168 | .expect("Failed executing ssh command"); 169 | assert!( 170 | output.status.success() || !running.load(Ordering::SeqCst), 171 | "Failed encoding" 172 | ); 173 | 174 | // Remember the encoded chunk 175 | encoded.push(enc_name); 176 | } 177 | debug!("Encoder thread {} exiting", host); 178 | 179 | encoded 180 | } 181 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # shepherd 2 | 3 | [![Latest version](https://img.shields.io/crates/v/shepherd)](https://crates.io/crates/shepherd) 4 | [![Documentation](https://docs.rs/shepherd/badge.svg)](https://docs.rs/shepherd) 5 | [![License](https://img.shields.io/crates/l/shepherd)](https://github.com/martindisch/shepherd#license) 6 | 7 | 8 | 9 | A distributed video encoder that splits files into chunks to encode them on 10 | multiple machines in parallel. 11 | 12 | ## Installation 13 | 14 | Using Cargo, you can do 15 | ```console 16 | $ cargo install shepherd 17 | ``` 18 | or just clone the repository and compile the binary with 19 | ```console 20 | $ git clone https://github.com/martindisch/shepherd 21 | $ cd shepherd 22 | $ cargo build --release 23 | ``` 24 | There's also a 25 | [direct download](https://github.com/martindisch/shepherd/releases/latest/download/shepherd) 26 | for the latest x86-64 ELF binary. 27 | 28 | ## Usage 29 | 30 | The prerequisites are one or more (you'll want more) computers—which we'll 31 | refer to as hosts—with `ffmpeg` installed and configured such that you can 32 | SSH into them directly. This means you'll have to `ssh-copy-id` your public 33 | key to them. I only tested it on Linux, but if you manage to set up 34 | `ffmpeg` and SSH, it might work on macOS or Windows directly or with little 35 | modification. 36 | 37 | The usage is pretty straightforward: 38 | ```text 39 | USAGE: 40 | shepherd [FLAGS] [OPTIONS] --clients [FFMPEG OPTIONS]... 41 | 42 | FLAGS: 43 | -h, --help Prints help information 44 | -k, --keep Don't clean up temporary files 45 | -V, --version Prints version information 46 | 47 | OPTIONS: 48 | -c, --clients Comma-separated list of encoding hosts 49 | -l, --length The length of video chunks in seconds 50 | -t, --tmp The path to the local temporary directory 51 | 52 | ARGS: 53 | The original video file 54 | The output video file 55 | ... Options/flags for ffmpeg encoding of chunks. The 56 | chunks are video only, so don't pass in anything 57 | concerning audio. Input/output file names are added 58 | by the application, so there is no need for that 59 | either. This is the last positional argument and 60 | needs to be preceeded by double hypens (--) as in: 61 | shepherd -c c1,c2 in.mp4 out.mp4 -- -c:v libx264 62 | -crf 26 -preset veryslow -profile:v high -level 4.2 63 | -pix_fmt yuv420p 64 | This is also the default that is used if no options 65 | are provided. 66 | ``` 67 | 68 | So if we have three machines c1, c2 and c3, we could do 69 | ```console 70 | $ shepherd -c c1,c2,c3 -l 30 source_file.mp4 output_file.mp4 71 | ``` 72 | to have it split the video in roughly 30 second chunks and encode them in 73 | parallel. By default it encodes in H.264 with a CRF value of 26 and the 74 | `veryslow` preset. If you want to supply your own `ffmpeg` options for more 75 | control over the codec, you can do so by adding them to the end of the 76 | invocation: 77 | ```console 78 | $ shepherd -c c1,c2 input.mkv output.mp4 -- -c:v libx264 -crf 40 79 | ``` 80 | 81 | ## How it works 82 | 83 | 1. Creates a temporary directory in your home directory. 84 | 2. Extracts the audio and encodes it. This is not parallelized, but the 85 | time this takes is negligible compared to the video anyway. 86 | 3. Splits the video into chunks. This can take relatively long, since 87 | you're basically writing the full file to disk again. It would be nice 88 | if we could read chunks of the file and directly transfer them to the 89 | hosts, but that might be tricky with `ffmpeg`. 90 | 4. Spawns a manager and an encoder thread for every host. The manager 91 | creates a temporary directory in the home directory of the remote and 92 | makes sure that the encoder always has something to encode. It will 93 | transfer a chunk, give it to the encoder to work on and meanwhile 94 | transfer another chunk, so the encoder can start directly with that once 95 | it's done, without wasting any time. But it will keep at most one chunk 96 | in reserve, to prevent the case where a slow machine takes too many 97 | chunks and is the only one still encoding while the faster ones are 98 | already done. 99 | 5. When an encoder is done and there are no more chunks to work on, it will 100 | quit and the manager transfers the encoded chunks back before 101 | terminating itself. 102 | 6. Once all encoded chunks have arrived, they're concatenated and the audio 103 | stream added. 104 | 7. All remote and the local temporary directory are removed. 105 | 106 | Thanks to the work stealing method of distribution, having some hosts that 107 | are significantly slower than others does not delay the overall operation. 108 | In the worst case, the slowest machine is the last to start encoding a 109 | chunk and remains the only working encoder for the duration it takes to 110 | encode this one chunk. This window can easily be reduced by using smaller 111 | chunks. 112 | 113 | ## Performance 114 | 115 | As with all things parallel, Amdahl's law rears its ugly head and you don't 116 | just get twice the speed with twice the processing power. With this 117 | approach, you pay for having to split the video into chunks before you 118 | begin, transferring them to the encoders and the results back, and 119 | reassembling them. Although I should clarify that transferring the chunks 120 | to the encoders only causes a noticeable delay until every encoder has its 121 | first chunk, the subsequent ones can be sent while the encoders are working 122 | so they don't waste time waiting for that. And returning and assembling the 123 | encoded chunks doesn't carry too big of a penalty, since we're dealing with 124 | much more compressed data then. 125 | 126 | To get a better understanding of the tradeoffs, I did some testing with a 127 | couple of computers I had access to. They were my main, pretty capable 128 | desktop, two older ones and a laptop. To figure out how capable each of 129 | them is so we can compare the actual to the expected speedup, I let each of 130 | them encode a relatively short clip of slightly less than 4 minutes taken 131 | from the real video I want to encode, using the same settings I'd use for 132 | the real job. And if you're wondering why encoding takes so long, it's 133 | because I'm using the `veryslow` preset for maximum efficiency, even though 134 | it's definitely not worth the huge increase in encoding time. But it's a 135 | nice simulation for how it would look if we were using an even more 136 | demanding codec like AV1. 137 | 138 | | machine | duration (s) | power | 139 | | --------- | ------------ | -------- | 140 | | desktop | 1373 | 1.000 | 141 | | old1 | 2571 | 0.53 | 142 | | old2 | 3292 | 0.42 | 143 | | laptop | 5572 | 0.25 | 144 | | **total** | - | **2.20** | 145 | 146 | By giving my desktop the "power" level 1, we can determine how powerful the 147 | others are at this encoding task, based on how long it takes them in 148 | comparison. By adding the three other, less capable machines to the mix, we 149 | slightly more than double the theoretical encoding capability of our 150 | system. 151 | 152 | I determined these power levels on a short clip, because encoding the full 153 | video would have taken very long on the less capable ones, especially the 154 | laptop. But I still needed to encode the full thing on at least one of them 155 | to make the comparison to the distributed encoding. I did that on my 156 | desktop since it's the fastest one, and to additionally verify that the 157 | power levels hold up for the full video, I bit the bullet and did the same 158 | on the second most powerful machine. 159 | 160 | | machine | duration (s) | power | 161 | | ------- | ------------- | ----- | 162 | | desktop | 9356 | 1.00 | 163 | | old1 | 17690 | 0.53 | 164 | 165 | Now we have the baseline we want to beat with parallel encoding, as well as 166 | confirmation that the power levels are valid for the full video. Let's see 167 | how much of the theoretical, but unreachable 2.2x speedup we can get. 168 | 169 | Encoding the video in parallel took 5283 seconds, so 56.5% of the time 170 | using my fastest computer, or a 1.77x speedup. We committed about twice the 171 | computing power and we're not too far off that two times speedup. It's 172 | making use of the additionally available resources with an 80% efficiency 173 | in this case. I also tried to encode the short clip in parallel, which was 174 | very fast, but had a somewhat disappointing speedup of only 1.32x. I 175 | suspect that we get better results with longer videos, since encoding a 176 | chunk always takes longer than creating and transferring it (otherwise 177 | distributing wouldn't make sense at all). The longer the video then, the 178 | larger the ratio of encoding (which we can parallelize) in the total amount 179 | of time the process takes, and the more effective doing so becomes. 180 | 181 | I've also looked at how the work is distributed over the nodes, depending 182 | on their processing power. At the end of a parallel encode, it's possible 183 | to determine how many chunks have been encoded by any given host. 184 | 185 | | host | chunks | power | 186 | | ------------- | ------ | ----- | 187 | | desktop | 73 | 1.00 | 188 | | old1 | 39 | 0.53 | 189 | | old2 | 31 | 0.42 | 190 | | laptop | 19 | 0.26 | 191 | 192 | Inferring the processing power from the number of chunks leads to almost 193 | exactly the same results as my initial determination, confirming it and 194 | proving that work is distributed efficiently. 195 | 196 | To further see how the system scales, I've added two more machines, 197 | bringing the total processing power up to 3.29. 198 | 199 | | machine | duration (s) | power | 200 | | --------- | ------------ | -------- | 201 | | desktop | 1373 | 1.00 | 202 | | c1 | 2129 | 0.64 | 203 | | old1 | 2571 | 0.53 | 204 | | c2 | 3022 | 0.45 | 205 | | old2 | 3292 | 0.42 | 206 | | laptop | 5572 | 0.25 | 207 | | **total** | - | **3.29** | 208 | 209 | Encoding the video on these 6 machines in parallel took 3865 seconds, so 210 | 41.3% of the time using my fastest computer, or a 2.42x speedup. It's 211 | making use of the additionally available resources with a 74% efficiency 212 | here. As expected, while we can accelerate by adding more resources, we're 213 | looking at diminishing returns. Although the factor by which the efficiency 214 | decreases is not as bad as it could be. 215 | 216 | ## Limitations 217 | 218 | While you can use your own `ffmpeg` options to control how the video is 219 | encoded, there is currently no such option for the audio, which is 192 kb/s 220 | AAC by default. 221 | 222 | 223 | 224 | ## License 225 | Licensed under either of 226 | 227 | * [Apache License, Version 2.0](LICENSE-APACHE) 228 | * [MIT license](LICENSE-MIT) 229 | 230 | at your option. 231 | 232 | ### Contribution 233 | 234 | Unless you explicitly state otherwise, any contribution intentionally submitted 235 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 236 | be dual licensed as above, without any additional terms or conditions. 237 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "ansi_term" 7 | version = "0.12.1" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" 10 | dependencies = [ 11 | "winapi", 12 | ] 13 | 14 | [[package]] 15 | name = "atty" 16 | version = "0.2.14" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 19 | dependencies = [ 20 | "hermit-abi", 21 | "libc", 22 | "winapi", 23 | ] 24 | 25 | [[package]] 26 | name = "autocfg" 27 | version = "1.1.0" 28 | source = "registry+https://github.com/rust-lang/crates.io-index" 29 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 30 | 31 | [[package]] 32 | name = "bitflags" 33 | version = "1.3.2" 34 | source = "registry+https://github.com/rust-lang/crates.io-index" 35 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 36 | 37 | [[package]] 38 | name = "cfg-if" 39 | version = "1.0.0" 40 | source = "registry+https://github.com/rust-lang/crates.io-index" 41 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 42 | 43 | [[package]] 44 | name = "clap" 45 | version = "2.34.0" 46 | source = "registry+https://github.com/rust-lang/crates.io-index" 47 | checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" 48 | dependencies = [ 49 | "ansi_term", 50 | "atty", 51 | "bitflags", 52 | "strsim", 53 | "textwrap", 54 | "unicode-width", 55 | "vec_map", 56 | ] 57 | 58 | [[package]] 59 | name = "crossbeam" 60 | version = "0.8.2" 61 | source = "registry+https://github.com/rust-lang/crates.io-index" 62 | checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" 63 | dependencies = [ 64 | "cfg-if", 65 | "crossbeam-channel", 66 | "crossbeam-deque", 67 | "crossbeam-epoch", 68 | "crossbeam-queue", 69 | "crossbeam-utils", 70 | ] 71 | 72 | [[package]] 73 | name = "crossbeam-channel" 74 | version = "0.5.6" 75 | source = "registry+https://github.com/rust-lang/crates.io-index" 76 | checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" 77 | dependencies = [ 78 | "cfg-if", 79 | "crossbeam-utils", 80 | ] 81 | 82 | [[package]] 83 | name = "crossbeam-deque" 84 | version = "0.8.2" 85 | source = "registry+https://github.com/rust-lang/crates.io-index" 86 | checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" 87 | dependencies = [ 88 | "cfg-if", 89 | "crossbeam-epoch", 90 | "crossbeam-utils", 91 | ] 92 | 93 | [[package]] 94 | name = "crossbeam-epoch" 95 | version = "0.9.10" 96 | source = "registry+https://github.com/rust-lang/crates.io-index" 97 | checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1" 98 | dependencies = [ 99 | "autocfg", 100 | "cfg-if", 101 | "crossbeam-utils", 102 | "memoffset", 103 | "once_cell", 104 | "scopeguard", 105 | ] 106 | 107 | [[package]] 108 | name = "crossbeam-queue" 109 | version = "0.3.6" 110 | source = "registry+https://github.com/rust-lang/crates.io-index" 111 | checksum = "1cd42583b04998a5363558e5f9291ee5a5ff6b49944332103f251e7479a82aa7" 112 | dependencies = [ 113 | "cfg-if", 114 | "crossbeam-utils", 115 | ] 116 | 117 | [[package]] 118 | name = "crossbeam-utils" 119 | version = "0.8.11" 120 | source = "registry+https://github.com/rust-lang/crates.io-index" 121 | checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" 122 | dependencies = [ 123 | "cfg-if", 124 | "once_cell", 125 | ] 126 | 127 | [[package]] 128 | name = "ctrlc" 129 | version = "3.2.3" 130 | source = "registry+https://github.com/rust-lang/crates.io-index" 131 | checksum = "1d91974fbbe88ec1df0c24a4f00f99583667a7e2e6272b2b92d294d81e462173" 132 | dependencies = [ 133 | "nix", 134 | "winapi", 135 | ] 136 | 137 | [[package]] 138 | name = "dirs" 139 | version = "4.0.0" 140 | source = "registry+https://github.com/rust-lang/crates.io-index" 141 | checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" 142 | dependencies = [ 143 | "dirs-sys", 144 | ] 145 | 146 | [[package]] 147 | name = "dirs-sys" 148 | version = "0.3.7" 149 | source = "registry+https://github.com/rust-lang/crates.io-index" 150 | checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" 151 | dependencies = [ 152 | "libc", 153 | "redox_users", 154 | "winapi", 155 | ] 156 | 157 | [[package]] 158 | name = "getrandom" 159 | version = "0.2.7" 160 | source = "registry+https://github.com/rust-lang/crates.io-index" 161 | checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" 162 | dependencies = [ 163 | "cfg-if", 164 | "libc", 165 | "wasi", 166 | ] 167 | 168 | [[package]] 169 | name = "hermit-abi" 170 | version = "0.1.19" 171 | source = "registry+https://github.com/rust-lang/crates.io-index" 172 | checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" 173 | dependencies = [ 174 | "libc", 175 | ] 176 | 177 | [[package]] 178 | name = "itoa" 179 | version = "1.0.3" 180 | source = "registry+https://github.com/rust-lang/crates.io-index" 181 | checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" 182 | 183 | [[package]] 184 | name = "libc" 185 | version = "0.2.133" 186 | source = "registry+https://github.com/rust-lang/crates.io-index" 187 | checksum = "c0f80d65747a3e43d1596c7c5492d95d5edddaabd45a7fcdb02b95f644164966" 188 | 189 | [[package]] 190 | name = "log" 191 | version = "0.4.17" 192 | source = "registry+https://github.com/rust-lang/crates.io-index" 193 | checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" 194 | dependencies = [ 195 | "cfg-if", 196 | ] 197 | 198 | [[package]] 199 | name = "memoffset" 200 | version = "0.6.5" 201 | source = "registry+https://github.com/rust-lang/crates.io-index" 202 | checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" 203 | dependencies = [ 204 | "autocfg", 205 | ] 206 | 207 | [[package]] 208 | name = "nix" 209 | version = "0.25.0" 210 | source = "registry+https://github.com/rust-lang/crates.io-index" 211 | checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" 212 | dependencies = [ 213 | "autocfg", 214 | "bitflags", 215 | "cfg-if", 216 | "libc", 217 | ] 218 | 219 | [[package]] 220 | name = "num_threads" 221 | version = "0.1.6" 222 | source = "registry+https://github.com/rust-lang/crates.io-index" 223 | checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" 224 | dependencies = [ 225 | "libc", 226 | ] 227 | 228 | [[package]] 229 | name = "once_cell" 230 | version = "1.15.0" 231 | source = "registry+https://github.com/rust-lang/crates.io-index" 232 | checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" 233 | 234 | [[package]] 235 | name = "proc-macro2" 236 | version = "1.0.44" 237 | source = "registry+https://github.com/rust-lang/crates.io-index" 238 | checksum = "7bd7356a8122b6c4a24a82b278680c73357984ca2fc79a0f9fa6dea7dced7c58" 239 | dependencies = [ 240 | "unicode-ident", 241 | ] 242 | 243 | [[package]] 244 | name = "quote" 245 | version = "1.0.21" 246 | source = "registry+https://github.com/rust-lang/crates.io-index" 247 | checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" 248 | dependencies = [ 249 | "proc-macro2", 250 | ] 251 | 252 | [[package]] 253 | name = "redox_syscall" 254 | version = "0.2.16" 255 | source = "registry+https://github.com/rust-lang/crates.io-index" 256 | checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" 257 | dependencies = [ 258 | "bitflags", 259 | ] 260 | 261 | [[package]] 262 | name = "redox_users" 263 | version = "0.4.3" 264 | source = "registry+https://github.com/rust-lang/crates.io-index" 265 | checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" 266 | dependencies = [ 267 | "getrandom", 268 | "redox_syscall", 269 | "thiserror", 270 | ] 271 | 272 | [[package]] 273 | name = "scopeguard" 274 | version = "1.1.0" 275 | source = "registry+https://github.com/rust-lang/crates.io-index" 276 | checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" 277 | 278 | [[package]] 279 | name = "shepherd" 280 | version = "0.1.5" 281 | dependencies = [ 282 | "clap", 283 | "crossbeam", 284 | "ctrlc", 285 | "dirs", 286 | "log", 287 | "simplelog", 288 | ] 289 | 290 | [[package]] 291 | name = "simplelog" 292 | version = "0.12.0" 293 | source = "registry+https://github.com/rust-lang/crates.io-index" 294 | checksum = "48dfff04aade74dd495b007c831cd6f4e0cee19c344dd9dc0884c0289b70a786" 295 | dependencies = [ 296 | "log", 297 | "termcolor", 298 | "time", 299 | ] 300 | 301 | [[package]] 302 | name = "strsim" 303 | version = "0.8.0" 304 | source = "registry+https://github.com/rust-lang/crates.io-index" 305 | checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" 306 | 307 | [[package]] 308 | name = "syn" 309 | version = "1.0.100" 310 | source = "registry+https://github.com/rust-lang/crates.io-index" 311 | checksum = "52205623b1b0f064a4e71182c3b18ae902267282930c6d5462c91b859668426e" 312 | dependencies = [ 313 | "proc-macro2", 314 | "quote", 315 | "unicode-ident", 316 | ] 317 | 318 | [[package]] 319 | name = "termcolor" 320 | version = "1.1.3" 321 | source = "registry+https://github.com/rust-lang/crates.io-index" 322 | checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" 323 | dependencies = [ 324 | "winapi-util", 325 | ] 326 | 327 | [[package]] 328 | name = "textwrap" 329 | version = "0.11.0" 330 | source = "registry+https://github.com/rust-lang/crates.io-index" 331 | checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" 332 | dependencies = [ 333 | "unicode-width", 334 | ] 335 | 336 | [[package]] 337 | name = "thiserror" 338 | version = "1.0.36" 339 | source = "registry+https://github.com/rust-lang/crates.io-index" 340 | checksum = "0a99cb8c4b9a8ef0e7907cd3b617cc8dc04d571c4e73c8ae403d80ac160bb122" 341 | dependencies = [ 342 | "thiserror-impl", 343 | ] 344 | 345 | [[package]] 346 | name = "thiserror-impl" 347 | version = "1.0.36" 348 | source = "registry+https://github.com/rust-lang/crates.io-index" 349 | checksum = "3a891860d3c8d66fec8e73ddb3765f90082374dbaaa833407b904a94f1a7eb43" 350 | dependencies = [ 351 | "proc-macro2", 352 | "quote", 353 | "syn", 354 | ] 355 | 356 | [[package]] 357 | name = "time" 358 | version = "0.3.14" 359 | source = "registry+https://github.com/rust-lang/crates.io-index" 360 | checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b" 361 | dependencies = [ 362 | "itoa", 363 | "libc", 364 | "num_threads", 365 | "time-macros", 366 | ] 367 | 368 | [[package]] 369 | name = "time-macros" 370 | version = "0.2.4" 371 | source = "registry+https://github.com/rust-lang/crates.io-index" 372 | checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" 373 | 374 | [[package]] 375 | name = "unicode-ident" 376 | version = "1.0.4" 377 | source = "registry+https://github.com/rust-lang/crates.io-index" 378 | checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" 379 | 380 | [[package]] 381 | name = "unicode-width" 382 | version = "0.1.10" 383 | source = "registry+https://github.com/rust-lang/crates.io-index" 384 | checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" 385 | 386 | [[package]] 387 | name = "vec_map" 388 | version = "0.8.2" 389 | source = "registry+https://github.com/rust-lang/crates.io-index" 390 | checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" 391 | 392 | [[package]] 393 | name = "wasi" 394 | version = "0.11.0+wasi-snapshot-preview1" 395 | source = "registry+https://github.com/rust-lang/crates.io-index" 396 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 397 | 398 | [[package]] 399 | name = "winapi" 400 | version = "0.3.9" 401 | source = "registry+https://github.com/rust-lang/crates.io-index" 402 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 403 | dependencies = [ 404 | "winapi-i686-pc-windows-gnu", 405 | "winapi-x86_64-pc-windows-gnu", 406 | ] 407 | 408 | [[package]] 409 | name = "winapi-i686-pc-windows-gnu" 410 | version = "0.4.0" 411 | source = "registry+https://github.com/rust-lang/crates.io-index" 412 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 413 | 414 | [[package]] 415 | name = "winapi-util" 416 | version = "0.1.5" 417 | source = "registry+https://github.com/rust-lang/crates.io-index" 418 | checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" 419 | dependencies = [ 420 | "winapi", 421 | ] 422 | 423 | [[package]] 424 | name = "winapi-x86_64-pc-windows-gnu" 425 | version = "0.4.0" 426 | source = "registry+https://github.com/rust-lang/crates.io-index" 427 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 428 | -------------------------------------------------------------------------------- /LICENSE-3RD-PARTY: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------------------------------- 2 | The MIT License 3 | 4 | applies to: 5 | - clap, Copyright (c) 2015-2016 Kevin B. Knapp 6 | - dirs, Copyright (c) 2018-2019 dirs-rs contributors 7 | - crossbeam, Copyright (c) 2019 The Crossbeam Project Developers 8 | - log, Copyright (c) 2014 The Rust Project Developers 9 | - simplelog, Copyright (c) 2015 Victor Brekenfeld 10 | - ctrlc 11 | ------------------------------------------------------------------------------- 12 | Permission is hereby granted, free of charge, to any person obtaining a copy 13 | of this software and associated documentation files (the "Software"), to deal 14 | in the Software without restriction, including without limitation the rights 15 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 | copies of the Software, and to permit persons to whom the Software is 17 | furnished to do so, subject to the following conditions: 18 | 19 | The above copyright notice and this permission notice shall be included in all 20 | copies or substantial portions of the Software. 21 | 22 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 28 | SOFTWARE. 29 | 30 | ------------------------------------------------------------------------------- 31 | The Apache License 32 | 33 | applies to: 34 | - dirs 35 | - crossbeam 36 | - log 37 | - simplelog 38 | - ctrlc 39 | ------------------------------------------------------------------------------- 40 | Apache License 41 | Version 2.0, January 2004 42 | http://www.apache.org/licenses/ 43 | 44 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 45 | 46 | 1. Definitions. 47 | 48 | "License" shall mean the terms and conditions for use, reproduction, 49 | and distribution as defined by Sections 1 through 9 of this document. 50 | 51 | "Licensor" shall mean the copyright owner or entity authorized by 52 | the copyright owner that is granting the License. 53 | 54 | "Legal Entity" shall mean the union of the acting entity and all 55 | other entities that control, are controlled by, or are under common 56 | control with that entity. For the purposes of this definition, 57 | "control" means (i) the power, direct or indirect, to cause the 58 | direction or management of such entity, whether by contract or 59 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 60 | outstanding shares, or (iii) beneficial ownership of such entity. 61 | 62 | "You" (or "Your") shall mean an individual or Legal Entity 63 | exercising permissions granted by this License. 64 | 65 | "Source" form shall mean the preferred form for making modifications, 66 | including but not limited to software source code, documentation 67 | source, and configuration files. 68 | 69 | "Object" form shall mean any form resulting from mechanical 70 | transformation or translation of a Source form, including but 71 | not limited to compiled object code, generated documentation, 72 | and conversions to other media types. 73 | 74 | "Work" shall mean the work of authorship, whether in Source or 75 | Object form, made available under the License, as indicated by a 76 | copyright notice that is included in or attached to the work 77 | (an example is provided in the Appendix below). 78 | 79 | "Derivative Works" shall mean any work, whether in Source or Object 80 | form, that is based on (or derived from) the Work and for which the 81 | editorial revisions, annotations, elaborations, or other modifications 82 | represent, as a whole, an original work of authorship. For the purposes 83 | of this License, Derivative Works shall not include works that remain 84 | separable from, or merely link (or bind by name) to the interfaces of, 85 | the Work and Derivative Works thereof. 86 | 87 | "Contribution" shall mean any work of authorship, including 88 | the original version of the Work and any modifications or additions 89 | to that Work or Derivative Works thereof, that is intentionally 90 | submitted to Licensor for inclusion in the Work by the copyright owner 91 | or by an individual or Legal Entity authorized to submit on behalf of 92 | the copyright owner. For the purposes of this definition, "submitted" 93 | means any form of electronic, verbal, or written communication sent 94 | to the Licensor or its representatives, including but not limited to 95 | communication on electronic mailing lists, source code control systems, 96 | and issue tracking systems that are managed by, or on behalf of, the 97 | Licensor for the purpose of discussing and improving the Work, but 98 | excluding communication that is conspicuously marked or otherwise 99 | designated in writing by the copyright owner as "Not a Contribution." 100 | 101 | "Contributor" shall mean Licensor and any individual or Legal Entity 102 | on behalf of whom a Contribution has been received by Licensor and 103 | subsequently incorporated within the Work. 104 | 105 | 2. Grant of Copyright License. Subject to the terms and conditions of 106 | this License, each Contributor hereby grants to You a perpetual, 107 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 108 | copyright license to reproduce, prepare Derivative Works of, 109 | publicly display, publicly perform, sublicense, and distribute the 110 | Work and such Derivative Works in Source or Object form. 111 | 112 | 3. Grant of Patent License. Subject to the terms and conditions of 113 | this License, each Contributor hereby grants to You a perpetual, 114 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 115 | (except as stated in this section) patent license to make, have made, 116 | use, offer to sell, sell, import, and otherwise transfer the Work, 117 | where such license applies only to those patent claims licensable 118 | by such Contributor that are necessarily infringed by their 119 | Contribution(s) alone or by combination of their Contribution(s) 120 | with the Work to which such Contribution(s) was submitted. If You 121 | institute patent litigation against any entity (including a 122 | cross-claim or counterclaim in a lawsuit) alleging that the Work 123 | or a Contribution incorporated within the Work constitutes direct 124 | or contributory patent infringement, then any patent licenses 125 | granted to You under this License for that Work shall terminate 126 | as of the date such litigation is filed. 127 | 128 | 4. Redistribution. You may reproduce and distribute copies of the 129 | Work or Derivative Works thereof in any medium, with or without 130 | modifications, and in Source or Object form, provided that You 131 | meet the following conditions: 132 | 133 | (a) You must give any other recipients of the Work or 134 | Derivative Works a copy of this License; and 135 | 136 | (b) You must cause any modified files to carry prominent notices 137 | stating that You changed the files; and 138 | 139 | (c) You must retain, in the Source form of any Derivative Works 140 | that You distribute, all copyright, patent, trademark, and 141 | attribution notices from the Source form of the Work, 142 | excluding those notices that do not pertain to any part of 143 | the Derivative Works; and 144 | 145 | (d) If the Work includes a "NOTICE" text file as part of its 146 | distribution, then any Derivative Works that You distribute must 147 | include a readable copy of the attribution notices contained 148 | within such NOTICE file, excluding those notices that do not 149 | pertain to any part of the Derivative Works, in at least one 150 | of the following places: within a NOTICE text file distributed 151 | as part of the Derivative Works; within the Source form or 152 | documentation, if provided along with the Derivative Works; or, 153 | within a display generated by the Derivative Works, if and 154 | wherever such third-party notices normally appear. The contents 155 | of the NOTICE file are for informational purposes only and 156 | do not modify the License. You may add Your own attribution 157 | notices within Derivative Works that You distribute, alongside 158 | or as an addendum to the NOTICE text from the Work, provided 159 | that such additional attribution notices cannot be construed 160 | as modifying the License. 161 | 162 | You may add Your own copyright statement to Your modifications and 163 | may provide additional or different license terms and conditions 164 | for use, reproduction, or distribution of Your modifications, or 165 | for any such Derivative Works as a whole, provided Your use, 166 | reproduction, and distribution of the Work otherwise complies with 167 | the conditions stated in this License. 168 | 169 | 5. Submission of Contributions. Unless You explicitly state otherwise, 170 | any Contribution intentionally submitted for inclusion in the Work 171 | by You to the Licensor shall be under the terms and conditions of 172 | this License, without any additional terms or conditions. 173 | Notwithstanding the above, nothing herein shall supersede or modify 174 | the terms of any separate license agreement you may have executed 175 | with Licensor regarding such Contributions. 176 | 177 | 6. Trademarks. This License does not grant permission to use the trade 178 | names, trademarks, service marks, or product names of the Licensor, 179 | except as required for reasonable and customary use in describing the 180 | origin of the Work and reproducing the content of the NOTICE file. 181 | 182 | 7. Disclaimer of Warranty. Unless required by applicable law or 183 | agreed to in writing, Licensor provides the Work (and each 184 | Contributor provides its Contributions) on an "AS IS" BASIS, 185 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 186 | implied, including, without limitation, any warranties or conditions 187 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 188 | PARTICULAR PURPOSE. You are solely responsible for determining the 189 | appropriateness of using or redistributing the Work and assume any 190 | risks associated with Your exercise of permissions under this License. 191 | 192 | 8. Limitation of Liability. In no event and under no legal theory, 193 | whether in tort (including negligence), contract, or otherwise, 194 | unless required by applicable law (such as deliberate and grossly 195 | negligent acts) or agreed to in writing, shall any Contributor be 196 | liable to You for damages, including any direct, indirect, special, 197 | incidental, or consequential damages of any character arising as a 198 | result of this License or out of the use or inability to use the 199 | Work (including but not limited to damages for loss of goodwill, 200 | work stoppage, computer failure or malfunction, or any and all 201 | other commercial damages or losses), even if such Contributor 202 | has been advised of the possibility of such damages. 203 | 204 | 9. Accepting Warranty or Additional Liability. While redistributing 205 | the Work or Derivative Works thereof, You may choose to offer, 206 | and charge a fee for, acceptance of support, warranty, indemnity, 207 | or other liability obligations and/or rights consistent with this 208 | License. However, in accepting such obligations, You may act only 209 | on Your own behalf and on Your sole responsibility, not on behalf 210 | of any other Contributor, and only if You agree to indemnify, 211 | defend, and hold each Contributor harmless for any liability 212 | incurred by, or claims asserted against, such Contributor by reason 213 | of your accepting any such warranty or additional liability. 214 | 215 | END OF TERMS AND CONDITIONS 216 | 217 | APPENDIX: How to apply the Apache License to your work. 218 | 219 | To apply the Apache License to your work, attach the following 220 | boilerplate notice, with the fields enclosed by brackets "[]" 221 | replaced with your own identifying information. (Don't include 222 | the brackets!) The text should be enclosed in the appropriate 223 | comment syntax for the file format. We also recommend that a 224 | file or class name and description of purpose be included on the 225 | same "printed page" as the copyright notice for easier 226 | identification within third-party archives. 227 | 228 | Copyright [yyyy] [name of copyright owner] 229 | 230 | Licensed under the Apache License, Version 2.0 (the "License"); 231 | you may not use this file except in compliance with the License. 232 | You may obtain a copy of the License at 233 | 234 | http://www.apache.org/licenses/LICENSE-2.0 235 | 236 | Unless required by applicable law or agreed to in writing, software 237 | distributed under the License is distributed on an "AS IS" BASIS, 238 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 239 | See the License for the specific language governing permissions and 240 | limitations under the License. 241 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! A distributed video encoder that splits files into chunks to encode them on 2 | //! multiple machines in parallel. 3 | //! 4 | //! ## Installation 5 | //! 6 | //! Using Cargo, you can do 7 | //! ```console 8 | //! $ cargo install shepherd 9 | //! ``` 10 | //! or just clone the repository and compile the binary with 11 | //! ```console 12 | //! $ git clone https://github.com/martindisch/shepherd 13 | //! $ cd shepherd 14 | //! $ cargo build --release 15 | //! ``` 16 | //! There's also a 17 | //! [direct download](https://github.com/martindisch/shepherd/releases/latest/download/shepherd) 18 | //! for the latest x86-64 ELF binary. 19 | //! 20 | //! ## Usage 21 | //! 22 | //! The prerequisites are one or more (you'll want more) computers—which we'll 23 | //! refer to as hosts—with `ffmpeg` installed and configured such that you can 24 | //! SSH into them directly. This means you'll have to `ssh-copy-id` your public 25 | //! key to them. I only tested it on Linux, but if you manage to set up 26 | //! `ffmpeg` and SSH, it might work on macOS or Windows directly or with little 27 | //! modification. 28 | //! 29 | //! The usage is pretty straightforward: 30 | //! ```text 31 | //! USAGE: 32 | //! shepherd [FLAGS] [OPTIONS] --clients [FFMPEG OPTIONS]... 33 | //! 34 | //! FLAGS: 35 | //! -h, --help Prints help information 36 | //! -k, --keep Don't clean up temporary files 37 | //! -V, --version Prints version information 38 | //! 39 | //! OPTIONS: 40 | //! -c, --clients Comma-separated list of encoding hosts 41 | //! -l, --length The length of video chunks in seconds 42 | //! -t, --tmp The path to the local temporary directory 43 | //! 44 | //! ARGS: 45 | //! The original video file 46 | //! The output video file 47 | //! ... Options/flags for ffmpeg encoding of chunks. The 48 | //! chunks are video only, so don't pass in anything 49 | //! concerning audio. Input/output file names are added 50 | //! by the application, so there is no need for that 51 | //! either. This is the last positional argument and 52 | //! needs to be preceeded by double hypens (--) as in: 53 | //! shepherd -c c1,c2 in.mp4 out.mp4 -- -c:v libx264 54 | //! -crf 26 -preset veryslow -profile:v high -level 4.2 55 | //! -pix_fmt yuv420p 56 | //! This is also the default that is used if no options 57 | //! are provided. 58 | //! ``` 59 | //! 60 | //! So if we have three machines c1, c2 and c3, we could do 61 | //! ```console 62 | //! $ shepherd -c c1,c2,c3 -l 30 source_file.mp4 output_file.mp4 63 | //! ``` 64 | //! to have it split the video in roughly 30 second chunks and encode them in 65 | //! parallel. By default it encodes in H.264 with a CRF value of 26 and the 66 | //! `veryslow` preset. If you want to supply your own `ffmpeg` options for more 67 | //! control over the codec, you can do so by adding them to the end of the 68 | //! invocation: 69 | //! ```console 70 | //! $ shepherd -c c1,c2 input.mkv output.mp4 -- -c:v libx264 -crf 40 71 | //! ``` 72 | //! 73 | //! ## How it works 74 | //! 75 | //! 1. Creates a temporary directory in your home directory. 76 | //! 2. Extracts the audio and encodes it. This is not parallelized, but the 77 | //! time this takes is negligible compared to the video anyway. 78 | //! 3. Splits the video into chunks. This can take relatively long, since 79 | //! you're basically writing the full file to disk again. It would be nice 80 | //! if we could read chunks of the file and directly transfer them to the 81 | //! hosts, but that might be tricky with `ffmpeg`. 82 | //! 4. Spawns a manager and an encoder thread for every host. The manager 83 | //! creates a temporary directory in the home directory of the remote and 84 | //! makes sure that the encoder always has something to encode. It will 85 | //! transfer a chunk, give it to the encoder to work on and meanwhile 86 | //! transfer another chunk, so the encoder can start directly with that once 87 | //! it's done, without wasting any time. But it will keep at most one chunk 88 | //! in reserve, to prevent the case where a slow machine takes too many 89 | //! chunks and is the only one still encoding while the faster ones are 90 | //! already done. 91 | //! 5. When an encoder is done and there are no more chunks to work on, it will 92 | //! quit and the manager transfers the encoded chunks back before 93 | //! terminating itself. 94 | //! 6. Once all encoded chunks have arrived, they're concatenated and the audio 95 | //! stream added. 96 | //! 7. All remote and the local temporary directory are removed. 97 | //! 98 | //! Thanks to the work stealing method of distribution, having some hosts that 99 | //! are significantly slower than others does not delay the overall operation. 100 | //! In the worst case, the slowest machine is the last to start encoding a 101 | //! chunk and remains the only working encoder for the duration it takes to 102 | //! encode this one chunk. This window can easily be reduced by using smaller 103 | //! chunks. 104 | //! 105 | //! ## Performance 106 | //! 107 | //! As with all things parallel, Amdahl's law rears its ugly head and you don't 108 | //! just get twice the speed with twice the processing power. With this 109 | //! approach, you pay for having to split the video into chunks before you 110 | //! begin, transferring them to the encoders and the results back, and 111 | //! reassembling them. Although I should clarify that transferring the chunks 112 | //! to the encoders only causes a noticeable delay until every encoder has its 113 | //! first chunk, the subsequent ones can be sent while the encoders are working 114 | //! so they don't waste time waiting for that. And returning and assembling the 115 | //! encoded chunks doesn't carry too big of a penalty, since we're dealing with 116 | //! much more compressed data then. 117 | //! 118 | //! To get a better understanding of the tradeoffs, I did some testing with a 119 | //! couple of computers I had access to. They were my main, pretty capable 120 | //! desktop, two older ones and a laptop. To figure out how capable each of 121 | //! them is so we can compare the actual to the expected speedup, I let each of 122 | //! them encode a relatively short clip of slightly less than 4 minutes taken 123 | //! from the real video I want to encode, using the same settings I'd use for 124 | //! the real job. And if you're wondering why encoding takes so long, it's 125 | //! because I'm using the `veryslow` preset for maximum efficiency, even though 126 | //! it's definitely not worth the huge increase in encoding time. But it's a 127 | //! nice simulation for how it would look if we were using an even more 128 | //! demanding codec like AV1. 129 | //! 130 | //! | machine | duration (s) | power | 131 | //! | --------- | ------------ | -------- | 132 | //! | desktop | 1373 | 1.000 | 133 | //! | old1 | 2571 | 0.53 | 134 | //! | old2 | 3292 | 0.42 | 135 | //! | laptop | 5572 | 0.25 | 136 | //! | **total** | - | **2.20** | 137 | //! 138 | //! By giving my desktop the "power" level 1, we can determine how powerful the 139 | //! others are at this encoding task, based on how long it takes them in 140 | //! comparison. By adding the three other, less capable machines to the mix, we 141 | //! slightly more than double the theoretical encoding capability of our 142 | //! system. 143 | //! 144 | //! I determined these power levels on a short clip, because encoding the full 145 | //! video would have taken very long on the less capable ones, especially the 146 | //! laptop. But I still needed to encode the full thing on at least one of them 147 | //! to make the comparison to the distributed encoding. I did that on my 148 | //! desktop since it's the fastest one, and to additionally verify that the 149 | //! power levels hold up for the full video, I bit the bullet and did the same 150 | //! on the second most powerful machine. 151 | //! 152 | //! | machine | duration (s) | power | 153 | //! | ------- | ------------- | ----- | 154 | //! | desktop | 9356 | 1.00 | 155 | //! | old1 | 17690 | 0.53 | 156 | //! 157 | //! Now we have the baseline we want to beat with parallel encoding, as well as 158 | //! confirmation that the power levels are valid for the full video. Let's see 159 | //! how much of the theoretical, but unreachable 2.2x speedup we can get. 160 | //! 161 | //! Encoding the video in parallel took 5283 seconds, so 56.5% of the time 162 | //! using my fastest computer, or a 1.77x speedup. We committed about twice the 163 | //! computing power and we're not too far off that two times speedup. It's 164 | //! making use of the additionally available resources with an 80% efficiency 165 | //! in this case. I also tried to encode the short clip in parallel, which was 166 | //! very fast, but had a somewhat disappointing speedup of only 1.32x. I 167 | //! suspect that we get better results with longer videos, since encoding a 168 | //! chunk always takes longer than creating and transferring it (otherwise 169 | //! distributing wouldn't make sense at all). The longer the video then, the 170 | //! larger the ratio of encoding (which we can parallelize) in the total amount 171 | //! of time the process takes, and the more effective doing so becomes. 172 | //! 173 | //! I've also looked at how the work is distributed over the nodes, depending 174 | //! on their processing power. At the end of a parallel encode, it's possible 175 | //! to determine how many chunks have been encoded by any given host. 176 | //! 177 | //! | host | chunks | power | 178 | //! | ------------- | ------ | ----- | 179 | //! | desktop | 73 | 1.00 | 180 | //! | old1 | 39 | 0.53 | 181 | //! | old2 | 31 | 0.42 | 182 | //! | laptop | 19 | 0.26 | 183 | //! 184 | //! Inferring the processing power from the number of chunks leads to almost 185 | //! exactly the same results as my initial determination, confirming it and 186 | //! proving that work is distributed efficiently. 187 | //! 188 | //! To further see how the system scales, I've added two more machines, 189 | //! bringing the total processing power up to 3.29. 190 | //! 191 | //! | machine | duration (s) | power | 192 | //! | --------- | ------------ | -------- | 193 | //! | desktop | 1373 | 1.00 | 194 | //! | c1 | 2129 | 0.64 | 195 | //! | old1 | 2571 | 0.53 | 196 | //! | c2 | 3022 | 0.45 | 197 | //! | old2 | 3292 | 0.42 | 198 | //! | laptop | 5572 | 0.25 | 199 | //! | **total** | - | **3.29** | 200 | //! 201 | //! Encoding the video on these 6 machines in parallel took 3865 seconds, so 202 | //! 41.3% of the time using my fastest computer, or a 2.42x speedup. It's 203 | //! making use of the additionally available resources with a 74% efficiency 204 | //! here. As expected, while we can accelerate by adding more resources, we're 205 | //! looking at diminishing returns. Although the factor by which the efficiency 206 | //! decreases is not as bad as it could be. 207 | //! 208 | //! ## Limitations 209 | //! 210 | //! While you can use your own `ffmpeg` options to control how the video is 211 | //! encoded, there is currently no such option for the audio, which is 192 kb/s 212 | //! AAC by default. 213 | 214 | use crossbeam::channel; 215 | use dirs; 216 | use log::{error, info}; 217 | use std::{ 218 | error::Error, 219 | fs, 220 | path::{Path, PathBuf}, 221 | process::Command, 222 | string::ToString, 223 | sync::atomic::{AtomicBool, Ordering}, 224 | sync::Arc, 225 | thread, 226 | time::Duration, 227 | }; 228 | 229 | mod local; 230 | mod remote; 231 | 232 | /// The name of the temporary directory in the home directory to collect 233 | /// intermediate files. 234 | const TMP_DIR: &str = "shepherd_tmp"; 235 | /// The name of the encoded audio track. 236 | const AUDIO: &str = "audio.aac"; 237 | /// The length of chunks to split the video into. 238 | const DEFAULT_LENGTH: &str = "60"; 239 | 240 | /// The generic result type for this crate. 241 | pub type Result = std::result::Result>; 242 | 243 | /// Starts the whole operation and cleans up afterwards. 244 | /// 245 | /// # Arguments 246 | /// * `input` - The path to the input file. 247 | /// * `output` - The path to the output file. 248 | /// * `args` - Arguments to `ffmpeg` for chunk encoding. 249 | /// * `hosts` - Comma-separated list of hosts. 250 | /// * `seconds` - The video chunk length. 251 | /// * `tmp_dir` - The path to the local temporary directory. 252 | /// * `keep` - Whether to keep temporary files on hosts (no cleanup). 253 | pub fn run( 254 | input: impl AsRef, 255 | output: impl AsRef, 256 | args: &[&str], 257 | hosts: Vec<&str>, 258 | seconds: Option<&str>, 259 | tmp_dir: Option<&str>, 260 | keep: bool, 261 | ) -> Result<()> { 262 | // Convert the length 263 | let seconds = seconds.unwrap_or(DEFAULT_LENGTH).parse::()?; 264 | // Convert the tmp_dir 265 | let mut tmp_dir = tmp_dir 266 | .map(PathBuf::from) 267 | .or_else(dirs::home_dir) 268 | .ok_or("Home directory not found")?; 269 | 270 | // Set up a shared boolean to check whether the user has aborted 271 | let running = Arc::new(AtomicBool::new(true)); 272 | let r = Arc::clone(&running); 273 | ctrlc::set_handler(move || { 274 | r.store(false, Ordering::SeqCst); 275 | info!( 276 | "Abort signal received. Waiting for remote encoders to finish the \ 277 | current chunk and quit gracefully." 278 | ); 279 | }) 280 | .expect("Error setting Ctrl-C handler"); 281 | 282 | tmp_dir.push(TMP_DIR); 283 | // Remove local temporary directory in case it's still around 284 | fs::remove_dir_all(&tmp_dir).ok(); 285 | // Create our local temporary directory 286 | fs::create_dir(&tmp_dir)?; 287 | 288 | // Start the operation 289 | let result = run_local( 290 | input.as_ref(), 291 | output.as_ref(), 292 | args, 293 | &tmp_dir, 294 | &hosts, 295 | seconds, 296 | Arc::clone(&running), 297 | ); 298 | 299 | if !keep { 300 | info!("Cleaning up"); 301 | // Remove remote temporary directories 302 | for &host in &hosts { 303 | // Clean up temporary directory on host 304 | let output = Command::new("ssh") 305 | .args(&[&host, "rm", "-r", remote::TMP_DIR]) 306 | .output() 307 | .expect("Failed executing ssh command"); 308 | // These checks for `running` are necessary, because Ctrl + C also 309 | // seems to terminate the commands we launch, which means they'll 310 | // return unsuccessfully. With this check we prevent an error 311 | // message in this case, because that's what the user wants. 312 | // Unfortunately this also means we have to litter the `running` 313 | // variable almost everyhwere. 314 | if !output.status.success() && running.load(Ordering::SeqCst) { 315 | error!( 316 | "Failed removing remote temporary directory on {}", 317 | host 318 | ); 319 | } 320 | } 321 | // Remove local temporary directory 322 | fs::remove_dir_all(&tmp_dir).ok(); 323 | } 324 | 325 | result 326 | } 327 | 328 | /// Does the actual work. 329 | /// 330 | /// This is separate so it can fail and return early, since cleanup is then 331 | /// handled in its caller function. 332 | fn run_local( 333 | input: &Path, 334 | output: &Path, 335 | args: &[&str], 336 | tmp_dir: &Path, 337 | hosts: &[&str], 338 | seconds: u64, 339 | running: Arc, 340 | ) -> Result<()> { 341 | // Build path to audio file 342 | let mut audio = tmp_dir.to_path_buf(); 343 | audio.push(AUDIO); 344 | // Start the extraction 345 | info!("Extracting audio"); 346 | local::extract_audio(input, &audio, &running)?; 347 | 348 | // We check whether the user has aborted before every time-intensive task. 349 | // It's a better experience, but a bit ugly code-wise. 350 | if !running.load(Ordering::SeqCst) { 351 | // Abort early 352 | return Ok(()); 353 | } 354 | 355 | // Create directory for video chunks 356 | let mut chunk_dir = tmp_dir.to_path_buf(); 357 | chunk_dir.push("chunks"); 358 | fs::create_dir(&chunk_dir)?; 359 | // Split the video 360 | info!("Splitting video into chunks"); 361 | local::split_video( 362 | input, 363 | &chunk_dir, 364 | Duration::from_secs(seconds), 365 | &running, 366 | )?; 367 | // Get the list of created chunks 368 | let mut chunks = fs::read_dir(&chunk_dir)? 369 | .map(|res| res.and_then(|readdir| Ok(readdir.path()))) 370 | .collect::>>()?; 371 | // Sort them so they're in order. That's not strictly necessary, but nicer 372 | // for the user to watch since it allows seeing the progress at a glance. 373 | chunks.sort(); 374 | 375 | if !running.load(Ordering::SeqCst) { 376 | // Abort early 377 | return Ok(()); 378 | } 379 | 380 | // Initialize the global channel for chunks 381 | let (sender, receiver) = channel::unbounded(); 382 | // Send all chunks into it 383 | for chunk in chunks { 384 | sender.send(chunk)?; 385 | } 386 | // Drop the sender so the channel gets disconnected 387 | drop(sender); 388 | 389 | // Since we want to share the ffmpeg arguments between the threads, we need 390 | // to first set up an owned version of them 391 | let args: Vec = args.iter().map(ToString::to_string).collect(); 392 | // and then create our Arc 393 | let args = Arc::new(args); 394 | 395 | // Create directory for encoded chunks 396 | let mut encoded_dir = tmp_dir.to_path_buf(); 397 | encoded_dir.push("encoded"); 398 | fs::create_dir(&encoded_dir)?; 399 | // Isolate output extension, since we want encoded chunks to have the same 400 | let out_ext = output 401 | .extension() 402 | .ok_or("Unable to find extension")? 403 | .to_str() 404 | .ok_or("Unable to convert OsString extension")? 405 | .to_string(); 406 | // Spawn threads for hosts 407 | info!("Starting remote encoding"); 408 | let mut host_threads = Vec::with_capacity(hosts.len()); 409 | for &host in hosts { 410 | // Clone the queue receiver for the thread 411 | let thread_receiver = receiver.clone(); 412 | // Create copy of running indicator for the thread 413 | let r = Arc::clone(&running); 414 | // And lots of other copies because it's easy and the extra allocations 415 | // are not a problem for this kind of application 416 | let host = host.to_string(); 417 | let enc = encoded_dir.clone(); 418 | let ext = out_ext.clone(); 419 | let a = Arc::clone(&args); 420 | // Start it 421 | let handle = 422 | thread::Builder::new().name(host.clone()).spawn(|| { 423 | remote::host_thread(host, thread_receiver, enc, ext, a, r); 424 | })?; 425 | host_threads.push(handle); 426 | } 427 | 428 | // Wait for all hosts to finish 429 | for handle in host_threads { 430 | if handle.join().is_err() { 431 | return Err("A host thread panicked".into()); 432 | } 433 | } 434 | 435 | if !running.load(Ordering::SeqCst) { 436 | // We aborted early 437 | return Ok(()); 438 | } 439 | 440 | // Combine encoded chunks and audio 441 | info!("Combining encoded chunks into final video"); 442 | local::combine(&encoded_dir, &audio, output, &running)?; 443 | 444 | Ok(()) 445 | } 446 | --------------------------------------------------------------------------------