├── Rakefile
├── .gitignore
├── .cargo
    └── config.toml
├── .github
    ├── ISSUE_TEMPLATE
    │   ├── feature-request.md
    │   └── bug_report.md
    └── workflows
    │   └── release.yml
├── src
    ├── main.rs
    ├── params.rs
    ├── formatter.rs
    ├── fileinfo.rs
    ├── server.rs
    ├── interactive.rs
    ├── scanner.rs
    └── processor.rs
├── LICENSE
├── Cargo.toml
├── CONTRIBUTING.md
├── rakelib
    └── benchmark.rake
├── README.md
└── Cargo.lock


/Rakefile:
--------------------------------------------------------------------------------
1 | # tasks are contained in the `rakelib` directory
2 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /target
2 | /Cargo.lock
3 | /result-bin
4 | /.bacon-locations
5 | 


--------------------------------------------------------------------------------
/.cargo/config.toml:
--------------------------------------------------------------------------------
1 | [build]
2 | rustflags = ["-C", "target-feature=+aes,+sse2"]
3 | 


--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/feature-request.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | name: Feature request
 3 | about: Suggest an idea for this project
 4 | title: "[Feature] Title"
 5 | labels: ''
 6 | assignees: ''
 7 | 
 8 | ---
 9 | 
10 | **Is your feature request related to a problem? Please describe.**
11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
12 | 
13 | **Describe the solution you'd like**
14 | A clear and concise description of what you want to happen.
15 | 
16 | **Describe alternatives you've considered**
17 | A clear and concise description of any alternative solutions or features you've considered.
18 | 
19 | **Additional context**
20 | Add any other context or screenshots about the feature request here.
21 | 


--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/bug_report.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | name: Bug report
 3 | about: Create a report to help us improve
 4 | title: "[Bug] Title"
 5 | labels: ''
 6 | assignees: ''
 7 | 
 8 | ---
 9 | 
10 | **Describe the bug**
11 | A clear and concise description of what the bug is.
12 | 
13 | ** Runtime Info **
14 | App Arguments: [e.g. `-i --nocache`] 
15 | Install Type: [e.g. `cargo install`]
16 | App Version: [e.g. v0.0.7]
17 | 
18 | **Expected behavior**
19 | A clear and concise description of what you expected to happen.
20 | 
21 | **Screenshots**
22 | If applicable, add screenshots to help explain your problem.
23 | 
24 | **Platform Details (please complete the following information):**
25 |  - OS: [e.g. Arch Linux]
26 |  - Terminal Emulator: [e.g Alacritty]
27 |  - Shell [e.g. Zshell]
28 | 
29 | **Additional context**
30 | Add any other context about the problem here.
31 | 


--------------------------------------------------------------------------------
/src/main.rs:
--------------------------------------------------------------------------------
 1 | mod fileinfo;
 2 | mod formatter;
 3 | mod interactive;
 4 | mod params;
 5 | mod processor;
 6 | mod scanner;
 7 | mod server;
 8 | 
 9 | use self::{formatter::Formatter, interactive::Interactive, server::Server};
10 | use anyhow::Result;
11 | use clap::Parser;
12 | use params::Params;
13 | use std::sync::atomic::Ordering;
14 | 
15 | fn main() -> Result<()> {
16 |     let app_args = Params::parse();
17 |     let server = Server::new(app_args.clone());
18 | 
19 |     server.start()?;
20 | 
21 |     match app_args.interactive {
22 |         false => {
23 |             Formatter::print(
24 |                 server.hw_duplicate_set,
25 |                 server.max_file_path_len.load(Ordering::Acquire),
26 |                 &app_args,
27 |             );
28 |         }
29 |         true => {
30 |             Interactive::init(server.hw_duplicate_set, &app_args)?;
31 |         }
32 |     };
33 | 
34 |     Ok(())
35 | }
36 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2022 Sreedev Kodichath
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "deduplicator"
 3 | version = "0.3.2"
 4 | edition = "2021"
 5 | description = "find,filter and delete duplicate files"
 6 | repository = "https://github.com/sreedevk/deduplicator"
 7 | license = "MIT"
 8 | authors = [
 9 |   "Sreedev Kodichath <sreedevpadmakumar@gmail.com>",
10 |   "Valentin Bersier <vbersier@gmail.com>",
11 |   "Dhruva Sagar <dhruva.sagar@gmail.com>",
12 | ]
13 | 
14 | [[bin]]
15 | name = "deduplicator"
16 | path = "src/main.rs"
17 | 
18 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
19 | [dependencies]
20 | anyhow = "1.0.68"
21 | bytesize = "2.0.1"
22 | chrono = "0.4.23"
23 | clap = { version = "4.0.32", features = ["derive"] }
24 | dashmap = { version = "6.1.0", features = ["rayon"] }
25 | globwalk = "0.9.1"
26 | gxhash = { version = "3.4.1", default-features = false }
27 | indicatif = { version = "0.18.0", features = ["rayon"] }
28 | memmap2 = "0.9.7"
29 | pathdiff = "0.2.1"
30 | prettytable-rs = "0.10.0"
31 | rand = "0.9.1"
32 | rayon = "1.6.1"
33 | threadpool = "1.8.1"
34 | unicode-segmentation = "1.12.0"
35 | 
36 | [profile.release]
37 | strip = true
38 | opt-level = 3
39 | lto = "thin"
40 | debug = false
41 | codegen-units = 1
42 | 
43 | # generated by 'cargo dist init'
44 | [profile.dist]
45 | inherits = "release"
46 | 
47 | [workspace.metadata.dist]
48 | rust-toolchain-version = "1.87.0"
49 | ci = ["github"]
50 | targets = [
51 |   "x86_64-unknown-linux-gnu",
52 |   "x86_64-apple-darwin",
53 |   "x86_64-pc-windows-msvc",
54 |   "aarch64-apple-darwin",
55 | ]
56 | cargo-dist-version = "0.0.7"
57 | 
58 | [dev-dependencies]
59 | tempfile = "3.20.0"
60 | 


--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
 1 | ## How to contribute to Deduplicator
 2 | 
 3 | #### **Did you find a bug?**
 4 | 
 5 | * **Ensure the bug was not already reported** by searching on GitHub under [Issues](https://github.com/sreedevk/deduplicator/issues).
 6 | 
 7 | * If you're unable to find an open issue addressing the problem, [open a new one](https://github.com/sreedevk/deduplicator/issues/new). Be sure to include a **title and clear description**, as much relevant information as possible, and a **code sample** or an **executable test case** demonstrating the expected behavior that is not occurring.
 8 | 
 9 | * If possible, use the [bug report template](https://github.com/sreedevk/deduplicator/blob/main/.github/ISSUE_TEMPLATE/bug_report.md) to create the issue.
10 | 
11 | #### **Would you like to write a fix for the bug?**
12 | * Assign the Issue to yourself (if unassigned) before you start working in order to avoid any conficts. 
13 | * Open a new GitHub pull request with the patch.
14 | * Ensure the PR description clearly describes the problem and solution. Include the relevant issue number.
15 | * Make sure that the PR points to the development branch.
16 | 
17 | #### **Did you fix whitespace, format code, or make a purely cosmetic patch?**
18 | 
19 | Changes that are cosmetic in nature and do not add anything substantial to the stability, functionality, or testability of Deduplicator will generally not be accepted/
20 | 
21 | #### **Do you intend to add a new feature or change an existing one?**
22 | 
23 | * First open an issue with the sugggestion using the [feature request template](https://github.com/sreedevk/deduplicator/blob/main/.github/ISSUE_TEMPLATE/feature-request.md)
24 | * Do not create a PR before one of the core contributors has conveyed acceptance for a feature request.
25 | 
26 | #### **Do you have questions about the source code?**
27 | 
28 | * If you have a question, raise an issue in the repository with a "question" label.
29 | 


--------------------------------------------------------------------------------
/src/params.rs:
--------------------------------------------------------------------------------
 1 | use std::{fs, path::PathBuf};
 2 | 
 3 | use anyhow::Result;
 4 | use clap::{Parser, ValueHint};
 5 | 
 6 | #[derive(Parser, Debug, Default, Clone)]
 7 | #[command(author, version, about, long_about = None)]
 8 | pub struct Params {
 9 |     /// Exclude Filetypes [default = none]
10 |     #[arg(short = 'T', long)]
11 |     pub exclude_types: Option<String>,
12 |     /// Filetypes to deduplicate [default = all]
13 |     #[arg(short, long)]
14 |     pub types: Option<String>,
15 |     /// Run Deduplicator on dir different from pwd (e.g., ~/Pictures )
16 |     #[arg(value_hint = ValueHint::DirPath, value_name = "scan_dir_path")]
17 |     pub dir: Option<PathBuf>,
18 |     /// Delete files interactively
19 |     #[arg(long, short)]
20 |     pub interactive: bool,
21 |     /// Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T).
22 |     #[arg(long, short = 'm', default_value = "1b")]
23 |     pub min_size: Option<String>,
24 |     /// Max Depth to scan while looking for duplicates
25 |     #[arg(long, short = 'D')]
26 |     pub max_depth: Option<usize>,
27 |     /// Min Depth to scan while looking for duplicates
28 |     #[arg(long, short = 'd')]
29 |     pub min_depth: Option<usize>,
30 |     /// Follow links while scanning directories
31 |     #[arg(long, short)]
32 |     pub follow_links: bool,
33 |     /// Guarantees that two files are duplicate (performs a full hash)
34 |     #[arg(long, short = 's', default_value = "false")]
35 |     pub strict: bool,
36 |     /// Show Progress spinners & metrics
37 |     #[arg(long, short = 'p', default_value = "false")]
38 |     pub progress: bool,
39 | }
40 | 
41 | impl Params {
42 |     pub fn get_min_size(&self) -> Option<u64> {
43 |         match &self.min_size {
44 |             Some(msize) => match msize.parse::<bytesize::ByteSize>() {
45 |                 Ok(units) => Some(units.0),
46 |                 Err(_) => None,
47 |             },
48 |             None => None,
49 |         }
50 |     }
51 | 
52 |     pub fn get_directory(&self) -> Result<PathBuf> {
53 |         let current_dir = std::env::current_dir()?;
54 |         let dir_path = self.dir.as_ref().unwrap_or(&current_dir).as_path();
55 |         let dir = fs::canonicalize(dir_path)?;
56 |         Ok(dir)
57 |     }
58 | }
59 | 


--------------------------------------------------------------------------------
/src/formatter.rs:
--------------------------------------------------------------------------------
 1 | use crate::{fileinfo::FileInfo, params::Params};
 2 | use anyhow::Result;
 3 | use chrono::{DateTime, Utc};
 4 | use dashmap::DashMap;
 5 | use pathdiff::diff_paths;
 6 | use rayon::prelude::*;
 7 | use std::sync::atomic::AtomicU64;
 8 | use std::{path::PathBuf, sync::Arc};
 9 | 
10 | const YELLOW: &str = "\x1b[33m";
11 | const RESET: &str = "\x1b[0m";
12 | 
13 | pub struct Formatter;
14 | impl Formatter {
15 |     pub fn human_path(file: &FileInfo, aargs: &Params, max_path_length: usize) -> Result<String> {
16 |         let base_directory: PathBuf = aargs.get_directory()?;
17 |         let relative_path = diff_paths(&file.path, base_directory).unwrap_or_default();
18 | 
19 |         let formatted_path = format!(
20 |             "{:<0width$}",
21 |             relative_path.to_str().unwrap_or_default().to_string(),
22 |             width = max_path_length
23 |         );
24 | 
25 |         Ok(formatted_path)
26 |     }
27 | 
28 |     pub fn human_filesize(file: &FileInfo) -> Result<String> {
29 |         Ok(format!("{:>12}", bytesize::ByteSize::b(file.size)))
30 |     }
31 | 
32 |     pub fn human_mtime(file: &FileInfo) -> Result<String> {
33 |         let modified_time: DateTime<Utc> = file.modified.into();
34 |         Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
35 |     }
36 | 
37 |     pub fn print(raw: Arc<DashMap<u128, Vec<FileInfo>>>, max_path_len: u64, aargs: &Params) {
38 |         print!("{}", "\n".repeat(if aargs.progress { 2 } else { 1 })); // spacing
39 | 
40 |         if raw.is_empty() {
41 |             println!("No duplicates found matching your search criteria.");
42 |         } else {
43 |             let printed_count: AtomicU64 = AtomicU64::new(0);
44 | 
45 |             raw.par_iter().for_each(|sref| {
46 |                 if sref.value().len() > 1 {
47 |                     printed_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
48 |                     let mut ostring = format!("{}{:32x}{}\n", YELLOW, sref.key(), RESET);
49 |                     let subfields = sref
50 |                         .value()
51 |                         .par_iter()
52 |                         .enumerate()
53 |                         .map(|(i, finfo)| {
54 |                             let nodechar = if i == sref.value().len() - 1 {
55 |                                 "└─"
56 |                             } else {
57 |                                 "├─"
58 |                             };
59 |                             format!(
60 |                                 "{}\t{}\t{}\t{}\n",
61 |                                 nodechar,
62 |                                 Self::human_path(finfo, aargs, max_path_len as usize)
63 |                                     .expect("path formatting failed."),
64 |                                 Self::human_filesize(finfo).expect("filesize formatting failed."),
65 |                                 Self::human_mtime(finfo).expect("modified time formatting failed.")
66 |                             )
67 |                         })
68 |                         .collect::<String>();
69 | 
70 |                     ostring.push_str(&subfields);
71 |                     println!("{ostring}");
72 |                 }
73 |             });
74 | 
75 |             if printed_count.load(std::sync::atomic::Ordering::Relaxed) < 1 {
76 |                 println!("No duplicates found matching your search criteria.");
77 |             }
78 |         }
79 |     }
80 | }
81 | 


--------------------------------------------------------------------------------
/src/fileinfo.rs:
--------------------------------------------------------------------------------
  1 | use anyhow::Result;
  2 | use gxhash::gxhash128;
  3 | use memmap2::Mmap;
  4 | use std::{
  5 |     fs,
  6 |     io::Read,
  7 |     path::{Path, PathBuf},
  8 |     sync::{Arc, Mutex},
  9 |     time::SystemTime,
 10 | };
 11 | 
 12 | #[derive(Debug, Clone, PartialEq)]
 13 | pub enum FileState {
 14 |     Unprocessed,
 15 |     SwProcessed,
 16 | }
 17 | 
 18 | #[derive(Debug, Clone)]
 19 | pub struct FileInfo {
 20 |     pub path: Box<Path>,
 21 |     pub size: u64,
 22 |     pub modified: SystemTime,
 23 |     pub state: Arc<Mutex<FileState>>,
 24 | }
 25 | 
 26 | impl FileInfo {
 27 |     pub fn hash(&self, seed: i64) -> Result<u128> {
 28 |         if self.size == 0 {
 29 |             return Ok(0u128);
 30 |         };
 31 | 
 32 |         let file = fs::File::open(&self.path)?;
 33 |         let mapper = unsafe { Mmap::map(&file)? };
 34 |         let content_hash = mapper
 35 |             .chunks(4096)
 36 |             .fold(0u128, |acc, chunk: &[u8]| acc ^ gxhash128(chunk, seed));
 37 | 
 38 |         // NOTE: avoids collision bw an empty file & a file full of null bytes.
 39 |         Ok(content_hash ^ gxhash128(&self.size.to_ne_bytes(), seed))
 40 |     }
 41 | 
 42 |     pub fn initpages_hash(&self, seed: i64) -> Result<u128> {
 43 |         let mut file = fs::File::open(&self.path)?;
 44 |         let mut buffer = [0; 16384];
 45 |         let bytes_read = file.read(&mut buffer)?;
 46 | 
 47 |         Ok(gxhash128(&buffer[..bytes_read], seed))
 48 |     }
 49 | 
 50 |     pub fn new(path: PathBuf) -> Result<Self> {
 51 |         let filemeta = std::fs::metadata(&path)?;
 52 |         Ok(Self {
 53 |             path: path.into_boxed_path(),
 54 |             size: filemeta.len(),
 55 |             modified: filemeta.modified()?,
 56 |             state: Arc::new(Mutex::new(FileState::Unprocessed)),
 57 |         })
 58 |     }
 59 | 
 60 |     pub fn sw_processed(&self) {
 61 |         let mut self_state = self.state.lock().unwrap();
 62 |         *self_state = FileState::SwProcessed;
 63 |     }
 64 | 
 65 |     pub fn is_sw_processed(&self) -> bool {
 66 |         let self_state = self.state.lock().unwrap();
 67 |         *self_state == FileState::SwProcessed
 68 |     }
 69 | }
 70 | 
 71 | #[cfg(test)]
 72 | mod test {
 73 |     use super::*;
 74 |     use tempfile::TempDir;
 75 |     use std::fs::File;
 76 |     use std::io::Write;
 77 |     use anyhow::Result;
 78 | 
 79 |     fn generate_null_bytes(size: usize) -> Vec<u8> {
 80 |         (0..size).map(|_| 0).collect::<Vec<u8>>()
 81 |     }
 82 | 
 83 |     #[test]
 84 |     fn hash_differentiates_between_a_file_of_null_bytes_vs_an_empty_file() -> Result<()> {
 85 |         let root = TempDir::new()?;
 86 |         let empty_file_name = root.path().join("empty_file.bin");
 87 | 
 88 |         File::create_new(&empty_file_name)?;
 89 | 
 90 |         let file_with_null_bytes_name = root.path().join("file_with_null_bytes.bin");
 91 |         let mut file_with_null_bytes = File::create_new(&file_with_null_bytes_name)?;
 92 | 
 93 |         file_with_null_bytes.write_all(&generate_null_bytes(1000 * 4096))?;
 94 | 
 95 |         let empty_file_info = FileInfo::new(empty_file_name)?;
 96 |         let file_with_empty_bytes_info = FileInfo::new(file_with_null_bytes_name)?;
 97 | 
 98 |         let seed: i64 = 246910456374;
 99 | 
100 |         assert_ne!(empty_file_info.hash(seed)?, file_with_empty_bytes_info.hash(seed)?);
101 | 
102 |         Ok(())
103 |     }
104 | }
105 | 


--------------------------------------------------------------------------------
/rakelib/benchmark.rake:
--------------------------------------------------------------------------------
  1 | require 'tempfile'
  2 | require 'fileutils'
  3 | require 'securerandom'
  4 | 
  5 | namespace :benchmark do
  6 |   file 'target/release/deduplicator' do
  7 |     sh "cargo build --release"
  8 |   end
  9 | 
 10 |   task :few_large_files => 'target/release/deduplicator' do
 11 |     # Benchmark 1: ./target/release/deduplicator bench_artifacts
 12 |     # Time (mean ± σ):       2.5 ms ±   0.6 ms    [User: 2.4 ms, System: 4.8 ms]
 13 |     # Range (min … max):     1.8 ms …   9.7 ms    1474 runs
 14 | 
 15 |     root = "bench_artifacts"
 16 |     FileUtils.rm_rf(root)
 17 |     Dir.mkdir(root)
 18 | 
 19 |     # files with same size
 20 |     puts "generating files of same size ..."
 21 |     2.times.map do |i|
 22 |       File.open(File.join(root, "file_#{i}_fwss.bin"), 'wb') do |f|
 23 |         f.write(SecureRandom.bytes(4096 * 100_000))
 24 |       end
 25 |     end
 26 | 
 27 |     # files with different sizes
 28 |     puts "generating files of different sizes ..."
 29 |     2.times.map do |i|
 30 |       File.open(File.join(root, "file_#{i}_fwds.bin"), 'wb') do |f|
 31 |         f.write(SecureRandom.bytes(4096 * (rand * 100_000).ceil))
 32 |       end
 33 |     end
 34 | 
 35 |     # files with same content & size
 36 |     puts "generating files of same content and sizes ..."
 37 |     2.times.each do |i|
 38 |       File.open(File.join(root, "file_#{i}_fwscas.bin"), 'wb') do |f|
 39 |         f.write("\0" * (4096 * 100_000))
 40 |       end
 41 |     end
 42 | 
 43 |     # files with different content but same size
 44 |     puts "generating files of different content but same sizes ..."
 45 |     2.times.each do |i|
 46 |       File.open(File.join(root, "file_#{i}_fwdcbss.bin"), 'wb') do |f|
 47 |         f.write(SecureRandom.bytes(4096 * 100_000))
 48 |       end
 49 |     end
 50 | 
 51 |     sh("hyperfine -N --warmup 80 './target/release/deduplicator #{root}'")
 52 |     sh("dust '#{root}'")
 53 | 
 54 |     FileUtils.rm_rf(root)
 55 |   end
 56 | 
 57 |   task :many_small_files => 'target/release/deduplicator' do
 58 |     # Benchmark 1: ./target/release/deduplicator bench_artifacts
 59 |     # Time (mean ± σ):      10.6 ms ±   1.0 ms    [User: 20.0 ms, System: 22.5 ms]
 60 |     # Range (min … max):     8.4 ms …  14.2 ms    235 runs
 61 | 
 62 |     root = "bench_artifacts"
 63 |     Dir.mkdir(root)
 64 | 
 65 |     # files with same size
 66 |     puts "generating 1000 files of the same size ... "
 67 |     1000.times.each do |i|
 68 |       File.open(File.join(root, "file_#{i}_fwss.bin"), 'wb') do |f|
 69 |         f.write(SecureRandom.bytes(4096 * 1000))
 70 |       end
 71 |     end
 72 | 
 73 |     # files with different sizes
 74 |     puts "generating 1000 files of different sizes ... "
 75 |     1000.times.each do |i|
 76 |       File.open(File.join(root, "file_#{i}_fwds.bin"), 'wb') do |f|
 77 |         f.write(SecureRandom.bytes(4096 * (rand * 100).ceil))
 78 |       end
 79 |     end
 80 | 
 81 |     # files with same content & size
 82 |     puts "generating files of same content and sizes ..."
 83 |     1000.times.each do |i|
 84 |       File.open(File.join(root, "file_#{i}_fwscas.bin"), 'wb') do |f|
 85 |         f.write("\0" * (4096 * 1000))
 86 |       end
 87 |     end
 88 | 
 89 |     # files with different content but same size
 90 |     puts "generating files of different content but same sizes ..."
 91 |     1000.times.each do |i|
 92 |       File.open(File.join(root, "file_#{i}_fwdcbss.bin"), 'wb') do |f|
 93 |         f.write(SecureRandom.bytes(4096 * 1000))
 94 |       end
 95 |     end
 96 | 
 97 |     sh("hyperfine --warmup 20 './target/release/deduplicator #{root}'")
 98 |     sh("dust '#{root}'")
 99 | 
100 |     FileUtils.rm_rf(root)
101 |   end
102 | end
103 | 


--------------------------------------------------------------------------------
/src/server.rs:
--------------------------------------------------------------------------------
  1 | use std::sync::atomic::{AtomicBool, AtomicU64};
  2 | use std::sync::{Arc, Mutex};
  3 | 
  4 | use crate::processor::Processor;
  5 | use crate::scanner::Scanner;
  6 | use anyhow::Result;
  7 | use dashmap::DashMap;
  8 | use indicatif::{MultiProgress, ProgressDrawTarget};
  9 | use rand::Rng;
 10 | use threadpool::ThreadPool;
 11 | 
 12 | use crate::fileinfo::FileInfo;
 13 | use crate::params::Params;
 14 | 
 15 | pub struct Server {
 16 |     filequeue: Arc<Mutex<Vec<FileInfo>>>,
 17 |     sw_duplicate_set: Arc<DashMap<u64, Vec<FileInfo>>>,
 18 |     pub hw_duplicate_set: Arc<DashMap<u128, Vec<FileInfo>>>,
 19 |     threadpool: ThreadPool,
 20 |     app_args: Arc<Params>,
 21 |     pub max_file_path_len: Arc<AtomicU64>,
 22 | }
 23 | 
 24 | impl Server {
 25 |     pub fn new(opts: Params) -> Self {
 26 |         Self {
 27 |             filequeue: Arc::new(Mutex::new(Vec::new())),
 28 |             sw_duplicate_set: Arc::new(DashMap::new()),
 29 |             hw_duplicate_set: Arc::new(DashMap::new()),
 30 |             threadpool: ThreadPool::new(4),
 31 |             app_args: Arc::new(opts),
 32 |             max_file_path_len: Arc::new(AtomicU64::new(0)),
 33 |         }
 34 |     }
 35 | 
 36 |     pub fn start(&self) -> Result<()> {
 37 |         let progbarbox = Arc::new(MultiProgress::new());
 38 |         let mut rng = rand::rng();
 39 |         let seed: i64 = rng.random();
 40 | 
 41 |         if !self.app_args.progress {
 42 |             progbarbox.set_draw_target(ProgressDrawTarget::hidden());
 43 |         }
 44 | 
 45 |         let (app_args_sc, app_args_sw, app_args_hw) = (
 46 |             Arc::clone(&self.app_args),
 47 |             Arc::clone(&self.app_args),
 48 |             Arc::clone(&self.app_args),
 49 |         );
 50 |         let (file_queue_sc, file_queue_pr) = (
 51 |             Arc::clone(&self.filequeue),
 52 |             Arc::clone(&self.filequeue),
 53 |         );
 54 |         let scanner_finished = Arc::new(AtomicBool::new(false));
 55 |         let sw_sort_finished = Arc::new(AtomicBool::new(false));
 56 |         let (sfin_sc, sfin_pr) = (
 57 |             Arc::clone(&scanner_finished),
 58 |             Arc::clone(&scanner_finished),
 59 |         );
 60 |         let (swfin_pr_sw, swfin_pr_hw) = (
 61 |             Arc::clone(&sw_sort_finished),
 62 |             Arc::clone(&sw_sort_finished),
 63 |         );
 64 |         let (store_sw, store_sw2, store_hw) = (
 65 |             Arc::clone(&self.sw_duplicate_set),
 66 |             Arc::clone(&self.sw_duplicate_set),
 67 |             Arc::clone(&self.hw_duplicate_set),
 68 |         );
 69 |         let max_file_path_len = Arc::clone(&self.max_file_path_len);
 70 |         let (prog_sc, prog_sw, prog_hw) = (
 71 |             Arc::clone(&progbarbox),
 72 |             Arc::clone(&progbarbox),
 73 |             Arc::clone(&progbarbox),
 74 |         );
 75 | 
 76 |         self.threadpool.execute(move || {
 77 |             Scanner::new(app_args_sc)
 78 |                 .expect("unable to initialize scanner.")
 79 |                 .scan(file_queue_sc, prog_sc)
 80 |                 .expect("scanner failed.");
 81 | 
 82 |             sfin_sc.store(true, std::sync::atomic::Ordering::Relaxed);
 83 |         });
 84 | 
 85 |         self.threadpool.execute(move || {
 86 |             Processor::sizewise(
 87 |                 app_args_sw,
 88 |                 sfin_pr,
 89 |                 store_sw,
 90 |                 file_queue_pr,
 91 |                 prog_sw,
 92 |             )
 93 |             .expect("sizewise scanner failed.");
 94 | 
 95 |             swfin_pr_sw.store(true, std::sync::atomic::Ordering::Relaxed);
 96 |         });
 97 | 
 98 |         self.threadpool.execute(move || {
 99 |             Processor::hashwise(
100 |                 app_args_hw,
101 |                 store_sw2,
102 |                 store_hw,
103 |                 prog_hw,
104 |                 max_file_path_len,
105 |                 seed,
106 |                 swfin_pr_hw,
107 |             )
108 |             .expect("sizewise scanner failed.");
109 |         });
110 | 
111 |         progbarbox.clear()?;
112 | 
113 |         self.threadpool.join();
114 | 
115 |         Ok(())
116 |     }
117 | }
118 | 


--------------------------------------------------------------------------------
/.github/workflows/release.yml:
--------------------------------------------------------------------------------
  1 | name: Deduplicator Release Build CI Pipeline
  2 | 
  3 | on:
  4 |   push:
  5 |     tags:
  6 |       - 'v*.*.*'
  7 | 
  8 | jobs:
  9 |   lint_test:
 10 |     runs-on: ubuntu-latest
 11 |     env:
 12 |       RUSTFLAGS: "-C target-feature=+aes,+sse2"
 13 |     steps:
 14 |       - name: Checkout repository
 15 |         uses: actions/checkout@v4
 16 | 
 17 |       - name: Install Rust
 18 |         uses: dtolnay/rust-toolchain@stable
 19 |         with:
 20 |           components: clippy
 21 | 
 22 |       - name: Run cargo check
 23 |         run: cargo check
 24 | 
 25 |       - name: Run cargo clippy
 26 |         run: cargo clippy -- -D warnings
 27 | 
 28 |       - name: Run tests
 29 |         run: cargo test -- --test-threads=1
 30 | 
 31 |   build:
 32 |     needs: lint_test
 33 |     runs-on: ${{ matrix.os }}
 34 |     strategy:
 35 |       matrix:
 36 |         include:
 37 |           - target: aarch64-unknown-linux-gnu
 38 |             os: ubuntu-latest
 39 |             artifact_name: linux-aarch64
 40 |             rustflags: "-C target-feature=+aes,+neon"
 41 |           - target: x86_64-unknown-linux-gnu
 42 |             os: ubuntu-latest
 43 |             artifact_name: linux-amd64
 44 |             rustflags: "-C target-feature=+aes,+sse2"
 45 |           - target: x86_64-apple-darwin
 46 |             os: macos-14
 47 |             artifact_name: macos-amd64
 48 |             rustflags: "-C target-feature=+aes,+sse2"
 49 |           - target: aarch64-apple-darwin
 50 |             os: macos-14
 51 |             artifact_name: macos-arm64
 52 |             rustflags: "-C target-feature=+aes,+neon"
 53 | 
 54 |     steps:
 55 |       - name: Checkout repository
 56 |         uses: actions/checkout@v4
 57 | 
 58 |       - name: Install Rust
 59 |         uses: dtolnay/rust-toolchain@stable
 60 | 
 61 |       - name: Add target
 62 |         run: rustup target add ${{ matrix.target }}
 63 | 
 64 |       - name: Install cross-compilation toolchain (Linux ARM only)
 65 |         if: matrix.target == 'aarch64-unknown-linux-gnu'
 66 |         run: |
 67 |           sudo apt-get update
 68 |           sudo apt-get install -y gcc-aarch64-linux-gnu
 69 | 
 70 |       - name: Build release binary
 71 |         env:
 72 |           RUSTFLAGS: ${{ matrix.rustflags }}
 73 |           CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: ${{ matrix.target == 'aarch64-unknown-linux-gnu' && 'aarch64-linux-gnu-gcc' || '' }}
 74 |         run: |
 75 |           if [ "${{ matrix.target }}" = "aarch64-unknown-linux-gnu" ]; then
 76 |             export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc
 77 |           fi
 78 |           cargo build --release --target ${{ matrix.target }}
 79 | 
 80 |       - name: Package binary
 81 |         run: |
 82 |           if [[ "${{ matrix.os }}" == macos* ]]; then
 83 |             BINARY_NAME=$(find target/${{ matrix.target }}/release -maxdepth 1 -type f -perm +111 -print0 | xargs -0 basename | head -1)
 84 |           else
 85 |             BINARY_NAME=$(find target/${{ matrix.target }}/release -maxdepth 1 -type f -executable -print0 | xargs -0 basename | head -1)
 86 |           fi
 87 |           
 88 |           echo "Packaging binary: $BINARY_NAME"
 89 |           mkdir -p release
 90 |           cp target/${{ matrix.target }}/release/$BINARY_NAME release/$BINARY_NAME
 91 |           tar -C release -czf ${{ matrix.artifact_name }}.tar.gz $BINARY_NAME
 92 | 
 93 |       - name: Upload artifact
 94 |         uses: actions/upload-artifact@v4
 95 |         with:
 96 |           name: ${{ matrix.artifact_name }}-binary
 97 |           path: ${{ matrix.artifact_name }}.tar.gz
 98 | 
 99 |   create_release:
100 |     needs: build
101 |     runs-on: ubuntu-latest
102 |     steps:
103 |       - name: Checkout repository
104 |         uses: actions/checkout@v4
105 | 
106 |       - name: Download all artifacts
107 |         uses: actions/download-artifact@v4
108 |         with:
109 |           path: artifacts
110 | 
111 |       - name: Create GitHub Release
112 |         id: create_release
113 |         uses: softprops/action-gh-release@v1
114 |         with:
115 |           tag_name: ${{ github.ref_name }}
116 |           name: Release ${{ github.ref_name }}
117 |           body: "Automated release for version ${{ github.ref_name }}"
118 |           files: |
119 |             artifacts/*/*.tar.gz
120 |         env:
121 |           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
122 | 


--------------------------------------------------------------------------------
/src/interactive.rs:
--------------------------------------------------------------------------------
  1 | use crate::{fileinfo::FileInfo, formatter::Formatter, params::Params};
  2 | use anyhow::Result;
  3 | use dashmap::DashMap;
  4 | use prettytable::{format, row, Table};
  5 | use std::sync::atomic::AtomicU64;
  6 | use std::{
  7 |     io::{self, Write},
  8 |     sync::Arc,
  9 | };
 10 | 
 11 | pub struct Interactive;
 12 | 
 13 | impl Interactive {
 14 |     pub fn init(result: Arc<DashMap<u128, Vec<FileInfo>>>, app_args: &Params) -> Result<()> {
 15 |         let store = result.clone();
 16 |         if store.is_empty() {
 17 |             println!("No duplicates found matching your search criteria.");
 18 |         }
 19 | 
 20 |         let printed_count: AtomicU64 = AtomicU64::new(0);
 21 | 
 22 |         store
 23 |             .iter()
 24 |             .filter(|i| i.value().len() > 1)
 25 |             .enumerate()
 26 |             .for_each(|(gindex, i)| {
 27 |                 printed_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
 28 |                 let group = i.value();
 29 |                 let mut itable = Table::new();
 30 |                 itable.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
 31 |                 itable.set_titles(row!["index", "filename", "size", "updated_at"]);
 32 | 
 33 |                 let max_path_size = group
 34 |                     .iter()
 35 |                     .map(|f| f.path.iter().count())
 36 |                     .max()
 37 |                     .unwrap_or_default();
 38 | 
 39 |                 group.iter().enumerate().for_each(|(index, file)| {
 40 |                     itable.add_row(row![
 41 |                         index,
 42 |                         Formatter::human_path(file, app_args, max_path_size).unwrap_or_default(),
 43 |                         Formatter::human_filesize(file).unwrap_or_default(),
 44 |                         Formatter::human_mtime(file).unwrap_or_default()
 45 |                     ]);
 46 |                 });
 47 | 
 48 |                 Self::process_group_action(group, gindex, result.len(), itable);
 49 |             });
 50 | 
 51 |         if printed_count.load(std::sync::atomic::Ordering::Relaxed) < 1 {
 52 |             println!("No duplicates found matching your search criteria.");
 53 |         }
 54 | 
 55 |         Ok(())
 56 |     }
 57 | 
 58 |     pub fn scan_group_confirmation() -> Result<bool> {
 59 |         print!("\nconfirm? [y/N]: ");
 60 |         std::io::stdout().flush()?;
 61 |         let mut user_input = String::new();
 62 |         io::stdin().read_line(&mut user_input)?;
 63 | 
 64 |         match user_input.trim() {
 65 |             "Y" | "y" => Ok(true),
 66 |             _ => Ok(false),
 67 |         }
 68 |     }
 69 | 
 70 |     pub fn scan_group_instruction() -> Result<String> {
 71 |         println!("\nEnter the indices of the files you want to delete.");
 72 |         println!("You can enter multiple files using commas to seperate file indices.");
 73 |         println!("example: 1,2");
 74 |         print!("\n> ");
 75 |         std::io::stdout().flush()?;
 76 |         let mut user_input = String::new();
 77 |         io::stdin().read_line(&mut user_input)?;
 78 | 
 79 |         Ok(user_input)
 80 |     }
 81 | 
 82 |     pub fn process_group_action(
 83 |         duplicates: &Vec<FileInfo>,
 84 |         dup_index: usize,
 85 |         dup_size: usize,
 86 |         table: Table,
 87 |     ) {
 88 |         println!("\nDuplicate Set {} of {}\n", dup_index + 1, dup_size);
 89 |         table.printstd();
 90 |         let files_to_delete = Self::scan_group_instruction().unwrap_or_default();
 91 |         let parsed_file_indices = files_to_delete
 92 |             .trim()
 93 |             .split(',')
 94 |             .filter(|element| !element.is_empty())
 95 |             .map(|index| index.parse::<usize>().unwrap_or_default())
 96 |             .collect::<Vec<usize>>();
 97 | 
 98 |         if parsed_file_indices
 99 |             .clone()
100 |             .into_iter()
101 |             .any(|index| index > (duplicates.len() - 1))
102 |         {
103 |             println!("Err: File Index Out of Bounds!");
104 |             return Self::process_group_action(duplicates, dup_index, dup_size, table);
105 |         }
106 | 
107 |         print!("{esc}[2J{esc}[1;1H", esc = 27 as char);
108 | 
109 |         if parsed_file_indices.is_empty() {
110 |             return;
111 |         }
112 | 
113 |         let files_to_delete = parsed_file_indices
114 |             .into_iter()
115 |             .map(|index| duplicates[index].clone());
116 | 
117 |         println!("\nThe following files will be deleted:");
118 |         files_to_delete
119 |             .clone()
120 |             .enumerate()
121 |             .for_each(|(index, file)| {
122 |                 println!("{}: {}", index, file.path.display());
123 |             });
124 | 
125 |         match Self::scan_group_confirmation().unwrap() {
126 |             true => {
127 |                 files_to_delete.into_iter().for_each(|file| {
128 |                     match std::fs::remove_file(file.path.clone()) {
129 |                         Ok(_) => println!("DELETED: {}", file.path.display()),
130 |                         Err(_) => println!("FAILED: {}", file.path.display()),
131 |                     }
132 |                 });
133 |             }
134 |             false => println!("\nCancelled Delete Operation."),
135 |         }
136 |     }
137 | }
138 | 


--------------------------------------------------------------------------------
/src/scanner.rs:
--------------------------------------------------------------------------------
  1 | use crate::{fileinfo::FileInfo, params::Params};
  2 | use anyhow::Result;
  3 | use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
  4 | use std::sync::{Arc, Mutex};
  5 | use std::{path::Path, time::Duration};
  6 | 
  7 | use globwalk::{GlobWalker, GlobWalkerBuilder};
  8 | 
  9 | pub struct Scanner {
 10 |     pub directory: Box<Path>,
 11 |     pub min_depth: Option<usize>,
 12 |     pub max_depth: Option<usize>,
 13 |     pub include_types: Option<String>,
 14 |     pub exclude_types: Option<String>,
 15 |     pub min_size: Option<u64>,
 16 |     pub follow_links: bool,
 17 |     pub progress: bool,
 18 | }
 19 | 
 20 | impl Scanner {
 21 |     pub fn new(app_args: Arc<Params>) -> Result<Self> {
 22 |         Ok(Self {
 23 |             directory: app_args.get_directory()?.into_boxed_path(),
 24 |             include_types: app_args.types.clone(),
 25 |             exclude_types: app_args.exclude_types.clone(),
 26 |             min_depth: app_args.min_depth,
 27 |             max_depth: app_args.max_depth,
 28 |             min_size: app_args.get_min_size(),
 29 |             follow_links: app_args.follow_links,
 30 |             progress: app_args.progress,
 31 |         })
 32 |     }
 33 | 
 34 |     fn scan_patterns(&self) -> Result<Vec<String>> {
 35 |         let include_types = match &self.include_types {
 36 |             Some(ftypes) => Some(format!("**/*.{{{ftypes}}}")),
 37 |             None => Some("**/*".to_string()),
 38 |         };
 39 | 
 40 |         let exclude_types = self
 41 |             .exclude_types
 42 |             .as_ref()
 43 |             .map(|ftypes| format!("!**/*.{{{ftypes}}}"));
 44 | 
 45 |         Ok(vec![include_types, exclude_types]
 46 |             .into_iter()
 47 |             .flatten()
 48 |             .collect())
 49 |     }
 50 | 
 51 |     fn attach_link_opts(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
 52 |         Ok(walker.follow_links(self.follow_links))
 53 |     }
 54 | 
 55 |     fn attach_walker_min_depth(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
 56 |         match self.min_depth {
 57 |             Some(min_depth) => Ok(walker.min_depth(min_depth)),
 58 |             None => Ok(walker),
 59 |         }
 60 |     }
 61 | 
 62 |     fn attach_walker_max_depth(&self, walker: GlobWalkerBuilder) -> Result<GlobWalkerBuilder> {
 63 |         match self.max_depth {
 64 |             Some(max_depth) => Ok(walker.max_depth(max_depth)),
 65 |             None => Ok(walker),
 66 |         }
 67 |     }
 68 |     fn build_walker(&self) -> Result<GlobWalker> {
 69 |         let walker = Ok(GlobWalkerBuilder::from_patterns(
 70 |             self.directory.clone(),
 71 |             &self.scan_patterns()?,
 72 |         ))
 73 |         .and_then(|walker| self.attach_walker_min_depth(walker))
 74 |         .and_then(|walker| self.attach_walker_max_depth(walker))
 75 |         .and_then(|walker| self.attach_link_opts(walker))?;
 76 | 
 77 |         Ok(walker.build()?)
 78 |     }
 79 | 
 80 |     pub fn scan(
 81 |         &self,
 82 |         files: Arc<Mutex<Vec<FileInfo>>>,
 83 |         progress_bar_box: Arc<MultiProgress>,
 84 |     ) -> Result<()> {
 85 |         let progress_bar = match self.progress {
 86 |             true => progress_bar_box.add(ProgressBar::new_spinner()),
 87 |             false => ProgressBar::hidden(),
 88 |         };
 89 | 
 90 |         let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
 91 |         progress_bar.set_style(progress_style);
 92 |         progress_bar.enable_steady_tick(Duration::from_millis(50));
 93 |         progress_bar.set_message("paths mapped");
 94 |         let min_size = self.min_size.unwrap_or(0);
 95 | 
 96 |         self.build_walker()?
 97 |             .filter_map(Result::ok)
 98 |             .map(|entity| entity.into_path())
 99 |             .inspect(|_path| progress_bar.inc(1))
100 |             .filter(|path| path.is_file())
101 |             .map(FileInfo::new)
102 |             .filter_map(Result::ok)
103 |             .filter(|file| file.size >= min_size)
104 |             .for_each(|file| {
105 |                 let mut flock = files.lock().unwrap();
106 |                 flock.push(file);
107 |             });
108 | 
109 |         progress_bar.finish_with_message("paths mapped");
110 |         Ok(())
111 |     }
112 | }
113 | 
114 | #[cfg(test)]
115 | mod tests {
116 |     use crate::fileinfo::FileInfo;
117 |     use crate::params::Params;
118 |     use std::fs::File;
119 |     use std::sync::{Arc, Mutex};
120 | 
121 |     use super::Scanner;
122 |     use indicatif::MultiProgress;
123 |     use tempfile::TempDir;
124 | 
125 |     #[test]
126 |     fn ensure_file_include_type_filter_includes_expected_file_types() {
127 |         let root =
128 |             TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir");
129 |         [
130 |             "this-is-a-js-file.js",
131 |             "this-is-a-css-file.css",
132 |             "this-is-a-csv-file.csv",
133 |             "this-is-a-rust-file.rs",
134 |         ]
135 |         .iter()
136 |         .for_each(|path| {
137 |             File::create_new(root.path().join(path)).unwrap_or_else(|_| {
138 |                 panic!("unable to create file {path}");
139 |             });
140 |         });
141 | 
142 |         let params = Params {
143 |             types: Some(String::from("js,csv")),
144 |             dir: Some(root.path().into()),
145 |             ..Default::default()
146 |         };
147 | 
148 |         let progress = Arc::new(MultiProgress::new());
149 |         let scanlist = Arc::new(Mutex::<Vec<FileInfo>>::new(vec![]));
150 |         let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed");
151 | 
152 |         scanner
153 |             .scan(scanlist.clone(), progress)
154 |             .expect("scanning failed.");
155 | 
156 |         let scan_list_mg = scanlist.lock().unwrap();
157 | 
158 |         assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
159 |             == root.path().join("this-is-a-js-file.js").to_str().unwrap()));
160 | 
161 |         assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
162 |             == root.path().join("this-is-a-csv-file.csv").to_str().unwrap()));
163 | 
164 |         assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
165 |             != root.path().join("this-is-a-css-file.css").to_str().unwrap()));
166 | 
167 |         assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
168 |             != root.path().join("this-is-a-rust-file.rs").to_str().unwrap()));
169 |     }
170 | 
171 |     #[test]
172 |     fn ensure_file_exclude_type_filter_excludes_expected_file_types() {
173 |         let root =
174 |             TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir");
175 |         [
176 |             "this-is-a-js-file.js",
177 |             "this-is-a-css-file.css",
178 |             "this-is-a-csv-file.csv",
179 |             "this-is-a-rust-file.rs",
180 |         ]
181 |         .iter()
182 |         .for_each(|path| {
183 |             File::create_new(root.path().join(path)).unwrap_or_else(|_| {
184 |                 panic!("unable to create file {path}");
185 |             });
186 |         });
187 | 
188 |         let params = Params {
189 |             exclude_types: Some(String::from("js,csv")),
190 |             dir: Some(root.path().into()),
191 |             ..Default::default()
192 |         };
193 | 
194 |         let progress = Arc::new(MultiProgress::new());
195 |         let scanlist = Arc::new(Mutex::<Vec<FileInfo>>::new(vec![]));
196 |         let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed");
197 | 
198 |         scanner
199 |             .scan(scanlist.clone(), progress)
200 |             .expect("scanning failed.");
201 | 
202 |         let scan_list_mg = scanlist.lock().unwrap();
203 | 
204 |         assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
205 |             != root.path().join("this-is-a-js-file.js").to_str().unwrap()));
206 | 
207 |         assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
208 |             != root.path().join("this-is-a-csv-file.csv").to_str().unwrap()));
209 | 
210 |         assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
211 |             == root.path().join("this-is-a-css-file.css").to_str().unwrap()));
212 | 
213 |         assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
214 |             == root.path().join("this-is-a-rust-file.rs").to_str().unwrap()));
215 |     }
216 | 
217 |     #[test]
218 |     fn complex_file_type_params() {
219 |         let root =
220 |             TempDir::with_prefix("deduplicator_test_root").expect("unable to create tempdir");
221 |         [
222 |             "this-is-a-js-file.js",
223 |             "this-is-a-css-file.css",
224 |             "this-is-a-csv-file.csv",
225 |             "this-is-a-rust-file.rs",
226 |         ]
227 |         .iter()
228 |         .for_each(|path| {
229 |             File::create_new(root.path().join(path)).unwrap_or_else(|_| {
230 |                 panic!("unable to create file {path}");
231 |             });
232 |         });
233 | 
234 |         let params = Params {
235 |             types: Some(String::from("js,csv,rs")),
236 |             exclude_types: Some(String::from("csv")),
237 |             dir: Some(root.path().into()),
238 |             ..Default::default()
239 |         };
240 | 
241 |         let progress = Arc::new(MultiProgress::new());
242 |         let scanlist = Arc::new(Mutex::<Vec<FileInfo>>::new(vec![]));
243 |         let scanner = Scanner::new(Arc::new(params)).expect("scanner initialization failed");
244 | 
245 |         scanner
246 |             .scan(scanlist.clone(), progress)
247 |             .expect("scanning failed.");
248 | 
249 |         let scan_list_mg = scanlist.lock().unwrap();
250 | 
251 |         assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
252 |             == root.path().join("this-is-a-js-file.js").to_str().unwrap()));
253 | 
254 |         assert!(scan_list_mg.iter().all(|f| f.path.to_str().unwrap()
255 |             != root.path().join("this-is-a-csv-file.csv").to_str().unwrap()));
256 | 
257 |         assert!(scan_list_mg.iter().any(|f| f.path.to_str().unwrap()
258 |             == root.path().join("this-is-a-rust-file.rs").to_str().unwrap()));
259 |     }
260 | }
261 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | <h1 align="center">Deduplicator</h1>
  2 | 
  3 | <p align="center">
  4 |   Find, Sort, Filter & Delete duplicate files 
  5 | </p>
  6 | 
  7 | ## Usage
  8 | 
  9 | ```bash
 10 | find,filter and delete duplicate files
 11 | 
 12 | Usage: deduplicator [OPTIONS] [scan_dir_path]
 13 | 
 14 | Arguments:
 15 |   [scan_dir_path]  Run Deduplicator on dir different from pwd (e.g., ~/Pictures )
 16 | 
 17 | Options:
 18 |   -T, --exclude-types <EXCLUDE_TYPES>  Exclude Filetypes [default = none]
 19 |   -t, --types <TYPES>                  Filetypes to deduplicate [default = all]
 20 |   -i, --interactive                    Delete files interactively
 21 |   -m, --min-size <MIN_SIZE>            Minimum filesize of duplicates to scan (e.g., 100B/1K/2M/3G/4T) [default: 1b]
 22 |   -D, --max-depth <MAX_DEPTH>          Max Depth to scan while looking for duplicates
 23 |   -d, --min-depth <MIN_DEPTH>          Min Depth to scan while looking for duplicates
 24 |   -f, --follow-links                   Follow links while scanning directories
 25 |   -s, --strict                         Guarantees that two files are duplicate (performs a full hash)
 26 |   -p, --progress                       Show Progress spinners & metrics
 27 |   -h, --help                           Print help
 28 |   -V, --version                        Print version
 29 | ```
 30 | ### Examples
 31 | 
 32 | ```bash
 33 | # Scan for duplicates recursively from the current dir, only look for png, jpg & pdf file types & interactively delete files
 34 | deduplicator -t pdf,jpg,png -i
 35 | 
 36 | # Scan for duplicates recursively from current dir, exclude png and jpg file types
 37 | deduplicator -T jpg,png
 38 | 
 39 | # Scan for duplicates recursively from the ~/Pictures dir, only look for png, jpeg, jpg & pdf file types & interactively delete files
 40 | deduplicator ~/Pictures/ -t png,jpeg,jpg,pdf -i
 41 | 
 42 | # Scan for duplicates in the ~/Pictures without recursing into subdirectories
 43 | deduplicator ~/Pictures --max-depth 0
 44 | 
 45 | # look for duplicates in the ~/.config directory while also recursing into symbolic link paths
 46 | deduplicator ~/.config --follow-links
 47 | 
 48 | # scan for duplicates that are greater than 100mb in the ~/Media directory
 49 | deduplicator ~/Media --min-size 100mb
 50 | ```
 51 | 
 52 | ## Demo
 53 | ![demo](https://github.com/user-attachments/assets/bdb95831-542d-4902-a458-4e0f5d171a33)
 54 | 
 55 | 
 56 | 
 57 | 
 58 | ## Installation
 59 | Currently, you can only install deduplicator using cargo package manager.
 60 | 
 61 | ### Cargo
 62 | > GxHash relies on aes hardware acceleration, so please set `RUSTFLAGS` to `"-C target-feature=+aes"` or `"-C target-cpu=native"` before
 63 | > installing.
 64 | 
 65 | #### install from crates.io (stable)
 66 | 
 67 | ```bash
 68 | $ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator
 69 | 
 70 | # or
 71 | 
 72 | $ RUSTFLAGS="-C target-feature=+aes,+sse2" cargo install deduplicator
 73 | ```
 74 | 
 75 | #### install from git (nightly)
 76 | ```bash
 77 | $ RUSTFLAGS="-C target-cpu=native" cargo install deduplicator --git https://github.com/sreedevk/deduplicator
 78 | 
 79 | # or
 80 | 
 81 | $ RUSTFLAGS="-C target-feature=+aes,+sse2" cargo install --git https://github.com/sreedevk/deduplicator
 82 | ```
 83 | 
 84 | ### Manual Installation
 85 | - Download the right pre-compiled binary archive for your platform from [github release page](https://github.com/sreedevk/deduplicator/releases/tag/latest).
 86 | - Decompress it using `tar -zxvf <archive>.tar.gz` 
 87 | - Move it to a directory included in `$PATH`. 
 88 |     - ideally `/usr/local/bin/`.
 89 | 
 90 | ## Performance
 91 | Deduplicator uses size comparison and [GxHash](https://docs.rs/gxhash/latest/gxhash/) to quickly check a large number of files to find duplicates. its also heavily parallelized. The default behavior of deduplicator is to only hash the first page (4K) of the file. This is to ensure that performance is the default priority. You can modify this behavior by using the `--strict` flag which will hash the whole file and ensure that 2 files are indeed duplicates. I'll add benchmarks in future versions.
 92 | 
 93 | ### Benchmarks
 94 | I've used hyperfine to run deduplicator on files generated by the rake file at `rakelib/benchmark.rake`. The Benchmarking accuracy can further be improved by isolating runs inside restricted docker containers. I'll include that in the future. For now, here's the hyperfine output on my i7-12800H laptop with 32G of RAM.
 95 | 
 96 | #### Fewer Large Files
 97 | ```
 98 | # hyperfine -N --warmup 80 './target/release/deduplicator bench_artifacts'
 99 | Benchmark 1: ./target/release/deduplicator bench_artifacts
100 |   Time (mean ± σ):       2.2 ms ±   0.4 ms    [User: 2.2 ms, System: 4.4 ms]
101 |   Range (min … max):     1.3 ms …   7.1 ms    1522 runs
102 | 
103 | dust 'bench_artifacts'
104 | 
105 |  54M   ┌── file_0_fwds.bin   │████                                                               │   2%
106 | 122M   ├── file_1_fwds.bin   │████████                                                           │   5%
107 | 390M   ├── file_0_fwdcbss.bin│██████████████████████████                                         │  15%
108 | 390M   ├── file_0_fwscas.bin │██████████████████████████                                         │  15%
109 | 390M   ├── file_0_fwss.bin   │██████████████████████████                                         │  15%
110 | 390M   ├── file_1_fwdcbss.bin│██████████████████████████                                         │  15%
111 | 390M   ├── file_1_fwscas.bin │██████████████████████████                                         │  15%
112 | 390M   ├── file_1_fwss.bin   │██████████████████████████                                         │  15%
113 | 2.5G ┌─┴ bench_artifacts     │██████████████████████████████████████████████████████████████████ │ 100%
114 | ```
115 | 
116 | #### Many Small Files
117 | ```
118 | # hyperfine --warmup 20 './target/release/deduplicator bench_artifacts'
119 | Benchmark 1: ./target/release/deduplicator bench_artifacts
120 |   Time (mean ± σ):      40.1 ms ±   2.3 ms    [User: 251.0 ms, System: 277.3 ms]
121 |   Range (min … max):    35.0 ms …  45.9 ms    72 runs
122 | 
123 | dust 'bench_artifacts'
124 | 3.9M   ┌── file_992_fwscas.bin │█                                                                │   0%
125 | 3.9M   ├── file_992_fwss.bin   │█                                                                │   0%
126 | 3.9M   ├── file_993_fwdcbss.bin│█                                                                │   0%
127 | 3.9M   ├── file_993_fwscas.bin │█                                                                │   0%
128 | 3.9M   ├── file_993_fwss.bin   │█                                                                │   0%
129 | 3.9M   ├── file_994_fwdcbss.bin│█                                                                │   0%
130 | 3.9M   ├── file_994_fwscas.bin │█                                                                │   0%
131 | 3.9M   ├── file_994_fwss.bin   │█                                                                │   0%
132 | 3.9M   ├── file_995_fwdcbss.bin│█                                                                │   0%
133 | 3.9M   ├── file_995_fwscas.bin │█                                                                │   0%
134 | 3.9M   ├── file_995_fwss.bin   │█                                                                │   0%
135 | 3.9M   ├── file_996_fwdcbss.bin│█                                                                │   0%
136 | 3.9M   ├── file_996_fwscas.bin │█                                                                │   0%
137 | 3.9M   ├── file_996_fwss.bin   │█                                                                │   0%
138 | 3.9M   ├── file_997_fwdcbss.bin│█                                                                │   0%
139 | 3.9M   ├── file_997_fwscas.bin │█                                                                │   0%
140 | 3.9M   ├── file_997_fwss.bin   │█                                                                │   0%
141 | 3.9M   ├── file_998_fwdcbss.bin│█                                                                │   0%
142 | 3.9M   ├── file_998_fwscas.bin │█                                                                │   0%
143 | 3.9M   ├── file_998_fwss.bin   │█                                                                │   0%
144 | 3.9M   ├── file_999_fwdcbss.bin│█                                                                │   0%
145 | 3.9M   ├── file_999_fwscas.bin │█                                                                │   0%
146 | 3.9M   ├── file_999_fwss.bin   │█                                                                │   0%
147 | 3.9M   ├── file_99_fwdcbss.bin │█                                                                │   0%
148 | 3.9M   ├── file_99_fwscas.bin  │█                                                                │   0%
149 | 3.9M   ├── file_99_fwss.bin    │█                                                                │   0%
150 | 3.9M   ├── file_9_fwdcbss.bin  │█                                                                │   0%
151 | 3.9M   ├── file_9_fwscas.bin   │█                                                                │   0%
152 | 3.9M   ├── file_9_fwss.bin     │█                                                                │   0%
153 |  11G ┌─┴ bench_artifacts       │████████████████████████████████████████████████████████████████ │ 100%
154 | ```
155 | 
156 | ## proposed
157 | - [ ] parallelization
158 |     - [ ] scanning + processing sw + processing hw + formatting + printing
159 | - [ ] user supplied cache file path for faster re-runs
160 | - [ ] hardlinks / symlinks support
161 | - [ ] max file path size should use the last set of duplicates
162 | - [ ] add more unit tests
163 |     - [ ] test against different filesystems
164 |     - [ ] test against different file name encodings
165 | - [ ] restore json output (was removed in 0.3 due to quality issues)
166 | - [ ] fix memory leak on very large filesystems
167 |     - [ ] maybe use a bloom filter
168 |     - [ ] reduce FileInfo size
169 | - [ ] tui
170 | - [ ] change the default hashing method to include the first & last page of a file (8K)
171 | - [ ] provide option to localize duplicate detection to arbitrary levels relative to current directory
172 | - [ ] localize file meta store locks to sub path levels to avoid global lock contention from multiple threads.
173 | - [ ] bulk operations
174 |     - [ ] --keep-latest
175 |     - [ ] --keep-oldest
176 |     - [ ] --keep-last-modified
177 |     - [ ] --keep-first-modified
178 | 
179 | - [ ] fix: partial hash collision - a file full of null bytes ("\0") and an empty file. This is a known trade off in gxhash.
180 |     - [ ] include initial pages and final pages of the file 
181 |     - [ ] append the offset between the last initial page hashed and the first final page hashed in the content passed to the hasher.
182 | - [ ] potential optimizations
183 |     -  [ ] lookup the memory efficiency gains if instead of directly inserting into a hashmap, deduplicator looksup the file in a bloom filter. this way, the duplicate store hashmap does
184 |        not require to be locked for every single file. The bloom filter can be stored in an atomically updateable type to improve performance as well.
185 | 
186 | ## v0.3.2
187 | - [x] fix: single file groups are printed to screen
188 | - [x] fix: --exclude-types and --types flag behave identically.
189 | 
190 | ## v0.3.1
191 | - [x] parallelization
192 |     - [x] (scanning + processing sw + processing hw) & formatting & printing
193 | - [x] remove formatting step and write directly to stdout
194 | - [x] simplify output to improve performance
195 | - [x] increase the number of pages hashed in partial hashing
196 | - [x] updated dependencies
197 | - [x] fix: full file hash collision between a file full of null bytes ("\0") and an empty file. This is a known trade off in gxhash.
198 |     - [x] appending the file size at the end of content before hashing.
199 | - [x] created an automated release pipeline
200 | 
201 | ## v0.3.0
202 | - [x] parallelization
203 |     - [x] (scanning) + (processing sw & processing hw & formatting & printing)
204 | - [x] reduce cloning values on the heap
205 | - [x] add a partial hashing mode (--strict)
206 | - [x] add unit tests
207 | - [x] add silent mode
208 | - [x] update documentation
209 | - [x] remove color output
210 | - [x] progress bar improvements
211 |     - [x] use progress bar groups
212 | - [x] remove broken json rendering
213 | - [x] add benchmarks
214 | 


--------------------------------------------------------------------------------
/src/processor.rs:
--------------------------------------------------------------------------------
  1 | use anyhow::Result;
  2 | use dashmap::DashMap;
  3 | use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
  4 | use rayon::iter::IntoParallelRefMutIterator;
  5 | use rayon::prelude::{IntoParallelIterator, ParallelIterator};
  6 | use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
  7 | use std::sync::{Arc, Mutex, TryLockError, TryLockResult};
  8 | use std::time::Duration;
  9 | use unicode_segmentation::UnicodeSegmentation;
 10 | 
 11 | use crate::fileinfo::FileInfo;
 12 | use crate::params::Params;
 13 | 
 14 | pub struct Processor {}
 15 | 
 16 | impl Processor {
 17 |     pub fn hashwise(
 18 |         app_args: Arc<Params>,
 19 |         sw_store: Arc<DashMap<u64, Vec<FileInfo>>>,
 20 |         hw_store: Arc<DashMap<u128, Vec<FileInfo>>>,
 21 |         progress_bar_box: Arc<MultiProgress>,
 22 |         max_file_size: Arc<AtomicU64>,
 23 |         seed: i64,
 24 |         sw_sorting_finished: Arc<AtomicBool>,
 25 |     ) -> Result<()> {
 26 |         let progress_bar = match app_args.progress {
 27 |             true => progress_bar_box.add(ProgressBar::new_spinner()),
 28 |             false => ProgressBar::hidden(),
 29 |         };
 30 | 
 31 |         let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
 32 |         progress_bar.set_style(progress_style);
 33 |         progress_bar.enable_steady_tick(Duration::from_millis(50));
 34 |         progress_bar.set_message("files grouped by hash.");
 35 | 
 36 |         loop {
 37 |             let keys: Vec<u64> = sw_store
 38 |                 .clone()
 39 |                 .iter()
 40 |                 .filter(|i| !i.value().iter().all(|x| x.is_sw_processed()))
 41 |                 .filter(|i| i.value().len() > 1)
 42 |                 .map(|i| *i.key())
 43 |                 .collect();
 44 | 
 45 |             if keys.is_empty() {
 46 |                 match sw_sorting_finished.load(std::sync::atomic::Ordering::Relaxed) {
 47 |                     true => {
 48 |                         progress_bar.finish_with_message("files grouped by hash.");
 49 |                         break Ok(());
 50 |                     }
 51 |                     false => continue,
 52 |                 }
 53 |             } else {
 54 |                 keys.into_par_iter().for_each(|key| {
 55 |                     let mut group: Vec<FileInfo> = sw_store.get(&key).unwrap().to_vec();
 56 |                     if group.len() > 1 {
 57 |                         group.par_iter_mut().for_each(|file| {
 58 |                             progress_bar.inc(1);
 59 |                             file.sw_processed();
 60 | 
 61 |                             let fhash = match app_args.strict {
 62 |                                 true => file.hash(seed).expect("hashing file failed."),
 63 |                                 false => file.initpages_hash(seed).expect("hashing file failed."),
 64 |                             };
 65 | 
 66 |                             Self::compare_and_update_max_path_len(
 67 |                                 max_file_size.clone(),
 68 |                                 file.path.to_string_lossy().graphemes(true).count() as u64,
 69 |                             );
 70 | 
 71 |                             hw_store
 72 |                                 .entry(fhash)
 73 |                                 .and_modify(|fileset| fileset.push(file.clone()))
 74 |                                 .or_insert_with(|| vec![file.clone()]);
 75 |                         });
 76 |                     };
 77 |                 });
 78 |             }
 79 |         }
 80 |     }
 81 | 
 82 |     pub fn compare_and_update_max_path_len(current: Arc<AtomicU64>, next: u64) {
 83 |         if current.load(Ordering::Relaxed) < next {
 84 |             current.store(next, Ordering::Release);
 85 |         }
 86 |     }
 87 | 
 88 |     pub fn sizewise(
 89 |         app_args: Arc<Params>,
 90 |         scanner_finished: Arc<AtomicBool>,
 91 |         store: Arc<DashMap<u64, Vec<FileInfo>>>,
 92 |         files: Arc<Mutex<Vec<FileInfo>>>,
 93 |         progress_bar_box: Arc<MultiProgress>,
 94 |     ) -> Result<()> {
 95 |         let progress_bar = match app_args.progress {
 96 |             true => progress_bar_box.add(ProgressBar::new_spinner()),
 97 |             false => ProgressBar::hidden(),
 98 |         };
 99 | 
100 |         let progress_style = ProgressStyle::with_template("[{elapsed_precise}] {pos:>7} {msg}")?;
101 |         progress_bar.set_style(progress_style);
102 |         progress_bar.enable_steady_tick(Duration::from_millis(50));
103 |         progress_bar.set_message("files grouped by size");
104 | 
105 |         loop {
106 |             let fileopt: Option<FileInfo> = {
107 |                 match files.try_lock() {
108 |                     Ok(mut flist) => flist.pop(),
109 |                     TryLockResult::Err(TryLockError::WouldBlock) => None,
110 |                     _ => None,
111 |                 }
112 |             };
113 | 
114 |             match fileopt {
115 |                 Some(file) => {
116 |                     progress_bar.inc(1);
117 |                     store
118 |                         .entry(file.size)
119 |                         .and_modify(|fileset| fileset.push(file.clone()))
120 |                         .or_insert_with(|| vec![file]);
121 |                     continue;
122 |                 }
123 |                 None => match scanner_finished.load(std::sync::atomic::Ordering::Relaxed) {
124 |                     true => {
125 |                         progress_bar.finish_with_message("files grouped by size");
126 |                         break Ok(());
127 |                     }
128 |                     false => continue,
129 |                 },
130 |             }
131 |         }
132 |     }
133 | }
134 | 
135 | #[cfg(test)]
136 | mod tests {
137 |     use anyhow::Result;
138 |     use dashmap::DashMap;
139 |     use indicatif::MultiProgress;
140 |     use rand::Rng;
141 |     use std::fs::File;
142 |     use std::io::Write;
143 |     use std::sync::atomic::{AtomicBool, AtomicU64};
144 |     use std::sync::{Arc, Mutex};
145 |     use tempfile::TempDir;
146 | 
147 |     use crate::{fileinfo::FileInfo, params::Params};
148 | 
149 |     use super::Processor;
150 | 
151 |     fn generate_bytes(size: usize) -> Vec<u8> {
152 |         let mut rng = rand::rng();
153 |         (0..size).map(|_| rng.random::<u8>()).collect::<Vec<u8>>()
154 |     }
155 | 
156 |     #[test]
157 |     fn hashwise_sorting_two_files_with_identical_init_pages_only_strict_mode() -> Result<()> {
158 |         let root = TempDir::new()?;
159 |         let content = generate_bytes(16384);
160 | 
161 |         let mut content_x = content.clone();
162 |         let mut content_y = content.clone();
163 | 
164 |         content_x.extend(generate_bytes(1720320));
165 |         content_y.extend(generate_bytes(1720320));
166 | 
167 |         let files = [
168 |             (root.path().join("fileone.bin"), content_x),
169 |             (root.path().join("filetwo.bin"), content_y),
170 |         ];
171 | 
172 |         for (fpath, content) in files.iter() {
173 |             let mut f = File::create_new(fpath)?;
174 |             f.write_all(content)?;
175 |         }
176 | 
177 |         let dupstore = Arc::new(DashMap::new());
178 |         let file_queue = Arc::new(Mutex::new(
179 |             files
180 |                 .iter()
181 |                 .map(|f| FileInfo::new(f.0.clone()).unwrap())
182 |                 .collect::<Vec<FileInfo>>(),
183 |         ));
184 | 
185 |         let hw_dupstore = Arc::new(DashMap::new());
186 |         Processor::sizewise(
187 |             Arc::new(Params::default()),
188 |             Arc::new(AtomicBool::new(true)),
189 |             dupstore.clone(),
190 |             file_queue,
191 |             Arc::new(MultiProgress::new()),
192 |         )?;
193 | 
194 |         let args = Params {
195 |             strict: true,
196 |             ..Default::default()
197 |         };
198 | 
199 |         Processor::hashwise(
200 |             Arc::new(args),
201 |             dupstore.clone(),
202 |             hw_dupstore.clone(),
203 |             Arc::new(MultiProgress::new()),
204 |             Arc::new(AtomicU64::new(32)),
205 |             300,
206 |             Arc::new(AtomicBool::new(true)),
207 |         )?;
208 | 
209 |         assert_eq!(hw_dupstore.len(), 2);
210 | 
211 |         Ok(())
212 |     }
213 | 
214 |     #[test]
215 |     fn hashwise_sorting_two_files_with_identical_init_pages_only_fast_mode() -> Result<()> {
216 |         let root = TempDir::new()?;
217 |         let content = generate_bytes(16384);
218 | 
219 |         let mut content_x = content.clone();
220 |         let mut content_y = content.clone();
221 | 
222 |         content_x.extend(generate_bytes(1720320));
223 |         content_y.extend(generate_bytes(1720320));
224 | 
225 |         let files = [
226 |             (root.path().join("fileone.bin"), content_x),
227 |             (root.path().join("filetwo.bin"), content_y),
228 |         ];
229 | 
230 |         for (fpath, content) in files.iter() {
231 |             let mut f = File::create_new(fpath)?;
232 |             f.write_all(content)?;
233 |         }
234 | 
235 |         let dupstore = Arc::new(DashMap::new());
236 |         let file_queue = Arc::new(Mutex::new(
237 |             files
238 |                 .iter()
239 |                 .map(|f| FileInfo::new(f.0.clone()).unwrap())
240 |                 .collect::<Vec<FileInfo>>(),
241 |         ));
242 | 
243 |         let hw_dupstore = Arc::new(DashMap::new());
244 |         Processor::sizewise(
245 |             Arc::new(Params::default()),
246 |             Arc::new(AtomicBool::new(true)),
247 |             dupstore.clone(),
248 |             file_queue,
249 |             Arc::new(MultiProgress::new()),
250 |         )?;
251 | 
252 |         Processor::hashwise(
253 |             Arc::new(Params::default()),
254 |             dupstore.clone(),
255 |             hw_dupstore.clone(),
256 |             Arc::new(MultiProgress::new()),
257 |             Arc::new(AtomicU64::new(32)),
258 |             300,
259 |             Arc::new(AtomicBool::new(true)),
260 |         )?;
261 | 
262 |         assert_eq!(hw_dupstore.len(), 1);
263 | 
264 |         Ok(())
265 |     }
266 | 
267 |     #[test]
268 |     fn hashwise_sorting_two_files_with_identical_data() -> Result<()> {
269 |         let root = TempDir::new()?;
270 |         let content = generate_bytes(282624);
271 |         let files = [
272 |             (root.path().join("fileone.bin"), content.clone()),
273 |             (root.path().join("filetwo.bin"), content.clone()),
274 |         ];
275 | 
276 |         for (fpath, content) in files.iter() {
277 |             let mut f = File::create_new(fpath)?;
278 |             f.write_all(content)?;
279 |         }
280 | 
281 |         let dupstore = Arc::new(DashMap::new());
282 |         let file_queue = Arc::new(Mutex::new(
283 |             files
284 |                 .iter()
285 |                 .map(|f| FileInfo::new(f.0.clone()).unwrap())
286 |                 .collect::<Vec<FileInfo>>(),
287 |         ));
288 | 
289 |         let hw_dupstore = Arc::new(DashMap::new());
290 |         Processor::sizewise(
291 |             Arc::new(Params::default()),
292 |             Arc::new(AtomicBool::new(true)),
293 |             dupstore.clone(),
294 |             file_queue,
295 |             Arc::new(MultiProgress::new()),
296 |         )?;
297 | 
298 |         Processor::hashwise(
299 |             Arc::new(Params::default()),
300 |             dupstore.clone(),
301 |             hw_dupstore.clone(),
302 |             Arc::new(MultiProgress::new()),
303 |             Arc::new(AtomicU64::new(32)),
304 |             300,
305 |             Arc::new(AtomicBool::new(true)),
306 |         )?;
307 | 
308 |         assert_eq!(hw_dupstore.len(), 1);
309 | 
310 |         Ok(())
311 |     }
312 | 
313 |     #[test]
314 |     fn sizewise_sorting_two_files_of_different_sizes() -> Result<()> {
315 |         let root = TempDir::new()?;
316 |         let files = [
317 |             (root.path().join("fileone.bin"), generate_bytes(282624)),
318 |             (root.path().join("filetwo.bin"), generate_bytes(1720320)),
319 |         ];
320 | 
321 |         for (fpath, content) in files.iter() {
322 |             let mut f = File::create_new(fpath)?;
323 |             f.write_all(content)?;
324 |         }
325 | 
326 |         let file_queue = Arc::new(Mutex::new(
327 |             files
328 |                 .iter()
329 |                 .map(|f| FileInfo::new(f.0.clone()).unwrap())
330 |                 .collect::<Vec<FileInfo>>(),
331 |         ));
332 | 
333 |         let dupstore = Arc::new(DashMap::new());
334 | 
335 |         Processor::sizewise(
336 |             Arc::new(Params::default()),
337 |             Arc::new(AtomicBool::new(true)),
338 |             dupstore.clone(),
339 |             file_queue,
340 |             Arc::new(MultiProgress::new()),
341 |         )?;
342 | 
343 |         assert_eq!(dupstore.len(), 2);
344 | 
345 |         Ok(())
346 |     }
347 | 
348 |     #[test]
349 |     fn sizewise_sorting_two_files_of_same_size() -> Result<()> {
350 |         let root = TempDir::new()?;
351 |         let files = [
352 |             (root.path().join("fileone.bin"), generate_bytes(282624)),
353 |             (root.path().join("filetwo.bin"), generate_bytes(282624)),
354 |         ];
355 | 
356 |         for (fpath, content) in files.iter() {
357 |             let mut f = File::create_new(fpath)?;
358 |             f.write_all(content)?;
359 |         }
360 | 
361 |         let file_queue = Arc::new(Mutex::new(
362 |             files
363 |                 .iter()
364 |                 .map(|f| FileInfo::new(f.0.clone()).unwrap())
365 |                 .collect::<Vec<FileInfo>>(),
366 |         ));
367 | 
368 |         let dupstore = Arc::new(DashMap::new());
369 | 
370 |         Processor::sizewise(
371 |             Arc::new(Params::default()),
372 |             Arc::new(AtomicBool::new(true)),
373 |             dupstore.clone(),
374 |             file_queue,
375 |             Arc::new(MultiProgress::new()),
376 |         )?;
377 | 
378 |         assert_eq!(dupstore.len(), 1);
379 | 
380 |         Ok(())
381 |     }
382 | }
383 | 


--------------------------------------------------------------------------------
/Cargo.lock:
--------------------------------------------------------------------------------
   1 | # This file is automatically @generated by Cargo.
   2 | # It is not intended for manual editing.
   3 | version = 4
   4 | 
   5 | [[package]]
   6 | name = "aho-corasick"
   7 | version = "1.1.3"
   8 | source = "registry+https://github.com/rust-lang/crates.io-index"
   9 | checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
  10 | dependencies = [
  11 |  "memchr",
  12 | ]
  13 | 
  14 | [[package]]
  15 | name = "android-tzdata"
  16 | version = "0.1.1"
  17 | source = "registry+https://github.com/rust-lang/crates.io-index"
  18 | checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
  19 | 
  20 | [[package]]
  21 | name = "android_system_properties"
  22 | version = "0.1.5"
  23 | source = "registry+https://github.com/rust-lang/crates.io-index"
  24 | checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
  25 | dependencies = [
  26 |  "libc",
  27 | ]
  28 | 
  29 | [[package]]
  30 | name = "anstream"
  31 | version = "0.6.19"
  32 | source = "registry+https://github.com/rust-lang/crates.io-index"
  33 | checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933"
  34 | dependencies = [
  35 |  "anstyle",
  36 |  "anstyle-parse",
  37 |  "anstyle-query",
  38 |  "anstyle-wincon",
  39 |  "colorchoice",
  40 |  "is_terminal_polyfill",
  41 |  "utf8parse",
  42 | ]
  43 | 
  44 | [[package]]
  45 | name = "anstyle"
  46 | version = "1.0.11"
  47 | source = "registry+https://github.com/rust-lang/crates.io-index"
  48 | checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd"
  49 | 
  50 | [[package]]
  51 | name = "anstyle-parse"
  52 | version = "0.2.7"
  53 | source = "registry+https://github.com/rust-lang/crates.io-index"
  54 | checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
  55 | dependencies = [
  56 |  "utf8parse",
  57 | ]
  58 | 
  59 | [[package]]
  60 | name = "anstyle-query"
  61 | version = "1.1.3"
  62 | source = "registry+https://github.com/rust-lang/crates.io-index"
  63 | checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9"
  64 | dependencies = [
  65 |  "windows-sys 0.59.0",
  66 | ]
  67 | 
  68 | [[package]]
  69 | name = "anstyle-wincon"
  70 | version = "3.0.9"
  71 | source = "registry+https://github.com/rust-lang/crates.io-index"
  72 | checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882"
  73 | dependencies = [
  74 |  "anstyle",
  75 |  "once_cell_polyfill",
  76 |  "windows-sys 0.59.0",
  77 | ]
  78 | 
  79 | [[package]]
  80 | name = "anyhow"
  81 | version = "1.0.98"
  82 | source = "registry+https://github.com/rust-lang/crates.io-index"
  83 | checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
  84 | 
  85 | [[package]]
  86 | name = "autocfg"
  87 | version = "1.5.0"
  88 | source = "registry+https://github.com/rust-lang/crates.io-index"
  89 | checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
  90 | 
  91 | [[package]]
  92 | name = "bitflags"
  93 | version = "2.9.1"
  94 | source = "registry+https://github.com/rust-lang/crates.io-index"
  95 | checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
  96 | 
  97 | [[package]]
  98 | name = "bstr"
  99 | version = "1.12.0"
 100 | source = "registry+https://github.com/rust-lang/crates.io-index"
 101 | checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
 102 | dependencies = [
 103 |  "memchr",
 104 |  "serde",
 105 | ]
 106 | 
 107 | [[package]]
 108 | name = "bumpalo"
 109 | version = "3.19.0"
 110 | source = "registry+https://github.com/rust-lang/crates.io-index"
 111 | checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
 112 | 
 113 | [[package]]
 114 | name = "bytesize"
 115 | version = "2.0.1"
 116 | source = "registry+https://github.com/rust-lang/crates.io-index"
 117 | checksum = "a3c8f83209414aacf0eeae3cf730b18d6981697fba62f200fcfb92b9f082acba"
 118 | 
 119 | [[package]]
 120 | name = "cc"
 121 | version = "1.2.30"
 122 | source = "registry+https://github.com/rust-lang/crates.io-index"
 123 | checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7"
 124 | dependencies = [
 125 |  "shlex",
 126 | ]
 127 | 
 128 | [[package]]
 129 | name = "cfg-if"
 130 | version = "1.0.1"
 131 | source = "registry+https://github.com/rust-lang/crates.io-index"
 132 | checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
 133 | 
 134 | [[package]]
 135 | name = "chrono"
 136 | version = "0.4.41"
 137 | source = "registry+https://github.com/rust-lang/crates.io-index"
 138 | checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d"
 139 | dependencies = [
 140 |  "android-tzdata",
 141 |  "iana-time-zone",
 142 |  "js-sys",
 143 |  "num-traits",
 144 |  "wasm-bindgen",
 145 |  "windows-link",
 146 | ]
 147 | 
 148 | [[package]]
 149 | name = "clap"
 150 | version = "4.5.41"
 151 | source = "registry+https://github.com/rust-lang/crates.io-index"
 152 | checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9"
 153 | dependencies = [
 154 |  "clap_builder",
 155 |  "clap_derive",
 156 | ]
 157 | 
 158 | [[package]]
 159 | name = "clap_builder"
 160 | version = "4.5.41"
 161 | source = "registry+https://github.com/rust-lang/crates.io-index"
 162 | checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d"
 163 | dependencies = [
 164 |  "anstream",
 165 |  "anstyle",
 166 |  "clap_lex",
 167 |  "strsim",
 168 | ]
 169 | 
 170 | [[package]]
 171 | name = "clap_derive"
 172 | version = "4.5.41"
 173 | source = "registry+https://github.com/rust-lang/crates.io-index"
 174 | checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491"
 175 | dependencies = [
 176 |  "heck",
 177 |  "proc-macro2",
 178 |  "quote",
 179 |  "syn",
 180 | ]
 181 | 
 182 | [[package]]
 183 | name = "clap_lex"
 184 | version = "0.7.5"
 185 | source = "registry+https://github.com/rust-lang/crates.io-index"
 186 | checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
 187 | 
 188 | [[package]]
 189 | name = "colorchoice"
 190 | version = "1.0.4"
 191 | source = "registry+https://github.com/rust-lang/crates.io-index"
 192 | checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
 193 | 
 194 | [[package]]
 195 | name = "console"
 196 | version = "0.16.0"
 197 | source = "registry+https://github.com/rust-lang/crates.io-index"
 198 | checksum = "2e09ced7ebbccb63b4c65413d821f2e00ce54c5ca4514ddc6b3c892fdbcbc69d"
 199 | dependencies = [
 200 |  "encode_unicode",
 201 |  "libc",
 202 |  "once_cell",
 203 |  "unicode-width 0.2.1",
 204 |  "windows-sys 0.60.2",
 205 | ]
 206 | 
 207 | [[package]]
 208 | name = "core-foundation-sys"
 209 | version = "0.8.7"
 210 | source = "registry+https://github.com/rust-lang/crates.io-index"
 211 | checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 212 | 
 213 | [[package]]
 214 | name = "crossbeam-deque"
 215 | version = "0.8.6"
 216 | source = "registry+https://github.com/rust-lang/crates.io-index"
 217 | checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
 218 | dependencies = [
 219 |  "crossbeam-epoch",
 220 |  "crossbeam-utils",
 221 | ]
 222 | 
 223 | [[package]]
 224 | name = "crossbeam-epoch"
 225 | version = "0.9.18"
 226 | source = "registry+https://github.com/rust-lang/crates.io-index"
 227 | checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
 228 | dependencies = [
 229 |  "crossbeam-utils",
 230 | ]
 231 | 
 232 | [[package]]
 233 | name = "crossbeam-utils"
 234 | version = "0.8.21"
 235 | source = "registry+https://github.com/rust-lang/crates.io-index"
 236 | checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
 237 | 
 238 | [[package]]
 239 | name = "csv"
 240 | version = "1.3.1"
 241 | source = "registry+https://github.com/rust-lang/crates.io-index"
 242 | checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf"
 243 | dependencies = [
 244 |  "csv-core",
 245 |  "itoa",
 246 |  "ryu",
 247 |  "serde",
 248 | ]
 249 | 
 250 | [[package]]
 251 | name = "csv-core"
 252 | version = "0.1.12"
 253 | source = "registry+https://github.com/rust-lang/crates.io-index"
 254 | checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d"
 255 | dependencies = [
 256 |  "memchr",
 257 | ]
 258 | 
 259 | [[package]]
 260 | name = "dashmap"
 261 | version = "6.1.0"
 262 | source = "registry+https://github.com/rust-lang/crates.io-index"
 263 | checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
 264 | dependencies = [
 265 |  "cfg-if",
 266 |  "crossbeam-utils",
 267 |  "hashbrown",
 268 |  "lock_api",
 269 |  "once_cell",
 270 |  "parking_lot_core",
 271 |  "rayon",
 272 | ]
 273 | 
 274 | [[package]]
 275 | name = "deduplicator"
 276 | version = "0.3.2"
 277 | dependencies = [
 278 |  "anyhow",
 279 |  "bytesize",
 280 |  "chrono",
 281 |  "clap",
 282 |  "dashmap",
 283 |  "globwalk",
 284 |  "gxhash",
 285 |  "indicatif",
 286 |  "memmap2",
 287 |  "pathdiff",
 288 |  "prettytable-rs",
 289 |  "rand",
 290 |  "rayon",
 291 |  "tempfile",
 292 |  "threadpool",
 293 |  "unicode-segmentation",
 294 | ]
 295 | 
 296 | [[package]]
 297 | name = "dirs-next"
 298 | version = "2.0.0"
 299 | source = "registry+https://github.com/rust-lang/crates.io-index"
 300 | checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
 301 | dependencies = [
 302 |  "cfg-if",
 303 |  "dirs-sys-next",
 304 | ]
 305 | 
 306 | [[package]]
 307 | name = "dirs-sys-next"
 308 | version = "0.1.2"
 309 | source = "registry+https://github.com/rust-lang/crates.io-index"
 310 | checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
 311 | dependencies = [
 312 |  "libc",
 313 |  "redox_users",
 314 |  "winapi",
 315 | ]
 316 | 
 317 | [[package]]
 318 | name = "either"
 319 | version = "1.15.0"
 320 | source = "registry+https://github.com/rust-lang/crates.io-index"
 321 | checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 322 | 
 323 | [[package]]
 324 | name = "encode_unicode"
 325 | version = "1.0.0"
 326 | source = "registry+https://github.com/rust-lang/crates.io-index"
 327 | checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
 328 | 
 329 | [[package]]
 330 | name = "errno"
 331 | version = "0.3.13"
 332 | source = "registry+https://github.com/rust-lang/crates.io-index"
 333 | checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
 334 | dependencies = [
 335 |  "libc",
 336 |  "windows-sys 0.60.2",
 337 | ]
 338 | 
 339 | [[package]]
 340 | name = "fastrand"
 341 | version = "2.3.0"
 342 | source = "registry+https://github.com/rust-lang/crates.io-index"
 343 | checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 344 | 
 345 | [[package]]
 346 | name = "getrandom"
 347 | version = "0.2.16"
 348 | source = "registry+https://github.com/rust-lang/crates.io-index"
 349 | checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
 350 | dependencies = [
 351 |  "cfg-if",
 352 |  "libc",
 353 |  "wasi 0.11.1+wasi-snapshot-preview1",
 354 | ]
 355 | 
 356 | [[package]]
 357 | name = "getrandom"
 358 | version = "0.3.3"
 359 | source = "registry+https://github.com/rust-lang/crates.io-index"
 360 | checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
 361 | dependencies = [
 362 |  "cfg-if",
 363 |  "libc",
 364 |  "r-efi",
 365 |  "wasi 0.14.2+wasi-0.2.4",
 366 | ]
 367 | 
 368 | [[package]]
 369 | name = "globset"
 370 | version = "0.4.16"
 371 | source = "registry+https://github.com/rust-lang/crates.io-index"
 372 | checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5"
 373 | dependencies = [
 374 |  "aho-corasick",
 375 |  "bstr",
 376 |  "log",
 377 |  "regex-automata",
 378 |  "regex-syntax",
 379 | ]
 380 | 
 381 | [[package]]
 382 | name = "globwalk"
 383 | version = "0.9.1"
 384 | source = "registry+https://github.com/rust-lang/crates.io-index"
 385 | checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757"
 386 | dependencies = [
 387 |  "bitflags",
 388 |  "ignore",
 389 |  "walkdir",
 390 | ]
 391 | 
 392 | [[package]]
 393 | name = "gxhash"
 394 | version = "3.5.0"
 395 | source = "registry+https://github.com/rust-lang/crates.io-index"
 396 | checksum = "f3ce1bab7aa741d4e7042b2aae415b78741f267a98a7271ea226cd5ba6c43d7d"
 397 | dependencies = [
 398 |  "rustversion",
 399 | ]
 400 | 
 401 | [[package]]
 402 | name = "hashbrown"
 403 | version = "0.14.5"
 404 | source = "registry+https://github.com/rust-lang/crates.io-index"
 405 | checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
 406 | 
 407 | [[package]]
 408 | name = "heck"
 409 | version = "0.5.0"
 410 | source = "registry+https://github.com/rust-lang/crates.io-index"
 411 | checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 412 | 
 413 | [[package]]
 414 | name = "hermit-abi"
 415 | version = "0.5.2"
 416 | source = "registry+https://github.com/rust-lang/crates.io-index"
 417 | checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
 418 | 
 419 | [[package]]
 420 | name = "iana-time-zone"
 421 | version = "0.1.63"
 422 | source = "registry+https://github.com/rust-lang/crates.io-index"
 423 | checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8"
 424 | dependencies = [
 425 |  "android_system_properties",
 426 |  "core-foundation-sys",
 427 |  "iana-time-zone-haiku",
 428 |  "js-sys",
 429 |  "log",
 430 |  "wasm-bindgen",
 431 |  "windows-core",
 432 | ]
 433 | 
 434 | [[package]]
 435 | name = "iana-time-zone-haiku"
 436 | version = "0.1.2"
 437 | source = "registry+https://github.com/rust-lang/crates.io-index"
 438 | checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
 439 | dependencies = [
 440 |  "cc",
 441 | ]
 442 | 
 443 | [[package]]
 444 | name = "ignore"
 445 | version = "0.4.23"
 446 | source = "registry+https://github.com/rust-lang/crates.io-index"
 447 | checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b"
 448 | dependencies = [
 449 |  "crossbeam-deque",
 450 |  "globset",
 451 |  "log",
 452 |  "memchr",
 453 |  "regex-automata",
 454 |  "same-file",
 455 |  "walkdir",
 456 |  "winapi-util",
 457 | ]
 458 | 
 459 | [[package]]
 460 | name = "indicatif"
 461 | version = "0.18.0"
 462 | source = "registry+https://github.com/rust-lang/crates.io-index"
 463 | checksum = "70a646d946d06bedbbc4cac4c218acf4bbf2d87757a784857025f4d447e4e1cd"
 464 | dependencies = [
 465 |  "console",
 466 |  "portable-atomic",
 467 |  "rayon",
 468 |  "unicode-width 0.2.1",
 469 |  "unit-prefix",
 470 |  "web-time",
 471 | ]
 472 | 
 473 | [[package]]
 474 | name = "is-terminal"
 475 | version = "0.4.16"
 476 | source = "registry+https://github.com/rust-lang/crates.io-index"
 477 | checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9"
 478 | dependencies = [
 479 |  "hermit-abi",
 480 |  "libc",
 481 |  "windows-sys 0.59.0",
 482 | ]
 483 | 
 484 | [[package]]
 485 | name = "is_terminal_polyfill"
 486 | version = "1.70.1"
 487 | source = "registry+https://github.com/rust-lang/crates.io-index"
 488 | checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
 489 | 
 490 | [[package]]
 491 | name = "itoa"
 492 | version = "1.0.15"
 493 | source = "registry+https://github.com/rust-lang/crates.io-index"
 494 | checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
 495 | 
 496 | [[package]]
 497 | name = "js-sys"
 498 | version = "0.3.77"
 499 | source = "registry+https://github.com/rust-lang/crates.io-index"
 500 | checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
 501 | dependencies = [
 502 |  "once_cell",
 503 |  "wasm-bindgen",
 504 | ]
 505 | 
 506 | [[package]]
 507 | name = "lazy_static"
 508 | version = "1.5.0"
 509 | source = "registry+https://github.com/rust-lang/crates.io-index"
 510 | checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 511 | 
 512 | [[package]]
 513 | name = "libc"
 514 | version = "0.2.174"
 515 | source = "registry+https://github.com/rust-lang/crates.io-index"
 516 | checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
 517 | 
 518 | [[package]]
 519 | name = "libredox"
 520 | version = "0.1.4"
 521 | source = "registry+https://github.com/rust-lang/crates.io-index"
 522 | checksum = "1580801010e535496706ba011c15f8532df6b42297d2e471fec38ceadd8c0638"
 523 | dependencies = [
 524 |  "bitflags",
 525 |  "libc",
 526 | ]
 527 | 
 528 | [[package]]
 529 | name = "linux-raw-sys"
 530 | version = "0.9.4"
 531 | source = "registry+https://github.com/rust-lang/crates.io-index"
 532 | checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
 533 | 
 534 | [[package]]
 535 | name = "lock_api"
 536 | version = "0.4.13"
 537 | source = "registry+https://github.com/rust-lang/crates.io-index"
 538 | checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765"
 539 | dependencies = [
 540 |  "autocfg",
 541 |  "scopeguard",
 542 | ]
 543 | 
 544 | [[package]]
 545 | name = "log"
 546 | version = "0.4.27"
 547 | source = "registry+https://github.com/rust-lang/crates.io-index"
 548 | checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
 549 | 
 550 | [[package]]
 551 | name = "memchr"
 552 | version = "2.7.5"
 553 | source = "registry+https://github.com/rust-lang/crates.io-index"
 554 | checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
 555 | 
 556 | [[package]]
 557 | name = "memmap2"
 558 | version = "0.9.7"
 559 | source = "registry+https://github.com/rust-lang/crates.io-index"
 560 | checksum = "483758ad303d734cec05e5c12b41d7e93e6a6390c5e9dae6bdeb7c1259012d28"
 561 | dependencies = [
 562 |  "libc",
 563 | ]
 564 | 
 565 | [[package]]
 566 | name = "num-traits"
 567 | version = "0.2.19"
 568 | source = "registry+https://github.com/rust-lang/crates.io-index"
 569 | checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
 570 | dependencies = [
 571 |  "autocfg",
 572 | ]
 573 | 
 574 | [[package]]
 575 | name = "num_cpus"
 576 | version = "1.17.0"
 577 | source = "registry+https://github.com/rust-lang/crates.io-index"
 578 | checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
 579 | dependencies = [
 580 |  "hermit-abi",
 581 |  "libc",
 582 | ]
 583 | 
 584 | [[package]]
 585 | name = "once_cell"
 586 | version = "1.21.3"
 587 | source = "registry+https://github.com/rust-lang/crates.io-index"
 588 | checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
 589 | 
 590 | [[package]]
 591 | name = "once_cell_polyfill"
 592 | version = "1.70.1"
 593 | source = "registry+https://github.com/rust-lang/crates.io-index"
 594 | checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
 595 | 
 596 | [[package]]
 597 | name = "parking_lot_core"
 598 | version = "0.9.11"
 599 | source = "registry+https://github.com/rust-lang/crates.io-index"
 600 | checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5"
 601 | dependencies = [
 602 |  "cfg-if",
 603 |  "libc",
 604 |  "redox_syscall",
 605 |  "smallvec",
 606 |  "windows-targets 0.52.6",
 607 | ]
 608 | 
 609 | [[package]]
 610 | name = "pathdiff"
 611 | version = "0.2.3"
 612 | source = "registry+https://github.com/rust-lang/crates.io-index"
 613 | checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3"
 614 | 
 615 | [[package]]
 616 | name = "portable-atomic"
 617 | version = "1.11.1"
 618 | source = "registry+https://github.com/rust-lang/crates.io-index"
 619 | checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
 620 | 
 621 | [[package]]
 622 | name = "ppv-lite86"
 623 | version = "0.2.21"
 624 | source = "registry+https://github.com/rust-lang/crates.io-index"
 625 | checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
 626 | dependencies = [
 627 |  "zerocopy",
 628 | ]
 629 | 
 630 | [[package]]
 631 | name = "prettytable-rs"
 632 | version = "0.10.0"
 633 | source = "registry+https://github.com/rust-lang/crates.io-index"
 634 | checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a"
 635 | dependencies = [
 636 |  "csv",
 637 |  "encode_unicode",
 638 |  "is-terminal",
 639 |  "lazy_static",
 640 |  "term",
 641 |  "unicode-width 0.1.14",
 642 | ]
 643 | 
 644 | [[package]]
 645 | name = "proc-macro2"
 646 | version = "1.0.95"
 647 | source = "registry+https://github.com/rust-lang/crates.io-index"
 648 | checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
 649 | dependencies = [
 650 |  "unicode-ident",
 651 | ]
 652 | 
 653 | [[package]]
 654 | name = "quote"
 655 | version = "1.0.40"
 656 | source = "registry+https://github.com/rust-lang/crates.io-index"
 657 | checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
 658 | dependencies = [
 659 |  "proc-macro2",
 660 | ]
 661 | 
 662 | [[package]]
 663 | name = "r-efi"
 664 | version = "5.3.0"
 665 | source = "registry+https://github.com/rust-lang/crates.io-index"
 666 | checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
 667 | 
 668 | [[package]]
 669 | name = "rand"
 670 | version = "0.9.1"
 671 | source = "registry+https://github.com/rust-lang/crates.io-index"
 672 | checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
 673 | dependencies = [
 674 |  "rand_chacha",
 675 |  "rand_core",
 676 | ]
 677 | 
 678 | [[package]]
 679 | name = "rand_chacha"
 680 | version = "0.9.0"
 681 | source = "registry+https://github.com/rust-lang/crates.io-index"
 682 | checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
 683 | dependencies = [
 684 |  "ppv-lite86",
 685 |  "rand_core",
 686 | ]
 687 | 
 688 | [[package]]
 689 | name = "rand_core"
 690 | version = "0.9.3"
 691 | source = "registry+https://github.com/rust-lang/crates.io-index"
 692 | checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
 693 | dependencies = [
 694 |  "getrandom 0.3.3",
 695 | ]
 696 | 
 697 | [[package]]
 698 | name = "rayon"
 699 | version = "1.10.0"
 700 | source = "registry+https://github.com/rust-lang/crates.io-index"
 701 | checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
 702 | dependencies = [
 703 |  "either",
 704 |  "rayon-core",
 705 | ]
 706 | 
 707 | [[package]]
 708 | name = "rayon-core"
 709 | version = "1.12.1"
 710 | source = "registry+https://github.com/rust-lang/crates.io-index"
 711 | checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
 712 | dependencies = [
 713 |  "crossbeam-deque",
 714 |  "crossbeam-utils",
 715 | ]
 716 | 
 717 | [[package]]
 718 | name = "redox_syscall"
 719 | version = "0.5.13"
 720 | source = "registry+https://github.com/rust-lang/crates.io-index"
 721 | checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6"
 722 | dependencies = [
 723 |  "bitflags",
 724 | ]
 725 | 
 726 | [[package]]
 727 | name = "redox_users"
 728 | version = "0.4.6"
 729 | source = "registry+https://github.com/rust-lang/crates.io-index"
 730 | checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
 731 | dependencies = [
 732 |  "getrandom 0.2.16",
 733 |  "libredox",
 734 |  "thiserror",
 735 | ]
 736 | 
 737 | [[package]]
 738 | name = "regex-automata"
 739 | version = "0.4.9"
 740 | source = "registry+https://github.com/rust-lang/crates.io-index"
 741 | checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
 742 | dependencies = [
 743 |  "aho-corasick",
 744 |  "memchr",
 745 |  "regex-syntax",
 746 | ]
 747 | 
 748 | [[package]]
 749 | name = "regex-syntax"
 750 | version = "0.8.5"
 751 | source = "registry+https://github.com/rust-lang/crates.io-index"
 752 | checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 753 | 
 754 | [[package]]
 755 | name = "rustix"
 756 | version = "1.0.8"
 757 | source = "registry+https://github.com/rust-lang/crates.io-index"
 758 | checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8"
 759 | dependencies = [
 760 |  "bitflags",
 761 |  "errno",
 762 |  "libc",
 763 |  "linux-raw-sys",
 764 |  "windows-sys 0.60.2",
 765 | ]
 766 | 
 767 | [[package]]
 768 | name = "rustversion"
 769 | version = "1.0.21"
 770 | source = "registry+https://github.com/rust-lang/crates.io-index"
 771 | checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
 772 | 
 773 | [[package]]
 774 | name = "ryu"
 775 | version = "1.0.20"
 776 | source = "registry+https://github.com/rust-lang/crates.io-index"
 777 | checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
 778 | 
 779 | [[package]]
 780 | name = "same-file"
 781 | version = "1.0.6"
 782 | source = "registry+https://github.com/rust-lang/crates.io-index"
 783 | checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
 784 | dependencies = [
 785 |  "winapi-util",
 786 | ]
 787 | 
 788 | [[package]]
 789 | name = "scopeguard"
 790 | version = "1.2.0"
 791 | source = "registry+https://github.com/rust-lang/crates.io-index"
 792 | checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 793 | 
 794 | [[package]]
 795 | name = "serde"
 796 | version = "1.0.219"
 797 | source = "registry+https://github.com/rust-lang/crates.io-index"
 798 | checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
 799 | dependencies = [
 800 |  "serde_derive",
 801 | ]
 802 | 
 803 | [[package]]
 804 | name = "serde_derive"
 805 | version = "1.0.219"
 806 | source = "registry+https://github.com/rust-lang/crates.io-index"
 807 | checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
 808 | dependencies = [
 809 |  "proc-macro2",
 810 |  "quote",
 811 |  "syn",
 812 | ]
 813 | 
 814 | [[package]]
 815 | name = "shlex"
 816 | version = "1.3.0"
 817 | source = "registry+https://github.com/rust-lang/crates.io-index"
 818 | checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 819 | 
 820 | [[package]]
 821 | name = "smallvec"
 822 | version = "1.15.1"
 823 | source = "registry+https://github.com/rust-lang/crates.io-index"
 824 | checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
 825 | 
 826 | [[package]]
 827 | name = "strsim"
 828 | version = "0.11.1"
 829 | source = "registry+https://github.com/rust-lang/crates.io-index"
 830 | checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 831 | 
 832 | [[package]]
 833 | name = "syn"
 834 | version = "2.0.104"
 835 | source = "registry+https://github.com/rust-lang/crates.io-index"
 836 | checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
 837 | dependencies = [
 838 |  "proc-macro2",
 839 |  "quote",
 840 |  "unicode-ident",
 841 | ]
 842 | 
 843 | [[package]]
 844 | name = "tempfile"
 845 | version = "3.20.0"
 846 | source = "registry+https://github.com/rust-lang/crates.io-index"
 847 | checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1"
 848 | dependencies = [
 849 |  "fastrand",
 850 |  "getrandom 0.3.3",
 851 |  "once_cell",
 852 |  "rustix",
 853 |  "windows-sys 0.59.0",
 854 | ]
 855 | 
 856 | [[package]]
 857 | name = "term"
 858 | version = "0.7.0"
 859 | source = "registry+https://github.com/rust-lang/crates.io-index"
 860 | checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
 861 | dependencies = [
 862 |  "dirs-next",
 863 |  "rustversion",
 864 |  "winapi",
 865 | ]
 866 | 
 867 | [[package]]
 868 | name = "thiserror"
 869 | version = "1.0.69"
 870 | source = "registry+https://github.com/rust-lang/crates.io-index"
 871 | checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
 872 | dependencies = [
 873 |  "thiserror-impl",
 874 | ]
 875 | 
 876 | [[package]]
 877 | name = "thiserror-impl"
 878 | version = "1.0.69"
 879 | source = "registry+https://github.com/rust-lang/crates.io-index"
 880 | checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 881 | dependencies = [
 882 |  "proc-macro2",
 883 |  "quote",
 884 |  "syn",
 885 | ]
 886 | 
 887 | [[package]]
 888 | name = "threadpool"
 889 | version = "1.8.1"
 890 | source = "registry+https://github.com/rust-lang/crates.io-index"
 891 | checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
 892 | dependencies = [
 893 |  "num_cpus",
 894 | ]
 895 | 
 896 | [[package]]
 897 | name = "unicode-ident"
 898 | version = "1.0.18"
 899 | source = "registry+https://github.com/rust-lang/crates.io-index"
 900 | checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
 901 | 
 902 | [[package]]
 903 | name = "unicode-segmentation"
 904 | version = "1.12.0"
 905 | source = "registry+https://github.com/rust-lang/crates.io-index"
 906 | checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
 907 | 
 908 | [[package]]
 909 | name = "unicode-width"
 910 | version = "0.1.14"
 911 | source = "registry+https://github.com/rust-lang/crates.io-index"
 912 | checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
 913 | 
 914 | [[package]]
 915 | name = "unicode-width"
 916 | version = "0.2.1"
 917 | source = "registry+https://github.com/rust-lang/crates.io-index"
 918 | checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c"
 919 | 
 920 | [[package]]
 921 | name = "unit-prefix"
 922 | version = "0.5.1"
 923 | source = "registry+https://github.com/rust-lang/crates.io-index"
 924 | checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817"
 925 | 
 926 | [[package]]
 927 | name = "utf8parse"
 928 | version = "0.2.2"
 929 | source = "registry+https://github.com/rust-lang/crates.io-index"
 930 | checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 931 | 
 932 | [[package]]
 933 | name = "walkdir"
 934 | version = "2.5.0"
 935 | source = "registry+https://github.com/rust-lang/crates.io-index"
 936 | checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
 937 | dependencies = [
 938 |  "same-file",
 939 |  "winapi-util",
 940 | ]
 941 | 
 942 | [[package]]
 943 | name = "wasi"
 944 | version = "0.11.1+wasi-snapshot-preview1"
 945 | source = "registry+https://github.com/rust-lang/crates.io-index"
 946 | checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
 947 | 
 948 | [[package]]
 949 | name = "wasi"
 950 | version = "0.14.2+wasi-0.2.4"
 951 | source = "registry+https://github.com/rust-lang/crates.io-index"
 952 | checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
 953 | dependencies = [
 954 |  "wit-bindgen-rt",
 955 | ]
 956 | 
 957 | [[package]]
 958 | name = "wasm-bindgen"
 959 | version = "0.2.100"
 960 | source = "registry+https://github.com/rust-lang/crates.io-index"
 961 | checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
 962 | dependencies = [
 963 |  "cfg-if",
 964 |  "once_cell",
 965 |  "rustversion",
 966 |  "wasm-bindgen-macro",
 967 | ]
 968 | 
 969 | [[package]]
 970 | name = "wasm-bindgen-backend"
 971 | version = "0.2.100"
 972 | source = "registry+https://github.com/rust-lang/crates.io-index"
 973 | checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
 974 | dependencies = [
 975 |  "bumpalo",
 976 |  "log",
 977 |  "proc-macro2",
 978 |  "quote",
 979 |  "syn",
 980 |  "wasm-bindgen-shared",
 981 | ]
 982 | 
 983 | [[package]]
 984 | name = "wasm-bindgen-macro"
 985 | version = "0.2.100"
 986 | source = "registry+https://github.com/rust-lang/crates.io-index"
 987 | checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
 988 | dependencies = [
 989 |  "quote",
 990 |  "wasm-bindgen-macro-support",
 991 | ]
 992 | 
 993 | [[package]]
 994 | name = "wasm-bindgen-macro-support"
 995 | version = "0.2.100"
 996 | source = "registry+https://github.com/rust-lang/crates.io-index"
 997 | checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
 998 | dependencies = [
 999 |  "proc-macro2",
1000 |  "quote",
1001 |  "syn",
1002 |  "wasm-bindgen-backend",
1003 |  "wasm-bindgen-shared",
1004 | ]
1005 | 
1006 | [[package]]
1007 | name = "wasm-bindgen-shared"
1008 | version = "0.2.100"
1009 | source = "registry+https://github.com/rust-lang/crates.io-index"
1010 | checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
1011 | dependencies = [
1012 |  "unicode-ident",
1013 | ]
1014 | 
1015 | [[package]]
1016 | name = "web-time"
1017 | version = "1.1.0"
1018 | source = "registry+https://github.com/rust-lang/crates.io-index"
1019 | checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
1020 | dependencies = [
1021 |  "js-sys",
1022 |  "wasm-bindgen",
1023 | ]
1024 | 
1025 | [[package]]
1026 | name = "winapi"
1027 | version = "0.3.9"
1028 | source = "registry+https://github.com/rust-lang/crates.io-index"
1029 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
1030 | dependencies = [
1031 |  "winapi-i686-pc-windows-gnu",
1032 |  "winapi-x86_64-pc-windows-gnu",
1033 | ]
1034 | 
1035 | [[package]]
1036 | name = "winapi-i686-pc-windows-gnu"
1037 | version = "0.4.0"
1038 | source = "registry+https://github.com/rust-lang/crates.io-index"
1039 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
1040 | 
1041 | [[package]]
1042 | name = "winapi-util"
1043 | version = "0.1.9"
1044 | source = "registry+https://github.com/rust-lang/crates.io-index"
1045 | checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
1046 | dependencies = [
1047 |  "windows-sys 0.59.0",
1048 | ]
1049 | 
1050 | [[package]]
1051 | name = "winapi-x86_64-pc-windows-gnu"
1052 | version = "0.4.0"
1053 | source = "registry+https://github.com/rust-lang/crates.io-index"
1054 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
1055 | 
1056 | [[package]]
1057 | name = "windows-core"
1058 | version = "0.61.2"
1059 | source = "registry+https://github.com/rust-lang/crates.io-index"
1060 | checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
1061 | dependencies = [
1062 |  "windows-implement",
1063 |  "windows-interface",
1064 |  "windows-link",
1065 |  "windows-result",
1066 |  "windows-strings",
1067 | ]
1068 | 
1069 | [[package]]
1070 | name = "windows-implement"
1071 | version = "0.60.0"
1072 | source = "registry+https://github.com/rust-lang/crates.io-index"
1073 | checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"
1074 | dependencies = [
1075 |  "proc-macro2",
1076 |  "quote",
1077 |  "syn",
1078 | ]
1079 | 
1080 | [[package]]
1081 | name = "windows-interface"
1082 | version = "0.59.1"
1083 | source = "registry+https://github.com/rust-lang/crates.io-index"
1084 | checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"
1085 | dependencies = [
1086 |  "proc-macro2",
1087 |  "quote",
1088 |  "syn",
1089 | ]
1090 | 
1091 | [[package]]
1092 | name = "windows-link"
1093 | version = "0.1.3"
1094 | source = "registry+https://github.com/rust-lang/crates.io-index"
1095 | checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
1096 | 
1097 | [[package]]
1098 | name = "windows-result"
1099 | version = "0.3.4"
1100 | source = "registry+https://github.com/rust-lang/crates.io-index"
1101 | checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
1102 | dependencies = [
1103 |  "windows-link",
1104 | ]
1105 | 
1106 | [[package]]
1107 | name = "windows-strings"
1108 | version = "0.4.2"
1109 | source = "registry+https://github.com/rust-lang/crates.io-index"
1110 | checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
1111 | dependencies = [
1112 |  "windows-link",
1113 | ]
1114 | 
1115 | [[package]]
1116 | name = "windows-sys"
1117 | version = "0.59.0"
1118 | source = "registry+https://github.com/rust-lang/crates.io-index"
1119 | checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
1120 | dependencies = [
1121 |  "windows-targets 0.52.6",
1122 | ]
1123 | 
1124 | [[package]]
1125 | name = "windows-sys"
1126 | version = "0.60.2"
1127 | source = "registry+https://github.com/rust-lang/crates.io-index"
1128 | checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
1129 | dependencies = [
1130 |  "windows-targets 0.53.2",
1131 | ]
1132 | 
1133 | [[package]]
1134 | name = "windows-targets"
1135 | version = "0.52.6"
1136 | source = "registry+https://github.com/rust-lang/crates.io-index"
1137 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
1138 | dependencies = [
1139 |  "windows_aarch64_gnullvm 0.52.6",
1140 |  "windows_aarch64_msvc 0.52.6",
1141 |  "windows_i686_gnu 0.52.6",
1142 |  "windows_i686_gnullvm 0.52.6",
1143 |  "windows_i686_msvc 0.52.6",
1144 |  "windows_x86_64_gnu 0.52.6",
1145 |  "windows_x86_64_gnullvm 0.52.6",
1146 |  "windows_x86_64_msvc 0.52.6",
1147 | ]
1148 | 
1149 | [[package]]
1150 | name = "windows-targets"
1151 | version = "0.53.2"
1152 | source = "registry+https://github.com/rust-lang/crates.io-index"
1153 | checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef"
1154 | dependencies = [
1155 |  "windows_aarch64_gnullvm 0.53.0",
1156 |  "windows_aarch64_msvc 0.53.0",
1157 |  "windows_i686_gnu 0.53.0",
1158 |  "windows_i686_gnullvm 0.53.0",
1159 |  "windows_i686_msvc 0.53.0",
1160 |  "windows_x86_64_gnu 0.53.0",
1161 |  "windows_x86_64_gnullvm 0.53.0",
1162 |  "windows_x86_64_msvc 0.53.0",
1163 | ]
1164 | 
1165 | [[package]]
1166 | name = "windows_aarch64_gnullvm"
1167 | version = "0.52.6"
1168 | source = "registry+https://github.com/rust-lang/crates.io-index"
1169 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
1170 | 
1171 | [[package]]
1172 | name = "windows_aarch64_gnullvm"
1173 | version = "0.53.0"
1174 | source = "registry+https://github.com/rust-lang/crates.io-index"
1175 | checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
1176 | 
1177 | [[package]]
1178 | name = "windows_aarch64_msvc"
1179 | version = "0.52.6"
1180 | source = "registry+https://github.com/rust-lang/crates.io-index"
1181 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
1182 | 
1183 | [[package]]
1184 | name = "windows_aarch64_msvc"
1185 | version = "0.53.0"
1186 | source = "registry+https://github.com/rust-lang/crates.io-index"
1187 | checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
1188 | 
1189 | [[package]]
1190 | name = "windows_i686_gnu"
1191 | version = "0.52.6"
1192 | source = "registry+https://github.com/rust-lang/crates.io-index"
1193 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
1194 | 
1195 | [[package]]
1196 | name = "windows_i686_gnu"
1197 | version = "0.53.0"
1198 | source = "registry+https://github.com/rust-lang/crates.io-index"
1199 | checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3"
1200 | 
1201 | [[package]]
1202 | name = "windows_i686_gnullvm"
1203 | version = "0.52.6"
1204 | source = "registry+https://github.com/rust-lang/crates.io-index"
1205 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
1206 | 
1207 | [[package]]
1208 | name = "windows_i686_gnullvm"
1209 | version = "0.53.0"
1210 | source = "registry+https://github.com/rust-lang/crates.io-index"
1211 | checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
1212 | 
1213 | [[package]]
1214 | name = "windows_i686_msvc"
1215 | version = "0.52.6"
1216 | source = "registry+https://github.com/rust-lang/crates.io-index"
1217 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
1218 | 
1219 | [[package]]
1220 | name = "windows_i686_msvc"
1221 | version = "0.53.0"
1222 | source = "registry+https://github.com/rust-lang/crates.io-index"
1223 | checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
1224 | 
1225 | [[package]]
1226 | name = "windows_x86_64_gnu"
1227 | version = "0.52.6"
1228 | source = "registry+https://github.com/rust-lang/crates.io-index"
1229 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
1230 | 
1231 | [[package]]
1232 | name = "windows_x86_64_gnu"
1233 | version = "0.53.0"
1234 | source = "registry+https://github.com/rust-lang/crates.io-index"
1235 | checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
1236 | 
1237 | [[package]]
1238 | name = "windows_x86_64_gnullvm"
1239 | version = "0.52.6"
1240 | source = "registry+https://github.com/rust-lang/crates.io-index"
1241 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
1242 | 
1243 | [[package]]
1244 | name = "windows_x86_64_gnullvm"
1245 | version = "0.53.0"
1246 | source = "registry+https://github.com/rust-lang/crates.io-index"
1247 | checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
1248 | 
1249 | [[package]]
1250 | name = "windows_x86_64_msvc"
1251 | version = "0.52.6"
1252 | source = "registry+https://github.com/rust-lang/crates.io-index"
1253 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
1254 | 
1255 | [[package]]
1256 | name = "windows_x86_64_msvc"
1257 | version = "0.53.0"
1258 | source = "registry+https://github.com/rust-lang/crates.io-index"
1259 | checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
1260 | 
1261 | [[package]]
1262 | name = "wit-bindgen-rt"
1263 | version = "0.39.0"
1264 | source = "registry+https://github.com/rust-lang/crates.io-index"
1265 | checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
1266 | dependencies = [
1267 |  "bitflags",
1268 | ]
1269 | 
1270 | [[package]]
1271 | name = "zerocopy"
1272 | version = "0.8.26"
1273 | source = "registry+https://github.com/rust-lang/crates.io-index"
1274 | checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f"
1275 | dependencies = [
1276 |  "zerocopy-derive",
1277 | ]
1278 | 
1279 | [[package]]
1280 | name = "zerocopy-derive"
1281 | version = "0.8.26"
1282 | source = "registry+https://github.com/rust-lang/crates.io-index"
1283 | checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181"
1284 | dependencies = [
1285 |  "proc-macro2",
1286 |  "quote",
1287 |  "syn",
1288 | ]
1289 | 


--------------------------------------------------------------------------------