├── .gitignore ├── clippy.toml ├── tests ├── integ │ ├── data │ │ ├── json │ │ │ ├── actual │ │ │ │ └── guy.json │ │ │ └── expected │ │ │ │ └── guy.json │ │ ├── images │ │ │ ├── diff_100_DPI.png │ │ │ ├── actual │ │ │ │ ├── SaveImage_100DPI_default_size.jpg │ │ │ │ └── SaveImage_100DPI_custom_size_not_uniform_500x500.jpg │ │ │ └── expected │ │ │ │ ├── SaveImage_100DPI_default_size.jpg │ │ │ │ └── SaveImage_100DPI_custom_size_not_uniform_500x500.jpg │ │ └── display_of_status_message_in_cm_tables │ │ │ ├── actual │ │ │ ├── Features_VolumeWithAllStatusMessage.csv │ │ │ ├── Features_VolumeWithAllStatusMessage2.csv │ │ │ ├── Features_VolumeWithAllStatusMessage3.csv │ │ │ ├── Features_VolumeWithAllStatusMessage3_2.csv │ │ │ ├── VolumeWithAllStatusMessage.csv │ │ │ ├── Volume1.csv │ │ │ ├── VolumeWithAllStatusMessage3.csv │ │ │ ├── VolumeWithAllStatusMessage3_2.csv │ │ │ └── VolumeWithAllStatusMessage2.csv │ │ │ └── expected │ │ │ ├── Features_VolumeWithAllStatusMessage.csv │ │ │ ├── Features_VolumeWithAllStatusMessage2.csv │ │ │ ├── Features_VolumeWithAllStatusMessage3.csv │ │ │ ├── Features_VolumeWithAllStatusMessage3_2.csv │ │ │ ├── VolumeWithAllStatusMessage.csv │ │ │ ├── Volume1.csv │ │ │ ├── VolumeWithAllStatusMessage3.csv │ │ │ ├── VolumeWithAllStatusMessage3_2.csv │ │ │ └── VolumeWithAllStatusMessage2.csv │ ├── json.yml │ ├── jpg_compare.yml │ ├── config.yml │ └── vgrf.yml ├── pdf │ ├── actual.pdf │ └── expected.pdf ├── csv │ └── data │ │ ├── CM_quality_threshold.csv │ │ ├── no_field_sep.csv │ │ ├── easy_pore_export_annoration_table_result.csv │ │ ├── Annotations.csv │ │ ├── Annotations_diff.csv │ │ ├── defects_headers.csv │ │ ├── Components.csv │ │ ├── defects.csv │ │ ├── Multi_Apply_Rotation.csv │ │ ├── DeviationHistogram.csv │ │ ├── DeviationHistogram_diff.csv │ │ └── CumulatedHistogram.csv ├── html │ ├── test.html │ └── html_changed.html └── integ.rs ├── .github └── workflows │ ├── rust.yml │ ├── release.yml │ └── coverage.yml ├── src ├── print_args.rs ├── external.rs ├── pdf.rs ├── hash.rs ├── main.rs ├── html.rs ├── csv │ ├── value.rs │ ├── tokenizer │ │ ├── guess_format.rs │ │ └── mod.rs │ └── preprocessing.rs ├── json.rs ├── directory.rs ├── properties.rs ├── image.rs └── lib.rs ├── LICENSE ├── Cargo.toml └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | .idea 3 | Cargo.lock 4 | -------------------------------------------------------------------------------- /clippy.toml: -------------------------------------------------------------------------------- 1 | allow-unwrap-in-tests = true 2 | allow-expect-in-tests = true -------------------------------------------------------------------------------- /tests/integ/data/json/actual/guy.json: -------------------------------------------------------------------------------- 1 | {"name":"Takumi", "age":18, "car": "Panda Trueno"} -------------------------------------------------------------------------------- /tests/pdf/actual.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VolumeGraphics/havocompare/HEAD/tests/pdf/actual.pdf -------------------------------------------------------------------------------- /tests/integ/data/json/expected/guy.json: -------------------------------------------------------------------------------- 1 | {"name":"Keisuke", "age":21, "car": "RX7", "brothers": ["Ryosuke"]} -------------------------------------------------------------------------------- /tests/pdf/expected.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VolumeGraphics/havocompare/HEAD/tests/pdf/expected.pdf -------------------------------------------------------------------------------- /tests/csv/data/CM_quality_threshold.csv: -------------------------------------------------------------------------------- 1 | Name,Status 2 | 90%,OK 3 | 95%,OK 4 | 99%,OK 5 | no quality threshold,OK 6 | -------------------------------------------------------------------------------- /tests/integ/json.yml: -------------------------------------------------------------------------------- 1 | rules: 2 | - name: "Compare JSON files" 3 | pattern_include: 4 | - "**/*.json" 5 | Json: {} -------------------------------------------------------------------------------- /tests/integ/data/images/diff_100_DPI.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VolumeGraphics/havocompare/HEAD/tests/integ/data/images/diff_100_DPI.png -------------------------------------------------------------------------------- /tests/integ/jpg_compare.yml: -------------------------------------------------------------------------------- 1 | rules: 2 | - name: "JPG comparison" 3 | pattern_include: 4 | - "**/*.jpg" 5 | Image: 6 | RGBA: Hybrid 7 | threshold: 0.9 8 | -------------------------------------------------------------------------------- /tests/integ/data/images/actual/SaveImage_100DPI_default_size.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VolumeGraphics/havocompare/HEAD/tests/integ/data/images/actual/SaveImage_100DPI_default_size.jpg -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/actual/Features_VolumeWithAllStatusMessage.csv: -------------------------------------------------------------------------------- 1 | Name,Type,Status 2 | Feature 1,Flatness,Out of sync 3 | Feature 1 [2],Flatness,Out of sync 4 | -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/actual/Features_VolumeWithAllStatusMessage2.csv: -------------------------------------------------------------------------------- 1 | Name,Type,Status 2 | Feature 1,Flatness,Out of sync 3 | Feature 1 [2],Flatness,Out of sync 4 | -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/actual/Features_VolumeWithAllStatusMessage3.csv: -------------------------------------------------------------------------------- 1 | Name,Type,Status 2 | Feature 1,Flatness,Out of sync 3 | Feature 1 [2],Flatness,Out of sync 4 | -------------------------------------------------------------------------------- /tests/integ/data/images/expected/SaveImage_100DPI_default_size.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VolumeGraphics/havocompare/HEAD/tests/integ/data/images/expected/SaveImage_100DPI_default_size.jpg -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/actual/Features_VolumeWithAllStatusMessage3_2.csv: -------------------------------------------------------------------------------- 1 | Name,Type,Status 2 | Feature 1,Flatness,Out of sync 3 | Feature 1 [2],Flatness,Out of sync 4 | -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/actual/VolumeWithAllStatusMessage.csv: -------------------------------------------------------------------------------- 1 | Name,Status 2 | Circle 1,OK 3 | Cylinder 1,OK 4 | Plane 1,Invalid: Automatic refit failed 5 | Plane 3,OK 6 | -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/expected/Features_VolumeWithAllStatusMessage.csv: -------------------------------------------------------------------------------- 1 | Name,Type,Status 2 | Feature 1,Flatness,Out of sync 3 | Feature 1 [2],Flatness,Out of sync 4 | -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/expected/Features_VolumeWithAllStatusMessage2.csv: -------------------------------------------------------------------------------- 1 | Name,Type,Status 2 | Feature 1,Flatness,Out of sync 3 | Feature 1 [2],Flatness,Out of sync 4 | -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/expected/Features_VolumeWithAllStatusMessage3.csv: -------------------------------------------------------------------------------- 1 | Name,Type,Status 2 | Feature 1,Flatness,Out of sync 3 | Feature 1 [2],Flatness,Out of sync 4 | -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/expected/Features_VolumeWithAllStatusMessage3_2.csv: -------------------------------------------------------------------------------- 1 | Name,Type,Status 2 | Feature 1,Flatness,Out of sync 3 | Feature 1 [2],Flatness,Out of sync 4 | -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/actual/Volume1.csv: -------------------------------------------------------------------------------- 1 | Name,Status 2 | Cylinder 1,"Isosurface not found for iteration > 0, but previous result could be taken" 3 | Cylinder 2,OK 4 | 0.0, 0.1 -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/expected/VolumeWithAllStatusMessage.csv: -------------------------------------------------------------------------------- 1 | Name,Status 2 | Circle 1,OK 3 | Cylinder 1,OK 4 | Plane 1,Invalid: Automatic refit failed 5 | Plane 3,OK 6 | -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/expected/Volume1.csv: -------------------------------------------------------------------------------- 1 | Name;Status 2 | Cylinder 1;"Isosurface not found for iteration > 0, but previous result could be taken" 3 | Cylinder 2;OK 4 | 0,0; 0,1 -------------------------------------------------------------------------------- /tests/integ/data/images/actual/SaveImage_100DPI_custom_size_not_uniform_500x500.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VolumeGraphics/havocompare/HEAD/tests/integ/data/images/actual/SaveImage_100DPI_custom_size_not_uniform_500x500.jpg -------------------------------------------------------------------------------- /tests/integ/data/images/expected/SaveImage_100DPI_custom_size_not_uniform_500x500.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VolumeGraphics/havocompare/HEAD/tests/integ/data/images/expected/SaveImage_100DPI_custom_size_not_uniform_500x500.jpg -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/actual/VolumeWithAllStatusMessage3.csv: -------------------------------------------------------------------------------- 1 | Name,Status 2 | Circle 1,OK 3 | Cylinder 1,OK 4 | Plane 1,Invalid: Automatic refit failed 5 | Plane 1 [2],Invalid: Automatic refit failed 6 | Plane 2,Invalid: Automatic refit failed 7 | Plane 3,OK 8 | -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/actual/VolumeWithAllStatusMessage3_2.csv: -------------------------------------------------------------------------------- 1 | Name,Status 2 | Circle 1,OK 3 | Cylinder 1,OK 4 | Plane 1,Invalid: Automatic refit failed 5 | Plane 1 [2],Invalid: Automatic refit failed 6 | Plane 2,Invalid: Automatic refit failed 7 | Plane 3,OK 8 | -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/expected/VolumeWithAllStatusMessage3.csv: -------------------------------------------------------------------------------- 1 | Name,Status 2 | Circle 1,OK 3 | Cylinder 1,OK 4 | Plane 1,Invalid: Automatic refit failed 5 | Plane 1 [2],Invalid: Automatic refit failed 6 | Plane 2,Invalid: Automatic refit failed 7 | Plane 3,OK 8 | -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/expected/VolumeWithAllStatusMessage3_2.csv: -------------------------------------------------------------------------------- 1 | Name,Status 2 | Circle 1,OK 3 | Cylinder 1,OK 4 | Plane 1,Invalid: Automatic refit failed 5 | Plane 1 [2],Invalid: Automatic refit failed 6 | Plane 2,Invalid: Automatic refit failed 7 | Plane 3,OK 8 | -------------------------------------------------------------------------------- /tests/csv/data/no_field_sep.csv: -------------------------------------------------------------------------------- 1 | Area [mm²] 2 | 35.23 3 | 266.22 4 | 303.87 5 | 422.84 6 | 429.09 7 | 579.11 8 | 704.57 9 | 704.60 10 | 803.87 11 | 805.02 12 | 959.84 13 | 1059.25 14 | 1106.24 15 | 1231.01 16 | 1248.21 17 | 1254.11 18 | 1322.43 19 | 1347.02 20 | 1418.68 21 | 1498.85 22 | 1731.46 23 | 1942.96 24 | 2139.71 25 | 2428.87 26 | 3450.56 27 | -------------------------------------------------------------------------------- /tests/csv/data/easy_pore_export_annoration_table_result.csv: -------------------------------------------------------------------------------- 1 | Name;Volume [mm³];Radius [mm];Diameter [mm];Surface [mm²] 2 | Defect 1;4,41600;1,06066;2,12132;19,92000 3 | Defect 2;34,41606;2,06034;4,12068;76,80000 4 | Defect 3;114,87995;3,06186;6,12373;171,12001 5 | Defect 4;271,07263;4,06264;8,12528;303,84006 6 | Defect 5;528,68768;5,06310;10,12621;474,00003 7 | 8 | -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/actual/VolumeWithAllStatusMessage2.csv: -------------------------------------------------------------------------------- 1 | Name,Status 2 | Circle 1,Invalid: Source elements are no longer present 3 | Cylinder 1,Invalid: Not synchronized with current situation (surface determination change?) 4 | Plane 1,Invalid: Not synchronized with current situation (surface determination change?) 5 | Plane 1 [2],Invalid: Not synchronized with current situation (surface determination change?) 6 | Plane 2,Invalid: Not synchronized with current situation (surface determination change?) 7 | -------------------------------------------------------------------------------- /tests/integ/data/display_of_status_message_in_cm_tables/expected/VolumeWithAllStatusMessage2.csv: -------------------------------------------------------------------------------- 1 | Name,Status 2 | Circle 1,Invalid: Source elements are no longer present 3 | Cylinder 1,Invalid: Not synchronized with current situation (surface determination change?) 4 | Plane 1,Invalid: Not synchronized with current situation (surface determination change?) 5 | Plane 1 [2],Invalid: Not synchronized with current situation (surface determination change?) 6 | Plane 2,Invalid: Not synchronized with current situation (surface determination change?) 7 | -------------------------------------------------------------------------------- /tests/csv/data/Annotations.csv: -------------------------------------------------------------------------------- 1 | Name,Pos. x [mm],Pos. y [mm],Pos. z [mm],Deviation [mm],Min. Deviation [mm],Max. Deviation [mm],Description,Image,Focused image,Captures,Tolerance status,Type 2 | Deviation 5,-22.663068771,39.907196045,-49.257804871,>= 0.600000024,n/a,n/a,,Off,Off,0,No tolerance,Rule based 3 | Deviation 4,-22.826931000,39.911468506,-20.156715393,>= 0.600000024,n/a,n/a,,Off,Off,0,No tolerance,Rule based 4 | Deviation 3,-22.591037750,39.901805878,-33.957965851,>= 0.600000024,n/a,n/a,,Off,Off,0,No tolerance,Rule based 5 | Deviation 2,-22.739957809,39.908485413,-65.457649231,>= 0.600000024,n/a,n/a,,Off,Off,0,No tolerance,Rule based 6 | Deviation 1,-22.650812149,41.070060730,-80.756362915,>= 0.600000024,n/a,n/a,,Off,Off,0,No tolerance,Rule based 7 | 8 | -------------------------------------------------------------------------------- /tests/integ/config.yml: -------------------------------------------------------------------------------- 1 | rules: 2 | - name: "VGRF-Reporting CSV comparing" 3 | pattern_include: 4 | - "**/*.csv" 5 | pattern_exclude: 6 | - "**/*_diff.csv" 7 | CSV: 8 | comparison_modes: 9 | - Absolute: 1.0 10 | - Relative: 0.1 11 | exclude_field_regex: "Excluded" 12 | 13 | - name: "HTML-Compare strict" 14 | pattern_exclude: 15 | - "**/*_changed.html" 16 | pattern_include: 17 | - "**/*.html" 18 | PlainText: 19 | threshold: 1.0 20 | ignore_lines: 21 | - "stylesheet" 22 | - "next_ignore" 23 | - "[A-Z]*[0-9]" 24 | 25 | - name: "HTML-Compare fuzzy" 26 | pattern_include: 27 | - "**/*.html" 28 | PlainText: 29 | threshold: 0.9 30 | 31 | 32 | -------------------------------------------------------------------------------- /tests/csv/data/Annotations_diff.csv: -------------------------------------------------------------------------------- 1 | Name,Position x [mm],Position y [mm],Position z [mm],Deviation [mm],Min. Deviation [mm],Max. Deviation [mm],Description,Image,Focused image,Captures,Tolerance status,Type 2 | Deviation 5,-22.663068771,39.907196045,-49.257804871,>= 0.600000024,n/a,n/a,,Off,Off,0,No tolerance,Rule based 3 | Deviation 4,-22.826931000,39.911468506,-20.156715393,>= 0.600000024,n/a,n/a,,Off,Off,0,No tolerance,Rule based 4 | Deviation 3,-22.591037750,39.901805878,-33.957965851,>= 0.600000024,n/a,n/a,,Off,Off,0,No tolerance,Rule based 5 | Deviation 2,-22.739957809,39.908485413,-65.457649231,>= 0.600000024,n/a,n/a,,Off,Off,0,No tolerance,Rule based 6 | Deviation 1,-22.650812149,41.070060730,-80.756362915,>= 0.600000024,n/a,n/a,,Off,Off,0,No tolerance,Rule based 7 | 8 | -------------------------------------------------------------------------------- /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: [ "main" ] 4 | pull_request: 5 | branches: [ "main" ] 6 | 7 | env: 8 | CARGO_TERM_COLOR: always 9 | 10 | jobs: 11 | build: 12 | strategy: 13 | fail-fast: true 14 | matrix: 15 | include: 16 | - os: ubuntu-latest 17 | - os: windows-latest 18 | - os: macos-latest 19 | 20 | name: rust-ci ${{ matrix.os }} 21 | runs-on: ${{ matrix.os }} 22 | steps: 23 | - uses: actions/checkout@v3 24 | - name: Build 25 | run: cargo build --verbose 26 | - name: Run tests 27 | run: cargo test --release --verbose 28 | - name: Run check 29 | run: cargo check --verbose 30 | - name: Run clippy 31 | run: cargo clippy --verbose 32 | -------------------------------------------------------------------------------- /tests/csv/data/defects_headers.csv: -------------------------------------------------------------------------------- 1 | Entry,Radius 2 | 1,0.00139 3 | 2,0.00711 4 | 3,0.00829 5 | 4,0.00774 6 | 5,0.00739 7 | 6,0.00882 8 | 7,0.00955 9 | 8,0.00265 10 | 9,0.00161 11 | 10,0.00275 12 | 11,0.00716 13 | 12,0.00579 14 | 13,0.00444 15 | 14,0.00938 16 | 15,0.00734 17 | 16,0.00352 18 | 17,0.00867 19 | 18,0.00105 20 | 19,0.00375 21 | 20,0.00692 22 | 21,0.00167 23 | 22,0.0055 24 | 23,0.00236 25 | 24,0.00775 26 | 25,0.00044 27 | 26,0.00441 28 | 27,0.00671 29 | 28,0.00529 30 | 29,0.00622 31 | 30,0.00153 32 | 31,0.00988 33 | 32,0.00762 34 | 33,0.00864 35 | 34,0.00818 36 | 35,0.00536 37 | 36,0.00603 38 | 37,0.007 39 | 38,0.00491 40 | 39,0.00868 41 | 40,0.00861 42 | 41,0.00766 43 | 42,0.00584 44 | 43,0.0044 45 | 44,0.0021 46 | 45,0.00523 47 | 46,0.00174 48 | 47,0.00562 49 | 48,0.0039 50 | 49,0.00279 51 | -------------------------------------------------------------------------------- /tests/csv/data/Components.csv: -------------------------------------------------------------------------------- 1 | Pos. x [mm];Pos. y [mm];Pos. z [mm];Component surface [mm²];Component min. deviation [mm];Component max. deviation [mm];Description;Image;Focused image;Captures;Tolerance status 2 | -29,334506989;3,739070892;-9,758856773;0,347590178;-0,598987699;-0,480000019;;Off;Off;0;No tolerance 3 | 49,488979340;12,218115807;-93,429969788;3,691705227;-0,507287800;-0,480000019;;Off;Off;0;No tolerance 4 | -22,663068771;39,907196045;-49,257804871;38,809677124;0,480000019;>= 0,600000024;;Off;Off;0;No tolerance 5 | -22,826931000;39,911468506;-20,156715393;38,821525574;0,480000019;>= 0,600000024;;Off;Off;0;No tolerance 6 | -22,591037750;39,901805878;-33,957965851;38,825820923;0,480000019;>= 0,600000024;;Off;Off;0;No tolerance 7 | -22,739957809;39,908485413;-65,457649231;38,827430725;0,480000019;>= 0,600000024;;Off;Off;0;No tolerance 8 | -22,650812149;41,070056915;-80,756362915;38,859138489;0,480000019;>= 0,600000024;;Off;Off;0;No tolerance 9 | 10 | -------------------------------------------------------------------------------- /tests/csv/data/defects.csv: -------------------------------------------------------------------------------- 1 | Probability,Radius [mm],Diameter [mm],Center x [mm],Center y [mm],Center z [mm],Volume [mm³],Voxel,Surface [mm²],Classification,Gap [mm],Compactness,Sphericity,Projected size x [mm],Projected size y [mm],Projected size z [mm],PCA deviation 1 [mm],PCA deviation 2 [mm],PCA deviation 3 [mm],PCA max. deviation ratio [%],PCA min. deviation ratio [%],Min. gray value,Max. gray value,Mean gray value,Deviation of gray values,Label,Projected area (yz-plane) [mm²],Projected area (xz-plane) [mm²],Projected area (xy-plane) [mm²],Tolerance status 2 | 0.00,0.02236,0.04472,-9.83108,-5.44005,15.91616,0.00002,0,0.00381,,0.07929,0.51,1.00,0.08943,0.04472,0.04472,0.01118,0.01118,0.01118,49.99999,50.00000,32696,32696,32696,0,6795,0.00200,0.00400,0.00400,Valid 3 | 0.00,0.02236,0.04472,7.45788,-8.50014,23.92020,0.00002,0,0.00381,,0.02511,0.51,1.00,0.08943,0.04472,0.04472,0.01118,0.01118,0.01118,49.99999,50.00000,27158,27158,27158,0,8991,0.00200,0.00400,0.00400,Valid 4 | 5 | -------------------------------------------------------------------------------- /src/print_args.rs: -------------------------------------------------------------------------------- 1 | use tracing::{info, Level}; 2 | use tracing_subscriber::FmtSubscriber; 3 | 4 | fn main() { 5 | // enable colors on windows cmd.exe 6 | // does not fail on powershell, even though powershell can do colors without this 7 | // will fail on jenkins/qa tough, that's why we need to ignore the result 8 | let _ = enable_ansi_support::enable_ansi_support(); 9 | 10 | let subscriber = FmtSubscriber::builder() 11 | .with_max_level(Level::INFO) 12 | .finish(); 13 | tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); 14 | info!("!Print-Args!"); 15 | let mut terminate_code = 0; 16 | for arg in std::env::args() { 17 | info!("Argument: {}", &arg); 18 | if arg.as_str() == "--exit-with-error" { 19 | eprintln!("E: setting error code to -1"); 20 | terminate_code = -1; 21 | } 22 | } 23 | std::process::exit(terminate_code); 24 | } 25 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # .github/workflows/release.yml 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | release: 9 | name: release ${{ matrix.target }} 10 | runs-on: ubuntu-latest 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | include: 15 | - target: x86_64-pc-windows-gnu 16 | archive: zip 17 | - target: x86_64-unknown-linux-musl 18 | archive: tar.gz tar.xz 19 | - target: x86_64-apple-darwin 20 | archive: zip 21 | steps: 22 | - uses: actions/checkout@v4 23 | - name: Compile and release 24 | uses: rust-build/rust-build.action@v1.4.5 25 | env: 26 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 27 | with: 28 | TOOLCHAIN_VERSION: stable 29 | RUSTTARGET: ${{ matrix.target }} 30 | ARCHIVE_TYPES: ${{ matrix.archive }} 31 | EXTRA_FILES: "README.md LICENSE config_scheme.json" 32 | -------------------------------------------------------------------------------- /tests/html/test.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | A Basic HTML5 Template 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /tests/html/html_changed.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | A Basic HTML5 Template 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Volume Graphics GmbH 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /tests/integ/vgrf.yml: -------------------------------------------------------------------------------- 1 | rules: 2 | - name: "VGRF-Reporting CSV comparing" 3 | pattern_include: 4 | - "**/*.csv" 5 | pattern_exclude: 6 | - "**/vg_report.csv" 7 | CSV: 8 | comparison_modes: 9 | - Absolute: 1.0 10 | - Relative: 0.1 11 | exclude_field_regex: "Excluded" 12 | preprocessing: 13 | - ExtractHeaders 14 | 15 | - name: "HTML-Compare strict" 16 | pattern_exclude: 17 | - "**/*_changed.html" 18 | pattern_include: 19 | - "**/*.html" 20 | PlainText: 21 | threshold: 1.0 22 | ignore_lines: 23 | - "stylesheet" 24 | - "next_ignore" 25 | - "[A-Z]*[0-9]" 26 | 27 | - name: "All files are lowercase, no spaces" 28 | pattern_include: 29 | - "**/*.*" 30 | pattern_exclude: null 31 | FileProperties: 32 | forbid_name_regex: "[\\s]" 33 | modification_date_tolerance_secs: 0 34 | file_size_tolerance_bytes: 0 35 | 36 | - name: "External checker" 37 | pattern_include: 38 | - "*.*" 39 | External: 40 | executable: "cargo" 41 | extra_params: 42 | - "run" 43 | - "--bin" 44 | - "print_args" 45 | - "--" 46 | - "--only-images" 47 | 48 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: coverage instrument based 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v3 12 | - name: Install latest nightly 13 | uses: actions-rs/toolchain@v1 14 | with: 15 | toolchain: nightly 16 | override: true 17 | components: rustfmt, clippy, llvm-tools-preview 18 | 19 | - name: Install lcov 20 | run: sudo apt-get install lcov 21 | 22 | - name: install grcov 23 | run: cargo install grcov 24 | 25 | - uses: actions/checkout@v3 26 | with: 27 | fetch-depth: 0 28 | 29 | - name: Run grcov 30 | env: 31 | PROJECT_NAME: "havocompare" 32 | RUSTFLAGS: "-Cinstrument-coverage -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" 33 | CARGO_INCREMENTAL: 0 34 | run: | 35 | cargo +nightly build --verbose 36 | cargo +nightly test --verbose 37 | grcov . -s . --binary-path ./target/debug/ -t lcov --llvm --branch --ignore-not-existing --ignore="/*" --ignore="target/*" --ignore="tests/*" -o lcov.info 38 | 39 | - name: Push grcov results to Coveralls via GitHub Action 40 | uses: coverallsapp/github-action@v2 41 | with: 42 | github-token: ${{ secrets.GITHUB_TOKEN }} 43 | file: "lcov.info" 44 | 45 | -------------------------------------------------------------------------------- /tests/csv/data/Multi_Apply_Rotation.csv: -------------------------------------------------------------------------------- 1 | Name,Coordinate system,Act. value [mm/deg],Status 2 | Feature XY,Datum system A_XY Rotation,0.00000 mm,OK 3 | Feature XY [2],Datum system A_XY Rotation,0.00000 mm,OK 4 | Feature XY [3],Datum system A_XY Rotation,0.00000 mm,OK 5 | Feature XY [4],Datum system A_XY Rotation,0.00000 mm,OK 6 | Feature XY [5],Datum system A_XY Rotation,0.00000 mm,OK 7 | Feature XY [6],Datum system A_XY Rotation,0.00000 mm,OK 8 | Feature XY [7],Datum system A_XY Rotation,0.00000 mm,OK 9 | Feature XY [8],Datum system A_XY Rotation,0.00000 mm,OK 10 | Feature XZ,Datum system A_XZ Rotation,0.00000 mm,OK 11 | Feature XZ [2],Datum system A_XZ Rotation,0.00000 mm,OK 12 | Feature XZ [3],Datum system A_XZ Rotation,0.00000 mm,OK 13 | Feature XZ [4],Datum system A_XZ Rotation,0.00000 mm,OK 14 | Feature XZ [5],Datum system A_XZ Rotation,0.00000 mm,OK 15 | Feature XZ [6],Datum system A_XZ Rotation,0.00000 mm,OK 16 | Feature XZ [7],Datum system A_XZ Rotation,0.00000 mm,OK 17 | Feature XZ [8],Datum system A_XZ Rotation,0.00000 mm,OK 18 | Feature YZ,Datum system A_YZ Rotation,0.00000 mm,OK 19 | Feature YZ [2],Datum system A_YZ Rotation,0.00000 mm,OK 20 | Feature YZ [3],Datum system A_YZ Rotation,0.00000 mm,OK 21 | Feature YZ [4],Datum system A_YZ Rotation,0.00000 mm,OK 22 | Feature YZ [5],Datum system A_YZ Rotation,0.00000 mm,OK 23 | Feature YZ [6],Datum system A_YZ Rotation,0.00000 mm,OK 24 | Feature YZ [7],Datum system A_YZ Rotation,0.00000 mm,OK 25 | Feature YZ [8],Datum system A_YZ Rotation,0.00000 mm,OK 26 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "havocompare" 3 | description = "A flexible rule-based file and folder comparison tool and crate including nice html reporting. Compares CSVs, JSON, text files, pdf-texts and images." 4 | repository = "https://github.com/VolumeGraphics/havocompare" 5 | homepage = "https://github.com/VolumeGraphics/havocompare" 6 | documentation = "https://docs.rs/havocompare" 7 | version = "0.8.0" 8 | edition = "2021" 9 | license = "MIT" 10 | authors = ["Volume Graphics GmbH"] 11 | exclude = ["tests/pdf", "tests/integ", "tests/html", "target", "tests/csv", ".github", "test_report"] 12 | keywords = ["diff", "compare", "csv", "image", "difference"] 13 | categories = ["filesystem"] 14 | default-run = "havocompare" 15 | 16 | [[bin]] 17 | name = "print_args" 18 | path = "src/print_args.rs" 19 | 20 | 21 | [dependencies] 22 | clap = { version = "4.5", features = ["derive"] } 23 | chrono = "0.4" 24 | serde = "1.0" 25 | serde_yaml = "0.9" 26 | schemars = "0.8" 27 | schemars_derive = "0.8" 28 | thiserror = "2.0" 29 | regex = "1.10" 30 | image = "0.25" 31 | image-compare = "0.4" 32 | tracing = "0.1" 33 | tracing-subscriber = "0.3" 34 | serde_json = "1.0" 35 | glob = "0.3" 36 | test-log = { version = "0.2", features = ["trace"] } 37 | strsim = "0.11" 38 | itertools = "0.14" 39 | tera = "1.19" 40 | sha2 = "0.10" 41 | data-encoding = "2.6" 42 | permutation = "0.4" 43 | pdf-extract = "0.9" 44 | vg_errortools = "0.1" 45 | rayon = "1.10.0" 46 | enable-ansi-support = "0.2" 47 | tempfile = "3.20" 48 | fs_extra = "1.3" 49 | opener = "0.7" 50 | anyhow = "1.0" 51 | json_diff_ng = { version = "0.6", default-features = false } 52 | 53 | [dev-dependencies] 54 | env_logger = "0.11" 55 | tracing = { version = "0.1", default-features = false } 56 | tracing-subscriber = { version = "0.3", default-features = false, features = ["env-filter", "fmt"] } 57 | -------------------------------------------------------------------------------- /tests/integ.rs: -------------------------------------------------------------------------------- 1 | use havocompare::compare_folders; 2 | use test_log::test; 3 | 4 | #[test] 5 | fn simple_test_identity() { 6 | let report_dir = 7 | tempfile::tempdir().expect("Could not generate temporary directory for report"); 8 | let result = compare_folders("tests/", "tests/", "tests/integ/config.yml", report_dir); 9 | assert!(result.unwrap()); 10 | } 11 | 12 | #[test] 13 | fn display_of_status_message_in_cm_tables() { 14 | let report_dir = 15 | tempfile::tempdir().expect("Could not generate temporary directory for report"); 16 | 17 | assert!(compare_folders( 18 | "tests/integ/data/display_of_status_message_in_cm_tables/expected/", 19 | "tests/integ/data/display_of_status_message_in_cm_tables/actual/", 20 | "tests/integ/vgrf.yml", 21 | report_dir 22 | ) 23 | .unwrap()); 24 | } 25 | 26 | #[test] 27 | fn images_test() { 28 | let report_dir = 29 | tempfile::tempdir().expect("Could not generate temporary directory for report"); 30 | 31 | assert!(compare_folders( 32 | "tests/integ/data/images/expected/", 33 | "tests/integ/data/images/actual/", 34 | "tests/integ/jpg_compare.yml", 35 | report_dir 36 | ) 37 | .unwrap()); 38 | } 39 | 40 | #[test] 41 | fn json_test() { 42 | let report_dir = 43 | tempfile::tempdir().expect("Could not generate temporary directory for report"); 44 | 45 | assert!(!compare_folders( 46 | "tests/integ/data/json/expected/", 47 | "tests/integ/data/json/actual/", 48 | "tests/integ/json.yml", 49 | report_dir 50 | ) 51 | .unwrap()); 52 | let report_dir = 53 | tempfile::tempdir().expect("Could not generate temporary directory for report"); 54 | 55 | assert!(compare_folders( 56 | "tests/integ/data/json/expected/", 57 | "tests/integ/data/json/expected/", 58 | "tests/integ/json.yml", 59 | report_dir 60 | ) 61 | .unwrap()); 62 | } 63 | -------------------------------------------------------------------------------- /src/external.rs: -------------------------------------------------------------------------------- 1 | use crate::report::{DiffDetail, Difference}; 2 | use crate::Error; 3 | use schemars::JsonSchema; 4 | use serde::{Deserialize, Serialize}; 5 | use std::path::Path; 6 | use tracing::{error, info}; 7 | 8 | #[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)] 9 | pub struct ExternalConfig { 10 | /// The executable to call - will be started like: `#executable #(#extra_params)* #nominal #actual` 11 | executable: String, 12 | /// Extra parameters to pass 13 | extra_params: Vec, 14 | } 15 | 16 | pub(crate) fn compare_files>( 17 | nominal: P, 18 | actual: P, 19 | config: &ExternalConfig, 20 | ) -> Result { 21 | let mut diff = Difference::new_for_file(&nominal, &actual); 22 | let compared_file_name = nominal.as_ref().to_string_lossy().into_owned(); 23 | let output = std::process::Command::new(&config.executable) 24 | .args(&config.extra_params) 25 | .arg(nominal.as_ref()) 26 | .arg(actual.as_ref()) 27 | .output(); 28 | if let Ok(output) = output { 29 | let stdout = String::from_utf8_lossy(&output.stdout).to_string(); 30 | let stderr = String::from_utf8_lossy(&output.stderr).to_string(); 31 | info!("External stdout: {}", stdout.as_str()); 32 | info!("External stderr: {}", stderr.as_str()); 33 | if !output.status.success() { 34 | let message = format!("External checker denied file {}", &compared_file_name); 35 | error!("{}", &message); 36 | diff.push_detail(DiffDetail::External { stdout, stderr }); 37 | diff.error(); 38 | }; 39 | } else { 40 | let error_message = format!( 41 | "External checker execution failed for file {}", 42 | &compared_file_name 43 | ); 44 | error!("{}", error_message); 45 | diff.push_detail(DiffDetail::Error(error_message)); 46 | diff.error(); 47 | }; 48 | Ok(diff) 49 | } 50 | #[cfg(test)] 51 | mod tests { 52 | use super::*; 53 | use test_log::test; 54 | 55 | #[test] 56 | fn test_non_existent_exe() { 57 | let result = compare_files( 58 | Path::new("file1"), 59 | Path::new("file2"), 60 | &ExternalConfig { 61 | extra_params: Vec::new(), 62 | executable: "non_existent".to_owned(), 63 | }, 64 | ) 65 | .unwrap(); 66 | assert!(result.is_error); 67 | } 68 | 69 | #[test] 70 | fn test_bad_output() { 71 | let result = compare_files( 72 | Path::new("file1"), 73 | Path::new("file2"), 74 | &ExternalConfig { 75 | extra_params: vec![ 76 | "run".to_owned(), 77 | "--bin".to_owned(), 78 | "print_args".to_owned(), 79 | "--".to_owned(), 80 | "--exit-with-error".to_owned(), 81 | ], 82 | executable: "cargo".to_owned(), 83 | }, 84 | ) 85 | .unwrap(); 86 | assert!(result.is_error); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/pdf.rs: -------------------------------------------------------------------------------- 1 | use crate::html::HTMLCompareConfig; 2 | use crate::report; 3 | use crate::report::{DiffDetail, Difference}; 4 | use pdf_extract::extract_text; 5 | use std::path::Path; 6 | use strsim::normalized_damerau_levenshtein; 7 | use thiserror::Error; 8 | use tracing::{error, info}; 9 | use vg_errortools::FatIOError; 10 | 11 | #[derive(Debug, Error)] 12 | /// Errors during html / plain text checking 13 | pub enum Error { 14 | #[error("Failed to compile regex {0}")] 15 | RegexCompilationFailure(#[from] regex::Error), 16 | #[error("Problem creating hash report {0}")] 17 | ReportingFailure(#[from] report::Error), 18 | #[error("File access failed {0}")] 19 | FileAccessProblem(#[from] FatIOError), 20 | #[error("PDF text extraction error {0}")] 21 | PdfTextExtractionFailed(#[from] pdf_extract::OutputError), 22 | } 23 | 24 | pub fn compare_files>( 25 | nominal_path: P, 26 | actual_path: P, 27 | config: &HTMLCompareConfig, 28 | ) -> Result { 29 | info!("Extracting text from actual pdf"); 30 | let actual = extract_text(actual_path.as_ref())?; 31 | 32 | info!("Extracting text from nominal pdf"); 33 | let nominal = extract_text(nominal_path.as_ref())?; 34 | 35 | let exclusion_list = config.get_ignore_list()?; 36 | let mut difference = Difference::new_for_file(&nominal_path, &actual_path); 37 | actual 38 | .lines() 39 | .enumerate() 40 | .zip(nominal.lines()) 41 | .filter(|((_, a), n)| 42 | exclusion_list.iter().all(|exc| !exc.is_match(a)) && exclusion_list.iter().all(|exc| !exc.is_match(n)) 43 | ) 44 | .for_each(|((l, a), n)| { 45 | let distance = normalized_damerau_levenshtein(a,n); 46 | if distance < config.threshold { 47 | 48 | let error = format!( 49 | "Missmatch in PDF-Text-file in line {}. Expected: '{}' found '{}' (diff: {}, threshold: {})", 50 | l, n, a, distance, config.threshold 51 | ); 52 | 53 | error!("{}" , &error); 54 | difference.push_detail(DiffDetail::Text {actual:a.to_owned(), nominal:n.to_owned(), score: distance, line: l}); 55 | difference.error(); 56 | } 57 | }); 58 | 59 | Ok(difference) 60 | } 61 | 62 | #[cfg(test)] 63 | mod test { 64 | use super::*; 65 | 66 | #[test] 67 | fn test_compare_pdf() { 68 | let result = compare_files( 69 | "tests/pdf/actual.pdf", 70 | "tests/pdf/expected.pdf", 71 | &HTMLCompareConfig::default(), 72 | ) 73 | .unwrap(); 74 | assert!(result.is_error); 75 | 76 | let result = compare_files( 77 | "tests/pdf/actual.pdf", 78 | "tests/pdf/actual.pdf", 79 | &HTMLCompareConfig::default(), 80 | ) 81 | .unwrap(); 82 | assert!(!result.is_error); 83 | } 84 | 85 | #[test] 86 | fn test_ignore_line_pdf() { 87 | let result = compare_files( 88 | "tests/pdf/actual.pdf", 89 | "tests/pdf/expected.pdf", 90 | &HTMLCompareConfig { 91 | threshold: 1.0, 92 | ignore_lines: Some(vec!["/workspace/".to_owned()]), 93 | }, 94 | ) 95 | .unwrap(); 96 | assert!(!result.is_error); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/hash.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::io::Read; 3 | use std::path::{Path, PathBuf}; 4 | 5 | use data_encoding::HEXLOWER; 6 | use schemars_derive::JsonSchema; 7 | use thiserror::Error; 8 | use vg_errortools::fat_io_wrap_std; 9 | use vg_errortools::FatIOError; 10 | 11 | use crate::report::{DiffDetail, Difference}; 12 | use crate::{report, Deserialize, Serialize}; 13 | 14 | #[derive(Debug, Deserialize, Serialize, JsonSchema, Clone, Copy)] 15 | pub enum HashFunction { 16 | Sha256, 17 | } 18 | 19 | #[derive(Debug, Error)] 20 | /// Errors during hash checking 21 | pub enum Error { 22 | #[error("Failed to compile regex {0}")] 23 | RegexCompilationFailed(#[from] regex::Error), 24 | #[error("Problem creating hash report {0}")] 25 | ReportingFailure(#[from] report::Error), 26 | #[error("File access failed {0}")] 27 | FileAccessProblem(#[from] FatIOError), 28 | } 29 | 30 | impl HashFunction { 31 | fn hash_file(&self, mut file: impl Read) -> Result<[u8; 32], Error> { 32 | match self { 33 | Self::Sha256 => { 34 | use sha2::{Digest, Sha256}; 35 | use std::io; 36 | 37 | let mut hasher = Sha256::new(); 38 | 39 | let _ = io::copy(&mut file, &mut hasher) 40 | .map_err(|e| FatIOError::from_std_io_err(e, PathBuf::new()))?; 41 | let hash_bytes = hasher.finalize(); 42 | Ok(hash_bytes.into()) 43 | } 44 | } 45 | } 46 | } 47 | 48 | #[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)] 49 | /// Configuration options for the hash comparison module 50 | pub struct HashConfig { 51 | /// Which hash function to use 52 | pub function: HashFunction, 53 | } 54 | 55 | impl Default for HashConfig { 56 | fn default() -> Self { 57 | HashConfig { 58 | function: HashFunction::Sha256, 59 | } 60 | } 61 | } 62 | 63 | pub fn compare_files>( 64 | nominal_path: P, 65 | actual_path: P, 66 | config: &HashConfig, 67 | ) -> Result { 68 | let act = config 69 | .function 70 | .hash_file(fat_io_wrap_std(actual_path.as_ref(), &File::open)?)?; 71 | let nom = config 72 | .function 73 | .hash_file(fat_io_wrap_std(nominal_path.as_ref(), &File::open)?)?; 74 | 75 | let mut difference = Difference::new_for_file(nominal_path, actual_path); 76 | if act != nom { 77 | difference.push_detail(DiffDetail::Hash { 78 | actual: HEXLOWER.encode(&act), 79 | nominal: HEXLOWER.encode(&nom), 80 | }); 81 | difference.error(); 82 | } 83 | Ok(difference) 84 | } 85 | 86 | #[cfg(test)] 87 | mod test { 88 | use crate::hash::HashFunction::Sha256; 89 | 90 | use super::*; 91 | 92 | #[test] 93 | fn identity() { 94 | let f1 = Sha256 95 | .hash_file(File::open("tests/integ.rs").unwrap()) 96 | .unwrap(); 97 | let f2 = Sha256 98 | .hash_file(File::open("tests/integ.rs").unwrap()) 99 | .unwrap(); 100 | assert_eq!(f1, f2); 101 | } 102 | 103 | #[test] 104 | fn hash_pinning() { 105 | let sum = "378f768a589f29fcbd23835ec4764a53610fc910e60b540e1e5204bdaf2c73a0"; 106 | let f1 = Sha256 107 | .hash_file(File::open("tests/integ/data/images/diff_100_DPI.png").unwrap()) 108 | .unwrap(); 109 | assert_eq!(HEXLOWER.encode(&f1), sum); 110 | } 111 | 112 | #[test] 113 | fn identity_outer() { 114 | let file = "tests/integ.rs"; 115 | let result = compare_files(file, file, &HashConfig::default()).unwrap(); 116 | assert!(!result.is_error); 117 | } 118 | 119 | #[test] 120 | fn different_files_throw_outer() { 121 | let file_act = "tests/integ/data/images/actual/SaveImage_100DPI_default_size.jpg"; 122 | let file_nominal = "tests/integ/data/images/expected/SaveImage_100DPI_default_size.jpg"; 123 | 124 | let result = compare_files(file_act, file_nominal, &HashConfig::default()).unwrap(); 125 | assert!(result.is_error); 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use anyhow::anyhow; 2 | use clap::Parser; 3 | use havocompare::{compare_files, compare_folders, get_schema, validate_config, ComparisonMode}; 4 | use std::path::{Path, PathBuf}; 5 | use tracing::{info, Level}; 6 | use tracing_subscriber::FmtSubscriber; 7 | 8 | const DEFAULT_REPORT_FOLDER: &str = "report"; 9 | 10 | #[derive(clap::Subcommand)] 11 | enum Commands { 12 | /// Compare two folders using a config file 13 | Compare { 14 | /// Nominal data folder 15 | nominal: String, 16 | /// Actual data folder 17 | actual: String, 18 | /// Path to compare config YAML 19 | compare_config: String, 20 | /// Optional: Folder to store the report to, if not set the default location will be chosen. 21 | #[arg(short, long = "report_path", default_value_t = DEFAULT_REPORT_FOLDER.to_string())] 22 | report_config: String, 23 | /// Open the report immediately after comparison 24 | #[arg(short, long)] 25 | open: bool, 26 | }, 27 | /// Compare two files given a config-string that contains a json-serialized config 28 | FileCompare { 29 | /// nominal file 30 | nominal: PathBuf, 31 | /// actual file 32 | actual: PathBuf, 33 | /// compare_configuration in json 34 | config: String, 35 | }, 36 | 37 | /// Export the JsonSchema for the config files 38 | Schema, 39 | 40 | /// Validate config yaml 41 | Validate { compare_config: String }, 42 | } 43 | 44 | #[derive(Parser)] 45 | #[command(author, version, about, long_about = None)] 46 | struct Arguments { 47 | #[clap(short, long)] 48 | /// print debug information about the run 49 | verbose: bool, 50 | #[clap(subcommand)] 51 | /// choose the command to run 52 | command: Commands, 53 | } 54 | 55 | fn main() -> Result<(), vg_errortools::MainError> { 56 | let args = Arguments::parse(); 57 | let level = if args.verbose { 58 | Level::DEBUG 59 | } else { 60 | Level::INFO 61 | }; 62 | 63 | // enable colors on windows cmd.exe 64 | // does not fail on powershell, even though powershell can do colors without this 65 | // will fail on jenkins/qa tough, that's why we need to ignore the result 66 | let _ = enable_ansi_support::enable_ansi_support(); 67 | 68 | let subscriber = FmtSubscriber::builder().with_max_level(level).finish(); 69 | tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); 70 | match args.command { 71 | Commands::Schema => { 72 | println!( 73 | "{}", 74 | get_schema().expect("Error occurred writing json schema") 75 | ); 76 | Ok(()) 77 | } 78 | Commands::Compare { 79 | compare_config, 80 | nominal, 81 | actual, 82 | report_config, 83 | open, 84 | } => { 85 | let report_path = Path::new(report_config.as_str()); 86 | let result = compare_folders(nominal, actual, compare_config, report_path)?; 87 | if open { 88 | info!("Opening report"); 89 | opener::open(report_path.join("index.html")).expect("Could not open report!"); 90 | } 91 | if result { 92 | Ok(()) 93 | } else { 94 | Err(anyhow!("Comparison failed!").into()) 95 | } 96 | } 97 | Commands::FileCompare { 98 | nominal, 99 | actual, 100 | config, 101 | } => { 102 | use anyhow::Context; 103 | let config: ComparisonMode = 104 | serde_json::from_str(&config).context("Couldn't deserialize the config string")?; 105 | let result = compare_files(nominal, actual, &config); 106 | info!("Diff results: {result:#?}"); 107 | if result.is_error { 108 | Err(anyhow!("Comparison failed!").into()) 109 | } else { 110 | Ok(()) 111 | } 112 | } 113 | Commands::Validate { compare_config } => { 114 | if validate_config(compare_config) { 115 | Ok(()) 116 | } else { 117 | Err(anyhow!("Validation failed!").into()) 118 | } 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/html.rs: -------------------------------------------------------------------------------- 1 | use crate::report; 2 | use crate::report::{DiffDetail, Difference}; 3 | use regex::Regex; 4 | use schemars_derive::JsonSchema; 5 | use serde::{Deserialize, Serialize}; 6 | use std::fs::File; 7 | use std::io::{BufRead, BufReader}; 8 | use std::path::Path; 9 | use strsim::normalized_damerau_levenshtein; 10 | use thiserror::Error; 11 | use tracing::error; 12 | use vg_errortools::fat_io_wrap_std; 13 | use vg_errortools::FatIOError; 14 | 15 | #[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)] 16 | /// Plain text comparison config, also used for PDF 17 | pub struct HTMLCompareConfig { 18 | /// Normalized Damerau-Levenshtein distance, 0.0 = bad, 1.0 = identity 19 | pub threshold: f64, 20 | /// Lines matching any of the given regex will be excluded from comparison 21 | pub ignore_lines: Option>, 22 | } 23 | 24 | impl HTMLCompareConfig { 25 | pub(crate) fn get_ignore_list(&self) -> Result, regex::Error> { 26 | let exclusion_list: Option, regex::Error>> = self 27 | .ignore_lines 28 | .as_ref() 29 | .map(|v| v.iter().map(|exc| Regex::new(exc)).collect()); 30 | let exclusion_list = match exclusion_list { 31 | Some(r) => r?, 32 | None => Vec::new(), 33 | }; 34 | Ok(exclusion_list) 35 | } 36 | } 37 | 38 | impl Default for HTMLCompareConfig { 39 | fn default() -> Self { 40 | HTMLCompareConfig { 41 | threshold: 1.0, 42 | ignore_lines: None, 43 | } 44 | } 45 | } 46 | 47 | #[derive(Debug, Error)] 48 | /// Errors during html / plain text checking 49 | pub enum Error { 50 | #[error("Failed to compile regex {0}")] 51 | RegexCompilationFailure(#[from] regex::Error), 52 | #[error("Problem creating hash report {0}")] 53 | ReportingProblem(#[from] report::Error), 54 | #[error("File access failed {0}")] 55 | FileAccessFailure(#[from] FatIOError), 56 | } 57 | 58 | pub fn compare_files>( 59 | nominal_path: P, 60 | actual_path: P, 61 | config: &HTMLCompareConfig, 62 | ) -> Result { 63 | let actual = BufReader::new(fat_io_wrap_std(actual_path.as_ref(), &File::open)?); 64 | let nominal = BufReader::new(fat_io_wrap_std(nominal_path.as_ref(), &File::open)?); 65 | 66 | let exclusion_list = config.get_ignore_list()?; 67 | let mut difference = Difference::new_for_file(nominal_path, actual_path); 68 | actual 69 | .lines() 70 | .enumerate() 71 | .filter_map(|l| l.1.ok().map(|a| (l.0, a))) 72 | .zip(nominal.lines().map_while(Result::ok)) 73 | .filter(|((_, a), n)| 74 | exclusion_list.iter().all(|exc| !exc.is_match(a)) && exclusion_list.iter().all(|exc| !exc.is_match(n)) 75 | ) 76 | .for_each(|((l, a), n)| { 77 | let distance = normalized_damerau_levenshtein(a.as_str(),n.as_str()); 78 | if distance < config.threshold { 79 | 80 | let error = format!( 81 | "Mismatch in HTML-file in line {}. Expected: '{}' found '{}' (diff: {}, threshold: {})", 82 | l, n, a, distance, config.threshold 83 | ); 84 | 85 | error!("{}" , &error); 86 | difference.push_detail(DiffDetail::Text {actual: a, nominal: n, score: distance, line: l}); 87 | difference.error(); 88 | } 89 | }); 90 | 91 | Ok(difference) 92 | } 93 | 94 | #[cfg(test)] 95 | mod test { 96 | use super::*; 97 | use test_log::test; 98 | #[test] 99 | fn test_identity() { 100 | assert!( 101 | !compare_files( 102 | "tests/html/test.html", 103 | "tests/html/test.html", 104 | &HTMLCompareConfig::default(), 105 | ) 106 | .unwrap() 107 | .is_error 108 | ); 109 | } 110 | 111 | #[test] 112 | fn test_modified() { 113 | let actual = "tests/html/test.html"; 114 | let nominal = "tests/html/html_changed.html"; 115 | 116 | let result = compare_files(actual, nominal, &HTMLCompareConfig::default()).unwrap(); 117 | 118 | assert!(result.is_error); 119 | } 120 | 121 | #[test] 122 | fn test_allow_modified_threshold() { 123 | assert!( 124 | !compare_files( 125 | "tests/html/test.html", 126 | "tests/html/html_changed.html", 127 | &HTMLCompareConfig { 128 | threshold: 0.9, 129 | ignore_lines: None 130 | }, 131 | ) 132 | .unwrap() 133 | .is_error 134 | ); 135 | } 136 | 137 | #[test] 138 | fn test_ignore_lines_regex() { 139 | assert!( 140 | !compare_files( 141 | "tests/html/test.html", 142 | "tests/html/html_changed.html", 143 | &HTMLCompareConfig { 144 | threshold: 1.0, 145 | ignore_lines: Some(vec!["stylesheet".to_owned()]) 146 | }, 147 | ) 148 | .unwrap() 149 | .is_error 150 | ); 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/csv/value.rs: -------------------------------------------------------------------------------- 1 | use schemars_derive::JsonSchema; 2 | use serde::{Deserialize, Serialize}; 3 | use std::borrow::Cow; 4 | use std::fmt::{Display, Formatter}; 5 | 6 | #[derive(Debug, Clone, JsonSchema, Deserialize, Serialize, PartialEq)] 7 | pub struct Quantity { 8 | pub(crate) value: f64, 9 | pub(crate) unit: Option, 10 | } 11 | 12 | fn next_up(val: f64) -> f64 { 13 | const TINY_BITS: u64 = 0x1; // Smallest positive f64. 14 | const CLEAR_SIGN_MASK: u64 = 0x7fff_ffff_ffff_ffff; 15 | 16 | let bits = val.to_bits(); 17 | if val.is_nan() || bits == f64::INFINITY.to_bits() { 18 | return val; 19 | } 20 | 21 | let abs = bits & CLEAR_SIGN_MASK; 22 | let next_bits = if abs == 0 { 23 | TINY_BITS 24 | } else if bits == abs { 25 | bits + 1 26 | } else { 27 | bits - 1 28 | }; 29 | f64::from_bits(next_bits) 30 | } 31 | 32 | fn next_down(val: f64) -> f64 { 33 | const NEG_TINY_BITS: u64 = 0x8000_0000_0000_0001; // Smallest (in magnitude) negative f64. 34 | const CLEAR_SIGN_MASK: u64 = 0x7fff_ffff_ffff_ffff; 35 | 36 | let bits = val.to_bits(); 37 | if val.is_nan() || bits == f64::NEG_INFINITY.to_bits() { 38 | return val; 39 | } 40 | 41 | let abs: u64 = bits & CLEAR_SIGN_MASK; 42 | let next_bits = if abs == 0 { 43 | NEG_TINY_BITS 44 | } else if bits == abs { 45 | bits - 1 46 | } else { 47 | bits + 1 48 | }; 49 | f64::from_bits(next_bits) 50 | } 51 | 52 | impl Quantity { 53 | #[cfg(test)] 54 | pub(crate) fn new(value: f64, unit: Option<&str>) -> Self { 55 | Self { 56 | unit: unit.map(|s| s.to_owned()), 57 | value, 58 | } 59 | } 60 | 61 | /// This avoids the issue of `(a - b) > d` for `b = a + d` with small `d` 62 | pub(crate) fn minimal_diff(&self, rhs: &Quantity) -> f64 { 63 | let min = self.value.min(rhs.value); 64 | let max = self.value.max(rhs.value); 65 | let min_up = next_up(min); 66 | let max_down = next_down(max); 67 | next_down(max_down - min_up) 68 | } 69 | } 70 | 71 | impl Display for Quantity { 72 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 73 | if let Some(unit) = self.unit.as_deref() { 74 | write!(f, "{} {}", self.value, unit) 75 | } else { 76 | write!(f, "{}", self.value) 77 | } 78 | } 79 | } 80 | 81 | #[derive(Debug, PartialEq, Clone, Serialize)] 82 | pub enum Value { 83 | Quantity(Quantity), 84 | String(String), 85 | } 86 | 87 | impl Display for Value { 88 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 89 | match &self { 90 | Value::Quantity(val) => { 91 | write!(f, "{val}").unwrap(); 92 | } 93 | Value::String(val) => { 94 | write!(f, "'{val}'").unwrap(); 95 | } 96 | } 97 | Ok(()) 98 | } 99 | } 100 | 101 | impl Value { 102 | pub fn deleted() -> Value { 103 | Value::from_str("DELETED", &None) 104 | } 105 | 106 | fn get_numerical_value(field_split: &[&str]) -> Option { 107 | if field_split.len() == 1 || field_split.len() == 2 { 108 | return field_split.first().and_then(|s| s.parse::().ok()); 109 | } 110 | None 111 | } 112 | 113 | pub fn from_str(s: &str, decimal_separator: &Option) -> Value { 114 | let field_string: String = if let Some(delim) = decimal_separator { 115 | s.replace(*delim, ".") 116 | } else { 117 | s.into() 118 | }; 119 | 120 | let field_split: Vec<_> = field_string.trim().split(' ').collect(); 121 | 122 | if let Some(float_value) = Self::get_numerical_value(field_split.as_slice()) { 123 | Value::Quantity(Quantity { 124 | value: float_value, 125 | unit: field_split.get(1).map(|&s| s.to_owned()), 126 | }) 127 | } else { 128 | Value::String(s.trim().to_owned()) 129 | } 130 | } 131 | 132 | pub fn get_quantity(&self) -> Option<&Quantity> { 133 | match self { 134 | Value::Quantity(quantity) => Some(quantity), 135 | _ => None, 136 | } 137 | } 138 | 139 | pub fn get_string(&self) -> Option { 140 | match self { 141 | Value::String(string) => Some(string.to_owned()), 142 | _ => None, 143 | } 144 | } 145 | 146 | pub fn as_str(&self) -> Cow { 147 | match self { 148 | Value::String(str) => str.as_str().into(), 149 | Value::Quantity(quant) => quant.to_string().into(), 150 | } 151 | } 152 | } 153 | 154 | #[cfg(test)] 155 | mod tests { 156 | use super::*; 157 | use crate::csv::Mode; 158 | #[test] 159 | fn trimming() { 160 | let val_spaced = Value::from_str(" value ", &None); 161 | let reference = Value::from_str("value", &None); 162 | assert_eq!(val_spaced, reference); 163 | } 164 | 165 | #[test] 166 | fn test_secure_diff() { 167 | for base in -30..=30 { 168 | for modulation in -30..=base { 169 | let magic_factor = 1.3; 170 | let num_one = magic_factor * 10.0f64.powi(base); 171 | let delta = magic_factor * 10.0f64.powi(modulation); 172 | let compare_mode = Mode::Absolute(delta.abs()); 173 | let num_modulated = num_one + delta; 174 | let q1 = Quantity::new(num_one, None); 175 | let q2 = Quantity::new(num_modulated, None); 176 | // assert!(u_diff <= delta); 177 | assert!(compare_mode.in_tolerance(&q1, &q2)); 178 | } 179 | } 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /src/json.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | 3 | use itertools::Itertools; 4 | use json_diff_ng::DiffType; 5 | use regex::Regex; 6 | use schemars_derive::JsonSchema; 7 | use serde::{Deserialize, Serialize}; 8 | use tracing::error; 9 | 10 | use crate::report::{DiffDetail, Difference}; 11 | use crate::Error; 12 | 13 | #[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)] 14 | /// configuration for the json compare module 15 | pub struct JsonConfig { 16 | #[serde(default)] 17 | ignore_keys: Vec, 18 | #[serde(default)] 19 | sort_arrays: bool, 20 | } 21 | impl JsonConfig { 22 | pub(crate) fn get_ignore_list(&self) -> Result, regex::Error> { 23 | self.ignore_keys.iter().map(|v| Regex::new(v)).collect() 24 | } 25 | } 26 | 27 | pub(crate) fn compare_files>( 28 | nominal: P, 29 | actual: P, 30 | config: &JsonConfig, 31 | ) -> Result { 32 | let mut diff = Difference::new_for_file(&nominal, &actual); 33 | let compared_file_name = nominal.as_ref().to_string_lossy().into_owned(); 34 | 35 | let nominal: String = vg_errortools::fat_io_wrap_std(&nominal, &std::fs::read_to_string)?; 36 | let actual: String = vg_errortools::fat_io_wrap_std(&actual, &std::fs::read_to_string)?; 37 | let ignores = config.get_ignore_list()?; 38 | 39 | let json_diff = json_diff_ng::compare_strs(&nominal, &actual, config.sort_arrays, &ignores); 40 | let json_diff = match json_diff { 41 | Ok(diff) => diff, 42 | Err(e) => { 43 | let error_message = 44 | format!("JSON comparison failed for {compared_file_name} (error: {e})"); 45 | error!("{}", error_message); 46 | diff.push_detail(DiffDetail::Error(error_message)); 47 | diff.error(); 48 | return Ok(diff); 49 | } 50 | }; 51 | let filtered_diff: Vec<_> = json_diff.all_diffs(); 52 | 53 | if !filtered_diff.is_empty() { 54 | for (d_type, key) in filtered_diff.iter() { 55 | error!("{d_type}: {key}"); 56 | } 57 | let left = filtered_diff 58 | .iter() 59 | .filter_map(|(k, v)| { 60 | if matches!(k, DiffType::LeftExtra) { 61 | Some(v.to_string()) 62 | } else { 63 | None 64 | } 65 | }) 66 | .join("\n"); 67 | let right = filtered_diff 68 | .iter() 69 | .filter_map(|(k, v)| { 70 | if matches!(k, DiffType::RightExtra) { 71 | Some(v.to_string()) 72 | } else { 73 | None 74 | } 75 | }) 76 | .join("\n"); 77 | let differences = filtered_diff 78 | .iter() 79 | .filter_map(|(k, v)| { 80 | if matches!(k, DiffType::Mismatch) { 81 | Some(v.to_string()) 82 | } else { 83 | None 84 | } 85 | }) 86 | .join("\n"); 87 | let root_mismatch = filtered_diff 88 | .iter() 89 | .find(|(k, _v)| matches!(k, DiffType::RootMismatch)) 90 | .map(|(_, v)| v.to_string()); 91 | 92 | diff.push_detail(DiffDetail::Json { 93 | differences, 94 | left, 95 | right, 96 | root_mismatch, 97 | }); 98 | 99 | diff.error(); 100 | } 101 | 102 | Ok(diff) 103 | } 104 | 105 | #[cfg(test)] 106 | mod test { 107 | use super::*; 108 | 109 | fn trim_split(list: &str) -> Vec<&str> { 110 | list.split('\n').map(|e| e.trim()).collect() 111 | } 112 | 113 | #[test] 114 | fn no_filter() { 115 | let cfg = JsonConfig { 116 | ignore_keys: vec![], 117 | sort_arrays: false, 118 | }; 119 | let result = compare_files( 120 | "tests/integ/data/json/expected/guy.json", 121 | "tests/integ/data/json/actual/guy.json", 122 | &cfg, 123 | ) 124 | .unwrap(); 125 | if let DiffDetail::Json { 126 | differences, 127 | left, 128 | right, 129 | root_mismatch, 130 | } = result.detail.first().unwrap() 131 | { 132 | let differences = trim_split(differences); 133 | 134 | assert!(differences.contains(&".car.(\"RX7\" != \"Panda Trueno\")")); 135 | assert!(differences.contains(&".age.(21 != 18)")); 136 | assert!(differences.contains(&".name.(\"Keisuke\" != \"Takumi\")")); 137 | assert_eq!(differences.len(), 3); 138 | 139 | assert_eq!(left.as_str(), ".brothers"); 140 | assert!(right.is_empty()); 141 | assert!(root_mismatch.is_none()); 142 | } else { 143 | panic!("wrong diffdetail"); 144 | } 145 | } 146 | 147 | #[test] 148 | fn filter_works() { 149 | let cfg = JsonConfig { 150 | ignore_keys: vec!["name".to_string(), "brother(s?)".to_string()], 151 | sort_arrays: false, 152 | }; 153 | let result = compare_files( 154 | "tests/integ/data/json/expected/guy.json", 155 | "tests/integ/data/json/actual/guy.json", 156 | &cfg, 157 | ) 158 | .unwrap(); 159 | if let DiffDetail::Json { 160 | differences, 161 | left, 162 | right, 163 | root_mismatch, 164 | } = result.detail.first().unwrap() 165 | { 166 | let differences = trim_split(differences); 167 | assert!(differences.contains(&".car.(\"RX7\" != \"Panda Trueno\")")); 168 | assert!(differences.contains(&".age.(21 != 18)")); 169 | assert_eq!(differences.len(), 2); 170 | assert!(right.is_empty()); 171 | assert!(left.is_empty()); 172 | assert!(root_mismatch.is_none()); 173 | } else { 174 | panic!("wrong diffdetail"); 175 | } 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /src/directory.rs: -------------------------------------------------------------------------------- 1 | use crate::report::{DiffDetail, Difference}; 2 | use schemars_derive::JsonSchema; 3 | use serde::{Deserialize, Serialize}; 4 | use std::path; 5 | use std::path::{Path, PathBuf}; 6 | use thiserror::Error; 7 | use tracing::error; 8 | 9 | #[derive(Debug, Error)] 10 | /// Errors during html / plain text checking 11 | pub enum Error { 12 | #[error("Failed to remove path's prefix")] 13 | StripPrefixError(#[from] path::StripPrefixError), 14 | } 15 | 16 | #[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)] 17 | pub struct DirectoryConfig { 18 | pub mode: Mode, 19 | } 20 | 21 | #[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)] 22 | pub enum Mode { 23 | /// check whether both paths are really the same: whether entry is missing in actual, and/or if entry exists in actual but not in nominal 24 | Identical, 25 | /// check only if entry is missing in actual, ignoring entries that exist in actual but not in nominal 26 | MissingOnly, 27 | } 28 | 29 | pub(crate) fn compare_paths>( 30 | nominal: P, 31 | actual: P, 32 | nominal_entries: &[PathBuf], 33 | actual_entries: &[PathBuf], 34 | config: &DirectoryConfig, 35 | ) -> Result { 36 | let nominal_path = nominal.as_ref(); 37 | let actual_path = actual.as_ref(); 38 | 39 | let mut difference = Difference::new_for_file(nominal_path, actual_path); 40 | 41 | //remove root paths! 42 | let nominal_entries: Result, path::StripPrefixError> = nominal_entries 43 | .iter() 44 | .map(|path| path.strip_prefix(nominal_path)) 45 | .collect(); 46 | let nominal_entries = nominal_entries?; 47 | 48 | let actual_entries: Result, path::StripPrefixError> = actual_entries 49 | .iter() 50 | .map(|path| path.strip_prefix(actual_path)) 51 | .collect(); 52 | let actual_entries = actual_entries?; 53 | 54 | let mut is_the_same = true; 55 | if matches!(config.mode, Mode::Identical | Mode::MissingOnly) { 56 | nominal_entries.iter().for_each(|entry| { 57 | let detail = if let Some(f) = actual_entries.iter().find(|a| *a == entry) { 58 | (f.to_string_lossy().to_string(), false) 59 | } else { 60 | error!("{} doesn't exist in the actual folder", entry.display()); 61 | is_the_same = false; 62 | ("".to_owned(), true) 63 | }; 64 | 65 | difference.push_detail(DiffDetail::File { 66 | nominal: entry.to_string_lossy().to_string(), 67 | actual: detail.0, 68 | error: detail.1, 69 | }); 70 | }); 71 | } 72 | 73 | if matches!(config.mode, Mode::Identical) { 74 | actual_entries.iter().for_each(|entry| { 75 | if !nominal_entries.iter().any(|n| n == entry) { 76 | difference.push_detail(DiffDetail::File { 77 | nominal: "".to_owned(), 78 | actual: entry.to_string_lossy().to_string(), 79 | error: true, 80 | }); 81 | 82 | error!( 83 | "Additional entry {} found in the actual folder", 84 | entry.display() 85 | ); 86 | is_the_same = false; 87 | } 88 | }); 89 | } 90 | 91 | if !is_the_same { 92 | difference.error(); 93 | } 94 | 95 | Ok(difference) 96 | } 97 | 98 | #[cfg(test)] 99 | 100 | mod test { 101 | use super::*; 102 | 103 | #[test] 104 | fn test_compare_directories() { 105 | let nominal_dir = tempfile::tempdir().expect("Could not create nominal temp dir"); 106 | 107 | std::fs::create_dir_all(nominal_dir.path().join("dir/a/aa")) 108 | .expect("Could not create directory"); 109 | std::fs::create_dir_all(nominal_dir.path().join("dir/b")) 110 | .expect("Could not create directory"); 111 | std::fs::create_dir_all(nominal_dir.path().join("dir/c")) 112 | .expect("Could not create directory"); 113 | 114 | let actual_dir = tempfile::tempdir().expect("Could not create actual temp dir"); 115 | 116 | std::fs::create_dir_all(actual_dir.path().join("dir/a/aa")) 117 | .expect("Could not create directory"); 118 | std::fs::create_dir_all(actual_dir.path().join("dir/b")) 119 | .expect("Could not create directory"); 120 | std::fs::create_dir_all(actual_dir.path().join("dir/c")) 121 | .expect("Could not create directory"); 122 | 123 | let pattern_include = ["**/*/"]; 124 | let pattern_exclude: Vec = Vec::new(); 125 | 126 | let nominal_entries = crate::get_files(&nominal_dir, &pattern_include, &pattern_exclude) 127 | .expect("Could not get files"); 128 | let actual_entries = crate::get_files(&actual_dir, &pattern_include, &pattern_exclude) 129 | .expect("Could not get files"); 130 | 131 | let result = compare_paths( 132 | nominal_dir.path(), 133 | actual_dir.path(), 134 | &nominal_entries, 135 | &actual_entries, 136 | &DirectoryConfig { 137 | mode: Mode::Identical, 138 | }, 139 | ) 140 | .expect("Could not compare paths"); 141 | 142 | assert!(!result.is_error); 143 | 144 | std::fs::create_dir_all(actual_dir.path().join("dir/d")) 145 | .expect("Could not create directory"); 146 | 147 | let nominal_entries = crate::get_files(&nominal_dir, &pattern_include, &pattern_exclude) 148 | .expect("Could not create directory"); 149 | let actual_entries = crate::get_files(&actual_dir, &pattern_include, &pattern_exclude) 150 | .expect("Could not create directory"); 151 | 152 | let result = compare_paths( 153 | nominal_dir.path(), 154 | actual_dir.path(), 155 | &nominal_entries, 156 | &actual_entries, 157 | &DirectoryConfig { 158 | mode: Mode::Identical, 159 | }, 160 | ) 161 | .expect("Could not compare paths"); 162 | 163 | assert!(result.is_error); 164 | 165 | let result = compare_paths( 166 | nominal_dir.path(), 167 | actual_dir.path(), 168 | &nominal_entries, 169 | &actual_entries, 170 | &DirectoryConfig { 171 | mode: Mode::MissingOnly, 172 | }, 173 | ) 174 | .expect("Could not compare paths"); 175 | 176 | assert!(!result.is_error); 177 | 178 | std::fs::create_dir_all(nominal_dir.path().join("dir/d")) 179 | .expect("Could not create directory"); 180 | std::fs::create_dir_all(nominal_dir.path().join("dir/e")) 181 | .expect("Could not create directory"); 182 | 183 | let nominal_entries = crate::get_files(&nominal_dir, &pattern_include, &pattern_exclude) 184 | .expect("Could not create directory"); 185 | let actual_entries = crate::get_files(&actual_dir, &pattern_include, &pattern_exclude) 186 | .expect("Could not create directory"); 187 | 188 | let result = compare_paths( 189 | nominal_dir.path(), 190 | actual_dir.path(), 191 | &nominal_entries, 192 | &actual_entries, 193 | &DirectoryConfig { 194 | mode: Mode::Identical, 195 | }, 196 | ) 197 | .expect("Could not compare paths"); 198 | 199 | assert!(result.is_error); 200 | 201 | let result = compare_paths( 202 | nominal_dir.path(), 203 | actual_dir.path(), 204 | &nominal_entries, 205 | &actual_entries, 206 | &DirectoryConfig { 207 | mode: Mode::MissingOnly, 208 | }, 209 | ) 210 | .expect("Could not compare paths"); 211 | 212 | assert!(result.is_error); 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /src/properties.rs: -------------------------------------------------------------------------------- 1 | use crate::report::{get_relative_path, DiffDetail, Difference}; 2 | use crate::Error; 3 | use chrono::offset::Utc; 4 | use chrono::DateTime; 5 | use regex::Regex; 6 | use schemars::JsonSchema; 7 | use serde::Deserialize; 8 | use serde::Serialize; 9 | use std::path::Path; 10 | use std::time::SystemTime; 11 | use tracing::error; 12 | 13 | /// the configuration struct for file property comparison 14 | #[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)] 15 | pub struct PropertiesConfig { 16 | /// Compare the file size, difference must be smaller then given value 17 | file_size_tolerance_bytes: Option, 18 | 19 | /// Compare the modification date, difference must be smaller then the given value 20 | modification_date_tolerance_secs: Option, 21 | 22 | /// Fail if the name contains that regex 23 | forbid_name_regex: Option, 24 | } 25 | 26 | #[derive(Serialize, Debug, Clone)] 27 | pub enum MetaDataPropertyDiff { 28 | Size { nominal: u64, actual: u64 }, 29 | IllegalName, 30 | CreationDate { nominal: String, actual: String }, 31 | } 32 | 33 | fn regex_matches_any_path( 34 | nominal_path: &str, 35 | actual_path: &str, 36 | regex: &str, 37 | ) -> Result, Error> { 38 | let regex = Regex::new(regex)?; 39 | if regex.is_match(nominal_path) || regex.is_match(actual_path) { 40 | error!("One of the files ({nominal_path}, {actual_path}) matched the regex {regex}"); 41 | let mut result = Difference::new_for_file(nominal_path, actual_path); 42 | result.error(); 43 | result.push_detail(DiffDetail::Properties(MetaDataPropertyDiff::IllegalName)); 44 | result.is_error = true; 45 | return Ok(Some(result)); 46 | } 47 | Ok(None) 48 | } 49 | 50 | fn file_size_out_of_tolerance(nominal: &Path, actual: &Path, tolerance: u64) -> Difference { 51 | let mut result = Difference::new_for_file(nominal, actual); 52 | if let (Ok(nominal_meta), Ok(actual_meta)) = (nominal.metadata(), actual.metadata()) { 53 | let size_diff = 54 | (nominal_meta.len() as i128 - actual_meta.len() as i128).unsigned_abs() as u64; 55 | if size_diff > tolerance { 56 | error!("File size tolerance exceeded, diff is {size_diff}, tolerance was {tolerance}"); 57 | result.error(); 58 | } 59 | result.push_detail(DiffDetail::Properties(MetaDataPropertyDiff::Size { 60 | nominal: nominal_meta.len(), 61 | actual: actual_meta.len(), 62 | })); 63 | } else { 64 | let msg = format!( 65 | "Could not get file metadata for either: {} or {}", 66 | &nominal.to_string_lossy(), 67 | &actual.to_string_lossy() 68 | ); 69 | error!("{}", &msg); 70 | result.push_detail(DiffDetail::Error(msg)); 71 | result.is_error = true; 72 | } 73 | result 74 | } 75 | 76 | fn file_modification_time_out_of_tolerance( 77 | nominal: &Path, 78 | actual: &Path, 79 | tolerance: u64, 80 | ) -> Difference { 81 | let mut result = Difference::new_for_file(nominal, actual); 82 | if let (Ok(nominal_meta), Ok(actual_meta)) = (nominal.metadata(), actual.metadata()) { 83 | if let (Ok(mod_time_nom), Ok(mod_time_act)) = 84 | (nominal_meta.modified(), actual_meta.modified()) 85 | { 86 | let nominal_datetime: DateTime = mod_time_nom.into(); 87 | let actual_datetime: DateTime = mod_time_act.into(); 88 | result.push_detail(DiffDetail::Properties(MetaDataPropertyDiff::CreationDate { 89 | nominal: nominal_datetime.format("%Y-%m-%d %T").to_string(), 90 | actual: actual_datetime.format("%Y-%m-%d %T").to_string(), 91 | })); 92 | 93 | let now = SystemTime::now(); 94 | 95 | if let (Ok(nom_age), Ok(act_age)) = ( 96 | now.duration_since(mod_time_nom), 97 | now.duration_since(mod_time_act), 98 | ) { 99 | let time_diff = 100 | (nom_age.as_secs() as i128 - act_age.as_secs() as i128).unsigned_abs() as u64; 101 | if time_diff > tolerance { 102 | error!("Modification times too far off difference in timestamps {time_diff} s - tolerance {tolerance} s"); 103 | result.is_error = true; 104 | } 105 | } else { 106 | let msg = 107 | "Could not calculate duration between modification timestamps".to_string(); 108 | error!("{}", &msg); 109 | result.push_detail(DiffDetail::Error(msg)); 110 | result.is_error = true; 111 | } 112 | } else { 113 | let msg = "Could not read file modification timestamps".to_string(); 114 | error!("{}", &msg); 115 | result.push_detail(DiffDetail::Error(msg)); 116 | result.is_error = true; 117 | } 118 | } else { 119 | let msg = format!( 120 | "Could not get file metadata for either: {} or {}", 121 | &nominal.to_string_lossy(), 122 | &actual.to_string_lossy() 123 | ); 124 | error!("{}", &msg); 125 | result.push_detail(DiffDetail::Error(msg)); 126 | 127 | result.is_error = true; 128 | } 129 | result 130 | } 131 | 132 | pub(crate) fn compare_files>( 133 | nominal: P, 134 | actual: P, 135 | config: &PropertiesConfig, 136 | ) -> Result { 137 | let nominal = nominal.as_ref(); 138 | let actual = actual.as_ref(); 139 | let compared_file_name_full = nominal.to_string_lossy(); 140 | let actual_file_name_full = actual.to_string_lossy(); 141 | get_relative_path(actual, nominal) 142 | .to_string_lossy() 143 | .to_string(); 144 | 145 | let mut total_diff = Difference::new_for_file(nominal, actual); 146 | let result = if let Some(name_regex) = config.forbid_name_regex.as_deref() { 147 | regex_matches_any_path(&compared_file_name_full, &actual_file_name_full, name_regex)? 148 | } else { 149 | None 150 | }; 151 | result.map(|r| total_diff.join(r)); 152 | 153 | let result = config 154 | .file_size_tolerance_bytes 155 | .map(|tolerance| file_size_out_of_tolerance(nominal, actual, tolerance)); 156 | result.map(|r| total_diff.join(r)); 157 | 158 | let result = config 159 | .modification_date_tolerance_secs 160 | .map(|tolerance| file_modification_time_out_of_tolerance(nominal, actual, tolerance)); 161 | result.map(|r| total_diff.join(r)); 162 | 163 | Ok(total_diff) 164 | } 165 | 166 | #[cfg(test)] 167 | mod tests { 168 | use super::*; 169 | 170 | #[test] 171 | fn name_regex_works() { 172 | let file_name_mock = "/dev/urandom"; 173 | let file_name_cap_mock = "/proc/cpuInfo"; 174 | let regex_no_capitals = r"[A-Z]"; 175 | let regex_no_spaces = r"[\s]"; 176 | assert!( 177 | regex_matches_any_path(file_name_mock, file_name_cap_mock, regex_no_capitals) 178 | .unwrap() 179 | .unwrap() 180 | .is_error 181 | ); 182 | assert!( 183 | regex_matches_any_path(file_name_mock, file_name_cap_mock, regex_no_spaces) 184 | .unwrap() 185 | .is_none() 186 | ); 187 | } 188 | 189 | #[test] 190 | fn file_size() { 191 | let toml_file = "Cargo.toml"; 192 | let lock_file = "Cargo.lock"; 193 | assert!( 194 | !file_size_out_of_tolerance(Path::new(toml_file), Path::new(toml_file), 0).is_error 195 | ); 196 | assert!(file_size_out_of_tolerance(Path::new(toml_file), Path::new(lock_file), 0).is_error); 197 | } 198 | #[test] 199 | fn modification_timestamps() { 200 | let toml_file = "Cargo.toml"; 201 | let lock_file = "Cargo.lock"; 202 | assert!( 203 | !file_modification_time_out_of_tolerance(Path::new(toml_file), Path::new(toml_file), 0) 204 | .is_error 205 | ); 206 | assert!( 207 | file_modification_time_out_of_tolerance(Path::new(toml_file), Path::new(lock_file), 0) 208 | .is_error 209 | ); 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /src/csv/tokenizer/guess_format.rs: -------------------------------------------------------------------------------- 1 | use crate::csv::{Delimiters, Error}; 2 | use itertools::Itertools; 3 | use regex::Regex; 4 | use std::collections::HashMap; 5 | use std::io::{BufRead, BufReader, Read, Seek}; 6 | use tracing::{debug, warn}; 7 | 8 | fn guess_format_from_line( 9 | line: &str, 10 | field_separator_hint: Option, 11 | ) -> Result<(Option, Option), Error> { 12 | let mut field_separator = field_separator_hint; 13 | 14 | if field_separator.is_none() { 15 | if line.find(';').is_some() { 16 | field_separator = Some(';'); 17 | } else { 18 | let field_sep_regex = Regex::new(r"\w([,|])[\W\w]")?; 19 | let capture = field_sep_regex.captures_iter(line).next(); 20 | if let Some(cap) = capture { 21 | field_separator = Some(cap[1].chars().next().ok_or_else(|| { 22 | Error::InvalidAccess(format!( 23 | "Could not capture field separator for guessing from '{line}'" 24 | )) 25 | })?); 26 | } 27 | } 28 | } 29 | 30 | let decimal_separator_candidates = [',', '.']; 31 | let context_acceptable_candidates = if let Some(field_separator) = field_separator { 32 | decimal_separator_candidates 33 | .into_iter() 34 | .filter(|c| *c != field_separator) 35 | .join("") 36 | } else { 37 | decimal_separator_candidates.into_iter().join("") 38 | }; 39 | 40 | let decimal_separator_regex_string = format!(r"\d([{context_acceptable_candidates}])\d"); 41 | debug!( 42 | "Regex for decimal sep: '{}'", 43 | decimal_separator_regex_string.as_str() 44 | ); 45 | let decimal_separator_regex = Regex::new(decimal_separator_regex_string.as_str())?; 46 | let mut separators: HashMap = HashMap::new(); 47 | 48 | for capture in decimal_separator_regex.captures_iter(line) { 49 | let sep = capture[1].chars().next().ok_or_else(|| { 50 | Error::InvalidAccess(format!( 51 | "Could not capture decimal separator for guessing from '{line}'" 52 | )) 53 | })?; 54 | if let Some(entry) = separators.get_mut(&sep) { 55 | *entry += 1; 56 | } else { 57 | separators.insert(sep, 1); 58 | } 59 | } 60 | 61 | debug!( 62 | "Found separator candidates with occurrence count: {:?}", 63 | separators 64 | ); 65 | 66 | let decimal_separator = separators 67 | .iter() 68 | .sorted_by(|a, b| b.1.cmp(a.1)) 69 | .map(|s| s.0.to_owned()) 70 | .next(); 71 | 72 | Ok((field_separator, decimal_separator)) 73 | } 74 | 75 | pub(crate) fn guess_format_from_reader( 76 | mut input: &mut R, 77 | ) -> Result { 78 | let mut format = (None, None); 79 | 80 | let bufreader = BufReader::new(&mut input); 81 | debug!("Guessing format from reader..."); 82 | for line in bufreader.lines().map_while(Result::ok) { 83 | debug!("Guessing format from line: '{}'", line.as_str()); 84 | format = guess_format_from_line(line.as_str(), format.0)?; 85 | debug!("Current format: {:?}", format); 86 | if format.0.is_some() && format.1.is_some() { 87 | break; 88 | } 89 | } 90 | 91 | input.rewind()?; 92 | 93 | if format.0.is_none() { 94 | warn!("Could not guess field delimiter, setting to default"); 95 | format.0 = Delimiters::default().field_delimiter; 96 | } 97 | 98 | let delim = Delimiters { 99 | field_delimiter: format.0, 100 | decimal_separator: format.1, 101 | }; 102 | debug!( 103 | "Inferring of csv delimiters resulted in decimal separators: '{:?}', field delimiter: '{:?}'", 104 | delim.decimal_separator, delim.field_delimiter 105 | ); 106 | Ok(delim) 107 | } 108 | 109 | #[cfg(test)] 110 | mod format_guessing_tests { 111 | use super::*; 112 | use std::fs::File; 113 | #[test] 114 | fn format_detection_basics() { 115 | let format = guess_format_from_line( 116 | "-0.969654597744788,-0.215275534510198,0.115869999295192,7.04555232210696", 117 | None, 118 | ) 119 | .unwrap(); 120 | assert_eq!(format, (Some(','), Some('.'))); 121 | 122 | let format = guess_format_from_line( 123 | "-0.969654597744788;-0.215275534510198;0.115869999295192;7.04555232210696", 124 | None, 125 | ) 126 | .unwrap(); 127 | assert_eq!(format, (Some(';'), Some('.'))); 128 | 129 | let format = guess_format_from_line( 130 | "-0.969654597744788,-0.215275534510198,0.115869999295192,7.04555232210696", 131 | None, 132 | ) 133 | .unwrap(); 134 | assert_eq!(format, (Some(','), Some('.'))); 135 | } 136 | 137 | #[test] 138 | fn format_detection_from_file() { 139 | let format = 140 | guess_format_from_reader(&mut File::open("tests/csv/data/Annotations.csv").unwrap()) 141 | .unwrap(); 142 | assert_eq!( 143 | format, 144 | Delimiters { 145 | field_delimiter: Some(','), 146 | decimal_separator: Some('.') 147 | } 148 | ); 149 | } 150 | 151 | #[test] 152 | fn format_detection_from_file_metrology_special() { 153 | let format = guess_format_from_reader( 154 | &mut File::open("tests/csv/data/Multi_Apply_Rotation.csv").unwrap(), 155 | ) 156 | .unwrap(); 157 | assert_eq!( 158 | format, 159 | Delimiters { 160 | field_delimiter: Some(','), 161 | decimal_separator: Some('.') 162 | } 163 | ); 164 | } 165 | 166 | #[test] 167 | fn format_detection_from_file_metrology_other_special() { 168 | let format = guess_format_from_reader( 169 | &mut File::open("tests/csv/data/CM_quality_threshold.csv").unwrap(), 170 | ) 171 | .unwrap(); 172 | assert_eq!( 173 | format, 174 | Delimiters { 175 | field_delimiter: Some(','), 176 | decimal_separator: None 177 | } 178 | ); 179 | } 180 | 181 | #[test] 182 | fn format_detection_from_file_analysis_pia_table() { 183 | let format = guess_format_from_reader( 184 | &mut File::open("tests/csv/data/easy_pore_export_annoration_table_result.csv").unwrap(), 185 | ) 186 | .unwrap(); 187 | assert_eq!( 188 | format, 189 | Delimiters { 190 | field_delimiter: Some(';'), 191 | decimal_separator: Some(',') 192 | } 193 | ); 194 | } 195 | 196 | #[test] 197 | fn format_detection_from_file_no_field_sep() { 198 | let format = 199 | guess_format_from_reader(&mut File::open("tests/csv/data/no_field_sep.csv").unwrap()) 200 | .unwrap(); 201 | assert_eq!( 202 | format, 203 | Delimiters { 204 | field_delimiter: None, 205 | decimal_separator: Some('.') 206 | } 207 | ); 208 | } 209 | #[test] 210 | fn format_detection_from_file_semicolon_formatting() { 211 | let format = guess_format_from_reader( 212 | &mut File::open( 213 | "tests/integ/data/display_of_status_message_in_cm_tables/expected/Volume1.csv", 214 | ) 215 | .unwrap(), 216 | ) 217 | .unwrap(); 218 | assert_eq!( 219 | format, 220 | Delimiters { 221 | field_delimiter: Some(';'), 222 | decimal_separator: Some(',') 223 | } 224 | ); 225 | } 226 | #[test] 227 | fn format_detection_from_file_semicolon_separators() { 228 | let format = 229 | guess_format_from_reader(&mut File::open("tests/csv/data/Components.csv").unwrap()) 230 | .unwrap(); 231 | assert_eq!( 232 | format, 233 | Delimiters { 234 | field_delimiter: Some(';'), 235 | decimal_separator: Some(',') 236 | } 237 | ); 238 | } 239 | 240 | #[test] 241 | fn format_detection_from_file_dot_comma_formatting() { 242 | let format = guess_format_from_reader( 243 | &mut File::open( 244 | "tests/integ/data/display_of_status_message_in_cm_tables/actual/Volume1.csv", 245 | ) 246 | .unwrap(), 247 | ) 248 | .unwrap(); 249 | assert_eq!( 250 | format, 251 | Delimiters { 252 | field_delimiter: Some(','), 253 | decimal_separator: Some('.') 254 | } 255 | ); 256 | } 257 | } 258 | -------------------------------------------------------------------------------- /src/image.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | 3 | use image::{DynamicImage, Rgb}; 4 | use image_compare::{Algorithm, Metric, Similarity}; 5 | use schemars_derive::JsonSchema; 6 | use serde::{Deserialize, Serialize}; 7 | use thiserror::Error; 8 | use tracing::error; 9 | 10 | use crate::report::DiffDetail; 11 | use crate::{get_file_name, report}; 12 | 13 | #[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)] 14 | pub enum RGBACompareMode { 15 | /// full RGBA comparison - probably not intuitive, rarely what you want outside of video processing 16 | /// Will do MSSIM on luma, then RMS on U and V and alpha channels. 17 | /// The calculation of the score is then pixel-wise the minimum of each pixels similarity. 18 | /// To account for perceived indifference in lower alpha regions, this down-weights the difference linearly with mean alpha channel. 19 | Hybrid, 20 | /// pre-blend the background in RGBA with this color, use the background RGB values you would assume the pictures to be seen on - usually either black or white 21 | HybridBlended { r: u8, b: u8, g: u8 }, 22 | } 23 | 24 | impl Default for RGBACompareMode { 25 | fn default() -> Self { 26 | Self::HybridBlended { r: 0, b: 0, g: 0 } 27 | } 28 | } 29 | 30 | #[allow(clippy::upper_case_acronyms)] 31 | #[derive(JsonSchema, Deserialize, Serialize, Debug, Clone, Default)] 32 | pub enum RGBCompareMode { 33 | ///Comparing rgb images using structure. RGB structure similarity is performed by doing a channel split and taking the maximum deviation (minimum similarity) for the result. The image contains the complete deviations. Algorithm: RMS 34 | RMS, 35 | ///Comparing rgb images using structure. RGB structure similarity is performed by doing a channel split and taking the maximum deviation (minimum similarity) for the result. The image contains the complete deviations. Algorithm: MSSIM 36 | MSSIM, 37 | ///Comparing structure via MSSIM on Y channel, comparing color-diff-vectors on U and V summing the squares Please mind that the RGBSimilarity-Image does not contain plain RGB here. Probably what you want. 38 | #[default] 39 | Hybrid, 40 | } 41 | 42 | #[allow(clippy::upper_case_acronyms)] 43 | #[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)] 44 | /// The distance algorithm to use for grayscale comparison, see 45 | /// https://github.com/ChrisRega/image-compare for equations 46 | pub enum GrayStructureAlgorithm { 47 | /// SSIM with 8x8 pixel windows and averaging over the result 48 | MSSIM, 49 | /// Classic RMS distance 50 | RMS, 51 | } 52 | 53 | /// See https://github.com/ChrisRega/image-compare for equations 54 | /// Distance metrics for histograms for grayscale comparison 55 | #[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)] 56 | pub enum GrayHistogramCompareMetric { 57 | /// Correlation $d(H_1,H_2) = \frac{\sum_I (H_1(I) - \bar{H_1}) (H_2(I) - \bar{H_2})}{\sqrt{\sum_I(H_1(I) - \bar{H_1})^2 \sum_I(H_2(I) - \bar{H_2})^2}}$ 58 | Correlation, 59 | /// Chi-Square $d(H_1,H_2) = \sum _I \frac{\left(H_1(I)-H_2(I)\right)^2}{H_1(I)}$ 60 | ChiSquare, 61 | /// Intersection $d(H_1,H_2) = \sum _I \min (H_1(I), H_2(I))$ 62 | Intersection, 63 | /// Hellinger distance $d(H_1,H_2) = \sqrt{1 - \frac{1}{\sqrt{\int{H_1} \int{H_2}}} \sum_I \sqrt{H_1(I) \cdot H_2(I)}}$ 64 | Hellinger, 65 | } 66 | 67 | #[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)] 68 | pub enum GrayCompareMode { 69 | /// Compare gray values pixel structure 70 | Structure(GrayStructureAlgorithm), 71 | /// Compare gray values by histogram 72 | Histogram(GrayHistogramCompareMetric), 73 | } 74 | 75 | #[allow(clippy::upper_case_acronyms)] 76 | #[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)] 77 | pub enum CompareMode { 78 | /// Compare images as RGB 79 | RGB(RGBCompareMode), 80 | /// Compare images as RGBA 81 | RGBA(RGBACompareMode), 82 | /// Compare images as luminance / grayscale 83 | Gray(GrayCompareMode), 84 | } 85 | 86 | #[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)] 87 | /// Image comparison config options 88 | pub struct ImageCompareConfig { 89 | /// Threshold for image comparison < 0.5 is very dissimilar, 1.0 is identical 90 | pub threshold: f64, 91 | #[serde(flatten)] 92 | /// How to compare the two images 93 | pub mode: CompareMode, 94 | } 95 | 96 | #[derive(Debug, Error)] 97 | pub enum Error { 98 | #[error("Error loading image {0}")] 99 | ImageDecoding(#[from] image::ImageError), 100 | #[error("Problem creating hash report {0}")] 101 | Reporting(#[from] report::Error), 102 | #[error("Image comparison algorithm failed {0}")] 103 | ImageComparison(#[from] image_compare::CompareError), 104 | #[error("Problem processing file name {0}")] 105 | FileNameParsing(String), 106 | } 107 | 108 | struct ComparisonResult { 109 | score: f64, 110 | image: Option, 111 | } 112 | 113 | impl From for ComparisonResult { 114 | fn from(value: Similarity) -> Self { 115 | Self { 116 | image: Some(value.image.to_color_map()), 117 | score: value.score, 118 | } 119 | } 120 | } 121 | 122 | pub fn compare_paths>( 123 | nominal_path: P, 124 | actual_path: P, 125 | config: &ImageCompareConfig, 126 | ) -> Result { 127 | let nominal = image::open(nominal_path.as_ref())?; 128 | let actual = image::open(actual_path.as_ref())?; 129 | let result: ComparisonResult = match &config.mode { 130 | CompareMode::RGBA(c) => { 131 | let nominal = nominal.into_rgba8(); 132 | let actual = actual.into_rgba8(); 133 | match c { 134 | RGBACompareMode::Hybrid => { 135 | image_compare::rgba_hybrid_compare(&nominal, &actual)?.into() 136 | } 137 | RGBACompareMode::HybridBlended { r, g, b } => { 138 | image_compare::rgba_blended_hybrid_compare( 139 | (&nominal).into(), 140 | (&actual).into(), 141 | Rgb([*r, *g, *b]), 142 | )? 143 | .into() 144 | } 145 | } 146 | } 147 | CompareMode::RGB(c) => { 148 | let nominal = nominal.into_rgb8(); 149 | let actual = actual.into_rgb8(); 150 | match c { 151 | RGBCompareMode::RMS => image_compare::rgb_similarity_structure( 152 | &Algorithm::RootMeanSquared, 153 | &nominal, 154 | &actual, 155 | )? 156 | .into(), 157 | RGBCompareMode::MSSIM => image_compare::rgb_similarity_structure( 158 | &Algorithm::MSSIMSimple, 159 | &nominal, 160 | &actual, 161 | )? 162 | .into(), 163 | RGBCompareMode::Hybrid => { 164 | image_compare::rgb_hybrid_compare(&nominal, &actual)?.into() 165 | } 166 | } 167 | } 168 | CompareMode::Gray(c) => { 169 | let nominal = nominal.into_luma8(); 170 | let actual = actual.into_luma8(); 171 | match c { 172 | GrayCompareMode::Structure(c) => match c { 173 | GrayStructureAlgorithm::MSSIM => image_compare::gray_similarity_structure( 174 | &Algorithm::MSSIMSimple, 175 | &nominal, 176 | &actual, 177 | )? 178 | .into(), 179 | GrayStructureAlgorithm::RMS => image_compare::gray_similarity_structure( 180 | &Algorithm::RootMeanSquared, 181 | &nominal, 182 | &actual, 183 | )? 184 | .into(), 185 | }, 186 | GrayCompareMode::Histogram(c) => { 187 | let metric = match c { 188 | GrayHistogramCompareMetric::Correlation => Metric::Correlation, 189 | GrayHistogramCompareMetric::ChiSquare => Metric::ChiSquare, 190 | GrayHistogramCompareMetric::Intersection => Metric::Intersection, 191 | GrayHistogramCompareMetric::Hellinger => Metric::Hellinger, 192 | }; 193 | let score = 194 | image_compare::gray_similarity_histogram(metric, &nominal, &actual)?; 195 | ComparisonResult { score, image: None } 196 | } 197 | } 198 | } 199 | }; 200 | 201 | let mut result_diff = report::Difference::new_for_file(&nominal_path, &actual_path); 202 | if result.score < config.threshold { 203 | let out_path_set = if let Some(i) = result.image { 204 | let nominal_file_name = 205 | get_file_name(nominal_path.as_ref()).ok_or(Error::FileNameParsing(format!( 206 | "Could not extract filename from path {:?}", 207 | nominal_path.as_ref() 208 | )))?; 209 | let out_path = (nominal_file_name + "diff_image.png").to_string(); 210 | i.save(&out_path)?; 211 | Some(out_path) 212 | } else { 213 | None 214 | }; 215 | 216 | let error_message = format!( 217 | "Diff for image {} was not met, expected {}, found {}", 218 | nominal_path.as_ref().to_string_lossy(), 219 | config.threshold, 220 | result.score 221 | ); 222 | error!("{}", &error_message); 223 | 224 | result_diff.push_detail(DiffDetail::Image { 225 | diff_image: out_path_set, 226 | score: result.score, 227 | }); 228 | result_diff.error(); 229 | } 230 | Ok(result_diff) 231 | } 232 | 233 | #[cfg(test)] 234 | mod test { 235 | use super::*; 236 | 237 | #[test] 238 | fn identity() { 239 | let result = compare_paths( 240 | "tests/integ/data/images/actual/SaveImage_100DPI_default_size.jpg", 241 | "tests/integ/data/images/actual/SaveImage_100DPI_default_size.jpg", 242 | &ImageCompareConfig { 243 | threshold: 1.0, 244 | mode: CompareMode::RGB(RGBCompareMode::Hybrid), 245 | }, 246 | ) 247 | .unwrap(); 248 | assert!(!result.is_error); 249 | } 250 | 251 | #[test] 252 | fn pin_diff_image() { 253 | let result = compare_paths( 254 | "tests/integ/data/images/expected/SaveImage_100DPI_default_size.jpg", 255 | "tests/integ/data/images/actual/SaveImage_100DPI_default_size.jpg", 256 | &ImageCompareConfig { 257 | threshold: 1.0, 258 | mode: CompareMode::RGBA(RGBACompareMode::Hybrid), 259 | }, 260 | ) 261 | .unwrap(); 262 | assert!(result.is_error); 263 | if let DiffDetail::Image { 264 | score: _, 265 | diff_image, 266 | } = result.detail.first().unwrap() 267 | { 268 | let img = image::open(diff_image.as_ref().unwrap()) 269 | .unwrap() 270 | .into_rgba8(); 271 | let nom = image::open("tests/integ/data/images/diff_100_DPI.png") 272 | .unwrap() 273 | .into_rgba8(); 274 | let diff_result = image_compare::rgba_hybrid_compare(&img, &nom) 275 | .expect("Wrong dimensions of diff images!"); 276 | assert_eq!(diff_result.score, 0.9879023078642883); 277 | } else { 278 | unreachable!(); 279 | } 280 | } 281 | } 282 | -------------------------------------------------------------------------------- /tests/csv/data/DeviationHistogram.csv: -------------------------------------------------------------------------------- 1 | Deviation [mm],Surface [mm²] 2 | -0.6,2.98287e-05 3 | -0.597656,0.000197238 4 | -0.595313,0.00038192 5 | -0.592969,0.000566594 6 | -0.590625,0.000751271 7 | -0.588281,0.000935978 8 | -0.585938,0.00112063 9 | -0.583594,0.00130531 10 | -0.58125,0.00149003 11 | -0.578906,0.00167467 12 | -0.576563,0.0018594 13 | -0.574219,0.00204398 14 | -0.571875,0.00222882 15 | -0.569531,0.00241333 16 | -0.567188,0.00260286 17 | -0.564844,0.00280005 18 | -0.5625,0.0029971 19 | -0.560156,0.00319449 20 | -0.557813,0.00339167 21 | -0.555469,0.00358901 22 | -0.553125,0.00378616 23 | -0.550781,0.00398353 24 | -0.548438,0.00418067 25 | -0.546094,0.00437781 26 | -0.54375,0.0045754 27 | -0.541406,0.00497736 28 | -0.539062,0.00562869 29 | -0.536719,0.00628644 30 | -0.534375,0.00694376 31 | -0.532031,0.00760131 32 | -0.529688,0.00825931 33 | -0.527344,0.00891594 34 | -0.525,0.00957441 35 | -0.522656,0.0102315 36 | -0.520313,0.0108888 37 | -0.517969,0.0115471 38 | -0.515625,0.0122039 39 | -0.513281,0.0128619 40 | -0.510938,0.0135191 41 | -0.508594,0.021742 42 | -0.50625,0.0979612 43 | -0.503906,0.226019 44 | -0.501563,0.283784 45 | -0.499219,0.305244 46 | -0.496875,0.345156 47 | -0.494531,0.343327 48 | -0.492188,0.370366 49 | -0.489844,0.391152 50 | -0.4875,0.374938 51 | -0.485156,0.390856 52 | -0.482813,0.441054 53 | -0.480469,0.473784 54 | -0.478125,0.487892 55 | -0.475781,0.533334 56 | -0.473438,0.607911 57 | -0.471094,0.569942 58 | -0.46875,0.546658 59 | -0.466406,0.534465 60 | -0.464063,0.521933 61 | -0.461719,0.540654 62 | -0.459375,0.550342 63 | -0.457031,0.544246 64 | -0.454688,0.539219 65 | -0.452344,0.565432 66 | -0.45,0.584493 67 | -0.447656,0.594559 68 | -0.445312,0.762362 69 | -0.442969,0.85163 70 | -0.440625,0.804051 71 | -0.438281,0.730209 72 | -0.435938,0.623656 73 | -0.433594,0.561942 74 | -0.43125,0.554678 75 | -0.428906,0.552584 76 | -0.426563,0.544543 77 | -0.424219,0.541332 78 | -0.421875,0.539419 79 | -0.419531,0.535715 80 | -0.417188,0.528287 81 | -0.414844,0.510097 82 | -0.4125,0.493129 83 | -0.410156,0.480034 84 | -0.407813,0.463376 85 | -0.405469,0.448299 86 | -0.403125,0.43594 87 | -0.400781,0.426014 88 | -0.398438,0.418224 89 | -0.396094,0.412027 90 | -0.39375,0.404904 91 | -0.391406,0.399111 92 | -0.389063,0.395023 93 | -0.386719,0.394628 94 | -0.384375,0.401207 95 | -0.382031,0.403121 96 | -0.379688,0.405621 97 | -0.377344,0.411265 98 | -0.375,0.420196 99 | -0.372656,0.43235 100 | -0.370313,0.4444 101 | -0.367969,0.453988 102 | -0.365625,0.464616 103 | -0.363281,0.481407 104 | -0.360938,0.497669 105 | -0.358594,0.520736 106 | -0.35625,0.540397 107 | -0.353906,0.564052 108 | -0.351562,0.590408 109 | -0.349219,0.611136 110 | -0.346875,0.629912 111 | -0.344531,0.644368 112 | -0.342188,0.656525 113 | -0.339844,0.673917 114 | -0.3375,0.6923 115 | -0.335156,0.712214 116 | -0.332813,0.738892 117 | -0.330469,0.784918 118 | -0.328125,0.813485 119 | -0.325781,0.845362 120 | -0.323438,0.891095 121 | -0.321094,0.926883 122 | -0.31875,0.962925 123 | -0.316406,1.01895 124 | -0.314063,1.08486 125 | -0.311719,1.14281 126 | -0.309375,1.16594 127 | -0.307031,1.18784 128 | -0.304688,1.21217 129 | -0.302344,1.26247 130 | -0.3,1.34998 131 | -0.297656,1.35485 132 | -0.295313,1.40214 133 | -0.292969,1.4822 134 | -0.290625,1.56808 135 | -0.288281,1.67817 136 | -0.285938,1.78134 137 | -0.283594,1.89517 138 | -0.28125,2.01535 139 | -0.278906,2.16215 140 | -0.276563,2.5158 141 | -0.274219,3.01218 142 | -0.271875,3.88853 143 | -0.269531,4.50542 144 | -0.267188,4.56822 145 | -0.264844,5.67477 146 | -0.2625,6.92868 147 | -0.260156,8.89476 148 | -0.257812,11.2662 149 | -0.255469,10.9973 150 | -0.253125,11.0213 151 | -0.250781,11.7452 152 | -0.248438,12.5683 153 | -0.246094,14.5304 154 | -0.24375,16.8417 155 | -0.241406,19.2301 156 | -0.239063,22.9344 157 | -0.236719,24.884 158 | -0.234375,25.9041 159 | -0.232031,25.7823 160 | -0.229688,24.1785 161 | -0.227344,21.8301 162 | -0.225,20.4353 163 | -0.222656,18.2923 164 | -0.220313,15.9943 165 | -0.217969,14.9076 166 | -0.215625,14.1258 167 | -0.213281,13.8547 168 | -0.210938,14.0053 169 | -0.208594,14.3485 170 | -0.20625,14.9161 171 | -0.203906,15.4934 172 | -0.201562,16.0577 173 | -0.199219,16.8416 174 | -0.196875,17.746 175 | -0.194531,19.4365 176 | -0.192188,21.7911 177 | -0.189844,24.7822 178 | -0.1875,28.3985 179 | -0.185156,32.6523 180 | -0.182813,37.4158 181 | -0.180469,39.436 182 | -0.178125,40.4951 183 | -0.175781,40.8298 184 | -0.173438,40.2616 185 | -0.171094,41.4706 186 | -0.16875,43.8043 187 | -0.166406,48.6796 188 | -0.164062,52.488 189 | -0.161719,60.3067 190 | -0.159375,67.6012 191 | -0.157031,75.4188 192 | -0.154687,85.221 193 | -0.152344,95.1454 194 | -0.15,108.193 195 | -0.147656,121.607 196 | -0.145313,138.552 197 | -0.142969,157.244 198 | -0.140625,178.366 199 | -0.138281,199.006 200 | -0.135938,214.883 201 | -0.133594,225.857 202 | -0.13125,244.21 203 | -0.128906,259.522 204 | -0.126563,289.639 205 | -0.124219,328.315 206 | -0.121875,362.679 207 | -0.119531,397.494 208 | -0.117188,440.275 209 | -0.114844,496.999 210 | -0.1125,553.358 211 | -0.110156,610.158 212 | -0.107812,669.913 213 | -0.105469,711.641 214 | -0.103125,758.797 215 | -0.100781,808.154 216 | -0.0984375,891.278 217 | -0.0960938,976.417 218 | -0.09375,1077.74 219 | -0.0914062,1169.26 220 | -0.0890625,1259.47 221 | -0.0867187,1332.88 222 | -0.084375,1364.03 223 | -0.0820312,1333.15 224 | -0.0796875,1276.47 225 | -0.0773438,1213.15 226 | -0.075,1163.26 227 | -0.0726563,1114.48 228 | -0.0703125,1090.39 229 | -0.0679687,1086.96 230 | -0.065625,1096.5 231 | -0.0632812,1136.48 232 | -0.0609375,1194.24 233 | -0.0585938,1288.38 234 | -0.05625,1382.61 235 | -0.0539063,1474.37 236 | -0.0515625,1513.37 237 | -0.0492188,1505.08 238 | -0.046875,1448.95 239 | -0.0445312,1393.91 240 | -0.0421875,1304.1 241 | -0.0398437,1216.91 242 | -0.0375,1126.85 243 | -0.0351562,1037.66 244 | -0.0328125,938.972 245 | -0.0304688,842.832 246 | -0.028125,759.245 247 | -0.0257813,699.029 248 | -0.0234375,665.831 249 | -0.0210937,657.703 250 | -0.01875,663.328 251 | -0.0164062,693.228 252 | -0.0140625,711.601 253 | -0.0117188,723.543 254 | -0.00937498,720.624 255 | -0.00703126,699.44 256 | -0.00468749,679.729 257 | -0.00234377,654.56 258 | 0,639.303 259 | 0.00234377,655.427 260 | 0.00468749,676.04 261 | 0.00703126,705.868 262 | 0.00937498,748.042 263 | 0.0117188,812.488 264 | 0.0140625,864.877 265 | 0.0164062,911.603 266 | 0.01875,960.975 267 | 0.0210937,992.786 268 | 0.0234375,1000.8 269 | 0.0257813,981.773 270 | 0.028125,931.843 271 | 0.0304688,862.485 272 | 0.0328125,783.891 273 | 0.0351562,713.606 274 | 0.0375,652.863 275 | 0.0398437,600.229 276 | 0.0421875,552.889 277 | 0.0445312,514.36 278 | 0.046875,480.352 279 | 0.0492188,449.506 280 | 0.0515625,424.929 281 | 0.0539063,393.91 282 | 0.05625,368.892 283 | 0.0585938,350.906 284 | 0.0609375,325.901 285 | 0.0632812,308.577 286 | 0.065625,287.256 287 | 0.0679687,256.804 288 | 0.0703125,219.251 289 | 0.0726563,187.177 290 | 0.075,161.527 291 | 0.0773438,149.918 292 | 0.0796875,146.647 293 | 0.0820312,144.546 294 | 0.084375,141.077 295 | 0.0867187,135.229 296 | 0.0890625,118.676 297 | 0.0914062,101.711 298 | 0.09375,84.8608 299 | 0.0960938,75.1614 300 | 0.0984375,69.1875 301 | 0.100781,59.0727 302 | 0.103125,40.8601 303 | 0.105469,25.3316 304 | 0.107813,16.134 305 | 0.110156,11.0083 306 | 0.1125,7.7022 307 | 0.114844,4.81511 308 | 0.117188,2.93038 309 | 0.119531,2.2669 310 | 0.121875,1.84511 311 | 0.124219,1.69962 312 | 0.126562,1.66358 313 | 0.128906,1.47109 314 | 0.13125,1.27072 315 | 0.133594,1.20311 316 | 0.135938,1.06163 317 | 0.138281,0.955001 318 | 0.140625,0.864637 319 | 0.142969,0.850746 320 | 0.145312,0.807083 321 | 0.147656,0.774698 322 | 0.15,0.756973 323 | 0.152344,0.719494 324 | 0.154688,0.66364 325 | 0.157031,0.631981 326 | 0.159375,0.60928 327 | 0.161719,0.588666 328 | 0.164062,0.5677 329 | 0.166406,0.550603 330 | 0.16875,0.537356 331 | 0.171094,0.527421 332 | 0.173438,0.525581 333 | 0.175781,0.511169 334 | 0.178125,0.491182 335 | 0.180469,0.474581 336 | 0.182813,0.470486 337 | 0.185156,0.457949 338 | 0.1875,0.44142 339 | 0.189844,0.425765 340 | 0.192187,0.415711 341 | 0.194531,0.411068 342 | 0.196875,0.407548 343 | 0.199219,0.404015 344 | 0.201563,0.401023 345 | 0.203906,0.398012 346 | 0.20625,0.394963 347 | 0.208594,0.391809 348 | 0.210938,0.388738 349 | 0.213281,0.38565 350 | 0.215625,0.381978 351 | 0.217969,0.378512 352 | 0.220313,0.375193 353 | 0.222656,0.371509 354 | 0.225,0.367862 355 | 0.227344,0.36521 356 | 0.229688,0.36282 357 | 0.232031,0.360546 358 | 0.234375,0.358451 359 | 0.236719,0.356484 360 | 0.239062,0.354275 361 | 0.241406,0.352045 362 | 0.24375,0.349844 363 | 0.246094,0.347861 364 | 0.248438,0.345825 365 | 0.250781,0.343863 366 | 0.253125,0.341873 367 | 0.255469,0.339872 368 | 0.257812,0.337686 369 | 0.260156,0.33565 370 | 0.2625,0.333442 371 | 0.264844,0.33124 372 | 0.267188,0.329882 373 | 0.269531,0.32868 374 | 0.271875,0.327488 375 | 0.274219,0.326319 376 | 0.276563,0.325142 377 | 0.278906,0.323918 378 | 0.28125,0.322675 379 | 0.283594,0.321432 380 | 0.285937,0.320002 381 | 0.288281,0.318855 382 | 0.290625,0.317812 383 | 0.292969,0.316835 384 | 0.295313,0.315851 385 | 0.297656,0.314898 386 | 0.3,0.3139 387 | 0.302344,0.312838 388 | 0.304688,0.311867 389 | 0.307031,0.310897 390 | 0.309375,0.31001 391 | 0.311719,0.309085 392 | 0.314063,0.30815 393 | 0.316406,0.307223 394 | 0.31875,0.306323 395 | 0.321094,0.305385 396 | 0.323438,0.304349 397 | 0.325781,0.303313 398 | 0.328125,0.302513 399 | 0.330469,0.301661 400 | 0.332812,0.300852 401 | 0.335156,0.300093 402 | 0.3375,0.299349 403 | 0.339844,0.29864 404 | 0.342188,0.297917 405 | 0.344531,0.297198 406 | 0.346875,0.296429 407 | 0.349219,0.295656 408 | 0.351562,0.295004 409 | 0.353906,0.294282 410 | 0.35625,0.293514 411 | 0.358594,0.292854 412 | 0.360938,0.292198 413 | 0.363281,0.291553 414 | 0.365625,0.290871 415 | 0.367969,0.290206 416 | 0.370313,0.289626 417 | 0.372656,0.289083 418 | 0.375,0.288527 419 | 0.377344,0.287986 420 | 0.379687,0.287446 421 | 0.382031,0.286902 422 | 0.384375,0.286339 423 | 0.386719,0.285829 424 | 0.389063,0.285289 425 | 0.391406,0.284807 426 | 0.39375,0.284297 427 | 0.396094,0.283778 428 | 0.398438,0.283281 429 | 0.400781,0.282782 430 | 0.403125,0.282257 431 | 0.405469,0.28173 432 | 0.407812,0.281221 433 | 0.410156,0.280692 434 | 0.4125,0.280162 435 | 0.414844,0.279632 436 | 0.417188,0.279058 437 | 0.419531,0.278564 438 | 0.421875,0.278049 439 | 0.424219,0.277561 440 | 0.426563,0.277097 441 | 0.428906,0.276625 442 | 0.43125,0.276196 443 | 0.433594,0.275705 444 | 0.435938,0.27525 445 | 0.438281,0.274752 446 | 0.440625,0.274261 447 | 0.442969,0.273795 448 | 0.445312,0.273324 449 | 0.447656,0.27288 450 | 0.45,0.272428 451 | 0.452344,0.271955 452 | 0.454687,0.271581 453 | 0.457031,0.271164 454 | 0.459375,0.270779 455 | 0.461719,0.270387 456 | 0.464063,0.269988 457 | 0.466406,0.269611 458 | 0.46875,0.269236 459 | 0.471094,0.268846 460 | 0.473438,0.268481 461 | 0.475781,0.268092 462 | 0.478125,0.267725 463 | 0.480469,0.267348 464 | 0.482813,0.266989 465 | 0.485156,0.266619 466 | 0.4875,0.266269 467 | 0.489844,0.26594 468 | 0.492188,0.265606 469 | 0.494531,0.265311 470 | 0.496875,0.264982 471 | 0.499219,0.264668 472 | 0.501562,0.264364 473 | 0.503906,0.264058 474 | 0.50625,0.263719 475 | 0.508594,0.263425 476 | 0.510938,0.263082 477 | 0.513281,0.26281 478 | 0.515625,0.262488 479 | 0.517969,0.262135 480 | 0.520313,0.261761 481 | 0.522656,0.261409 482 | 0.525,0.261105 483 | 0.527344,0.260841 484 | 0.529688,0.260583 485 | 0.532031,0.260326 486 | 0.534375,0.26006 487 | 0.536719,0.259831 488 | 0.539062,0.259563 489 | 0.541406,0.259304 490 | 0.54375,0.259037 491 | 0.546094,0.258764 492 | 0.548437,0.258501 493 | 0.550781,0.258262 494 | 0.553125,0.258013 495 | 0.555469,0.257744 496 | 0.557813,0.25746 497 | 0.560156,0.257206 498 | 0.5625,0.256931 499 | 0.564844,0.2567 500 | 0.567188,0.256486 501 | 0.569531,0.256233 502 | 0.571875,0.256014 503 | 0.574219,0.255742 504 | 0.576563,0.255424 505 | 0.578906,0.255228 506 | 0.58125,0.255004 507 | 0.583594,0.254787 508 | 0.585938,0.254551 509 | 0.588281,0.254322 510 | 0.590625,0.254048 511 | 0.592969,0.253832 512 | 0.595312,0.253599 513 | 0.597656,0.253375 514 | > 0.6,180.689 515 | -------------------------------------------------------------------------------- /tests/csv/data/DeviationHistogram_diff.csv: -------------------------------------------------------------------------------- 1 | Deviation [mm],Surface [mm²] - also differs 2 | -0.6,2.98287e-05 3 | -0.597656,0.000197238 4 | -0.595313,0.00038192 5 | -0.592969,0.000566594 6 | -0.590625,0.000751271 7 | -0.6,0.000935978 8 | -0.7,0.00112063 9 | -0.8,0.00130531 10 | -1.0,0.00149003 11 | -1.5,0.00167467 12 | -2.5,0.0018594 13 | -0.574219,different_type_here 14 | -0.571875,0.00222882 15 | -0.569531,0.00241333 16 | -0.567188,0.00260286 17 | -0.564844,0.00280005 18 | -0.5625,0.0029971 19 | -0.560156,0.00319449 20 | -0.557813,0.00339167 21 | -0.555469,0.00358901 22 | -0.553125,0.00378616 23 | -0.550781,0.00398353 24 | -0.548438,0.00418067 25 | -0.546094,0.00437781 26 | -0.54375,0.0045754 27 | -0.541406,0.00497736 28 | -0.539062,0.00562869 29 | -0.536719,0.00628644 30 | -0.534375,0.00694376 31 | -0.532031,0.00760131 32 | -0.529688,0.00825931 33 | -0.527344,0.00891594 34 | -0.525,0.00957441 35 | -0.522656,0.0102315 36 | -0.520313,0.0108888 37 | -0.517969,0.0115471 38 | -0.515625,0.0122039 39 | -0.513281,0.0128619 40 | -0.510938,0.0135191 41 | -0.508594,0.021742 42 | -0.50625,0.0979612 43 | -0.503906,0.226019 44 | -0.501563,0.283784 45 | -0.499219,0.305244 46 | -0.496875,0.345156 47 | -0.494531,0.343327 48 | -0.492188,0.370366 49 | -0.489844,0.391152 50 | -0.4875,0.374938 51 | -0.485156,0.390856 52 | -0.482813,0.441054 53 | -0.480469,0.473784 54 | -0.478125,0.487892 55 | -0.475781,0.533334 56 | -0.473438,0.607911 57 | -0.471094,0.569942 58 | -0.46875,0.546658 59 | -0.466406,0.534465 60 | -0.464063,0.521933 61 | -0.461719,0.540654 62 | -0.459375,0.550342 63 | -0.457031,0.544246 64 | -0.454688,0.539219 65 | -0.452344,0.565432 66 | -0.45,0.584493 67 | -0.447656,0.594559 68 | -0.445312,0.762362 69 | -0.442969,0.85163 70 | -0.440625,0.804051 71 | -0.438281,0.730209 72 | -0.435938,0.623656 73 | -0.433594,0.561942 74 | -0.43125,0.554678 75 | -0.428906,0.552584 76 | -0.426563,0.544543 77 | -0.424219,0.541332 78 | -0.421875,0.539419 79 | -0.419531,0.535715 80 | -0.417188,0.528287 81 | -0.414844,0.510097 82 | -0.4125,0.493129 83 | -0.410156,0.480034 84 | -0.407813,0.463376 85 | -0.405469,0.448299 86 | -0.403125,0.43594 87 | -0.400781,0.426014 88 | -0.398438,0.418224 89 | -0.396094,0.412027 90 | -0.39375,0.404904 91 | -0.391406,0.399111 92 | -0.389063,0.395023 93 | -0.386719,0.394628 94 | -0.384375,0.401207 95 | -0.382031,0.403121 96 | -0.379688,0.405621 97 | -0.377344,0.411265 98 | -0.375,0.420196 99 | -0.372656,0.43235 100 | -0.370313,0.4444 101 | -0.367969,0.453988 102 | -0.365625,0.464616 103 | -0.363281,0.481407 104 | -0.360938,0.497669 105 | -0.358594,0.520736 106 | -0.35625,0.540397 107 | -0.353906,0.564052 108 | -0.351562,0.590408 109 | -0.349219,0.611136 110 | -0.346875,0.629912 111 | -0.344531,0.644368 112 | -0.342188,0.656525 113 | -0.339844,0.673917 114 | -0.3375,0.6923 115 | -0.335156,0.712214 116 | -0.332813,0.738892 117 | -0.330469,0.784918 118 | -0.328125,0.813485 119 | -0.325781,0.845362 120 | -0.323438,0.891095 121 | -0.321094,0.926883 122 | -0.31875,0.962925 123 | -0.316406,1.01895 124 | -0.314063,1.08486 125 | -0.311719,1.14281 126 | -0.309375,1.16594 127 | -0.307031,1.18784 128 | -0.304688,1.21217 129 | -0.302344,1.26247 130 | -0.3,1.34998 131 | -0.297656,1.35485 132 | -0.295313,1.40214 133 | -0.292969,1.4822 134 | -0.290625,1.56808 135 | -0.288281,1.67817 136 | -0.285938,1.78134 137 | -0.283594,1.89517 138 | -0.28125,2.01535 139 | -0.278906,2.16215 140 | -0.276563,2.5158 141 | -0.274219,3.01218 142 | -0.271875,3.88853 143 | -0.269531,4.50542 144 | -0.267188,4.56822 145 | -0.264844,5.67477 146 | -0.2625,6.92868 147 | -0.260156,8.89476 148 | -0.257812,11.2662 149 | -0.255469,10.9973 150 | -0.253125,11.0213 151 | -0.250781,11.7452 152 | -0.248438,12.5683 153 | -0.246094,14.5304 154 | -0.24375,16.8417 155 | -0.241406,19.2301 156 | -0.239063,22.9344 157 | -0.236719,24.884 158 | -0.234375,25.9041 159 | -0.232031,25.7823 160 | -0.229688,24.1785 161 | -0.227344,21.8301 162 | -0.225,20.4353 163 | -0.222656,18.2923 164 | -0.220313,15.9943 165 | -0.217969,14.9076 166 | -0.215625,14.1258 167 | -0.213281,13.8547 168 | -0.210938,14.0053 169 | -0.208594,14.3485 170 | -0.20625,14.9161 171 | -0.203906,15.4934 172 | -0.201562,16.0577 173 | -0.199219,16.8416 174 | -0.196875,17.746 175 | -0.194531,19.4365 176 | -0.192188,21.7911 177 | -0.189844,24.7822 178 | -0.1875,28.3985 179 | -0.185156,32.6523 180 | -0.182813,37.4158 181 | -0.180469,39.436 182 | -0.178125,40.4951 183 | -0.175781,40.8298 184 | -0.173438,40.2616 185 | -0.171094,41.4706 186 | -0.16875,43.8043 187 | -0.166406,48.6796 188 | -0.164062,52.488 189 | -0.161719,60.3067 190 | -0.159375,67.6012 191 | -0.157031,75.4188 192 | -0.154687,85.221 193 | -0.152344,95.1454 194 | -0.15,108.193 195 | -0.147656,121.607 196 | -0.145313,138.552 197 | -0.142969,157.244 198 | -0.140625,178.366 199 | -0.138281,199.006 200 | -0.135938,214.883 201 | -0.133594,225.857 202 | -0.13125,244.21 203 | -0.128906,259.522 204 | -0.126563,289.639 205 | -0.124219,328.315 206 | -0.121875,362.679 207 | -0.119531,397.494 208 | -0.117188,440.275 209 | -0.114844,496.999 210 | -0.1125,553.358 211 | -0.110156,610.158 212 | -0.107812,669.913 213 | -0.105469,711.641 214 | -0.103125,758.797 215 | -0.100781,808.154 216 | -0.0984375,891.278 217 | -0.0960938,976.417 218 | -0.09375,1077.74 219 | -0.0914062,1169.26 220 | -0.0890625,1259.47 221 | -0.0867187,1332.88 222 | -0.084375,1364.03 223 | -0.0820312,1333.15 224 | -0.0796875,1276.47 225 | -0.0773438,1213.15 226 | -0.075,1163.26 227 | -0.0726563,1114.48 228 | -0.0703125,1090.39 229 | -0.0679687,1086.96 230 | -0.065625,1096.5 231 | -0.0632812,1136.48 232 | -0.0609375,1194.24 233 | -0.0585938,1288.38 234 | -0.05625,1382.61 235 | -0.0539063,1474.37 236 | -0.0515625,1513.37 237 | -0.0492188,1505.08 238 | -0.046875,1448.95 239 | -0.0445312,1393.91 240 | -0.0421875,1304.1 241 | -0.0398437,1216.91 242 | -0.0375,1126.85 243 | -0.0351562,1037.66 244 | -0.0328125,938.972 245 | -0.0304688,842.832 246 | -0.028125,759.245 247 | -0.0257813,699.029 248 | -0.0234375,665.831 249 | -0.0210937,657.703 250 | -0.01875,663.328 251 | -0.0164062,693.228 252 | -0.0140625,711.601 253 | -0.0117188,723.543 254 | -0.00937498,720.624 255 | -0.00703126,699.44 256 | -0.00468749,679.729 257 | -0.00234377,654.56 258 | 0,639.303 259 | 0.00234377,655.427 260 | 0.00468749,676.04 261 | 0.00703126,705.868 262 | 0.00937498,748.042 263 | 0.0117188,812.488 264 | 0.0140625,864.877 265 | 0.0164062,911.603 266 | 0.01875,960.975 267 | 0.0210937,992.786 268 | 0.0234375,1000.8 269 | 0.0257813,981.773 270 | 0.028125,931.843 271 | 0.0304688,862.485 272 | 0.0328125,783.891 273 | 0.0351562,713.606 274 | 0.0375,652.863 275 | 0.0398437,600.229 276 | 0.0421875,552.889 277 | 0.0445312,514.36 278 | 0.046875,480.352 279 | 0.0492188,449.506 280 | 0.0515625,424.929 281 | 0.0539063,393.91 282 | 0.05625,368.892 283 | 0.0585938,350.906 284 | 0.0609375,325.901 285 | 0.0632812,308.577 286 | 0.065625,287.256 287 | 0.0679687,256.804 288 | 0.0703125,219.251 289 | 0.0726563,187.177 290 | 0.075,161.527 291 | 0.0773438,149.918 292 | 0.0796875,146.647 293 | 0.0820312,144.546 294 | 0.084375,141.077 295 | 0.0867187,135.229 296 | 0.0890625,118.676 297 | 0.0914062,101.711 298 | 0.09375,84.8608 299 | 0.0960938,75.1614 300 | 0.0984375,69.1875 301 | 0.100781,59.0727 302 | 0.103125,40.8601 303 | 0.105469,25.3316 304 | 0.107813,16.134 305 | 0.110156,11.0083 306 | 0.1125,7.7022 307 | 0.114844,4.81511 308 | 0.117188,2.93038 309 | 0.119531,2.2669 310 | 0.121875,1.84511 311 | 0.124219,1.69962 312 | 0.126562,1.66358 313 | 0.128906,1.47109 314 | 0.13125,1.27072 315 | 0.133594,1.20311 316 | 0.135938,1.06163 317 | 0.138281,0.955001 318 | 0.140625,0.864637 319 | 0.142969,0.850746 320 | 0.145312,0.807083 321 | 0.147656,0.774698 322 | 0.15,0.756973 323 | 0.152344,0.719494 324 | 0.154688,0.66364 325 | 0.157031,0.631981 326 | 0.159375,0.60928 327 | 0.161719,0.588666 328 | 0.164062,0.5677 329 | 0.166406,0.550603 330 | 0.16875,0.537356 331 | 0.171094,0.527421 332 | 0.173438,0.525581 333 | 0.175781,0.511169 334 | 0.178125,0.491182 335 | 0.180469,0.474581 336 | 0.182813,0.470486 337 | 0.185156,0.457949 338 | 0.1875,0.44142 339 | 0.189844,0.425765 340 | 0.192187,0.415711 341 | 0.194531,0.411068 342 | 0.196875,0.407548 343 | 0.199219,0.404015 344 | 0.201563,0.401023 345 | 0.203906,0.398012 346 | 0.20625,0.394963 347 | 0.208594,0.391809 348 | 0.210938,0.388738 349 | 0.213281,0.38565 350 | 0.215625,0.381978 351 | 0.217969,0.378512 352 | 0.220313,0.375193 353 | 0.222656,0.371509 354 | 0.225,0.367862 355 | 0.227344,0.36521 356 | 0.229688,0.36282 357 | 0.232031,0.360546 358 | 0.234375,0.358451 359 | 0.236719,0.356484 360 | 0.239062,0.354275 361 | 0.241406,0.352045 362 | 0.24375,0.349844 363 | 0.246094,0.347861 364 | 0.248438,0.345825 365 | 0.250781,0.343863 366 | 0.253125,0.341873 367 | 0.255469,0.339872 368 | 0.257812,0.337686 369 | 0.260156,0.33565 370 | 0.2625,0.333442 371 | 0.264844,0.33124 372 | 0.267188,0.329882 373 | 0.269531,0.32868 374 | 0.271875,0.327488 375 | 0.274219,0.326319 376 | 0.276563,0.325142 377 | 0.278906,0.323918 378 | 0.28125,0.322675 379 | 0.283594,0.321432 380 | 0.285937,0.320002 381 | 0.288281,0.318855 382 | 0.290625,0.317812 383 | 0.292969,0.316835 384 | 0.295313,0.315851 385 | 0.297656,0.314898 386 | 0.3,0.3139 387 | 0.302344,0.312838 388 | 0.304688,0.311867 389 | 0.307031,0.310897 390 | 0.309375,0.31001 391 | 0.311719,0.309085 392 | 0.314063,0.30815 393 | 0.316406,0.307223 394 | 0.31875,0.306323 395 | 0.321094,0.305385 396 | 0.323438,0.304349 397 | 0.325781,0.303313 398 | 0.328125,0.302513 399 | 0.330469,0.301661 400 | 0.332812,0.300852 401 | 0.335156,0.300093 402 | 0.3375,0.299349 403 | 0.339844,0.29864 404 | 0.342188,0.297917 405 | 0.344531,0.297198 406 | 0.346875,0.296429 407 | 0.349219,0.295656 408 | 0.351562,0.295004 409 | 0.353906,0.294282 410 | 0.35625,0.293514 411 | 0.358594,0.292854 412 | 0.360938,0.292198 413 | 0.363281,0.291553 414 | 0.365625,0.290871 415 | 0.367969,0.290206 416 | 0.370313,0.289626 417 | 0.372656,0.289083 418 | 0.375,0.288527 419 | 0.377344,0.287986 420 | 0.379687,0.287446 421 | 0.382031,0.286902 422 | 0.384375,0.286339 423 | 0.386719,0.285829 424 | 0.389063,0.285289 425 | 0.391406,0.284807 426 | 0.39375,0.284297 427 | 0.396094,0.283778 428 | 0.398438,0.283281 429 | 0.400781,0.282782 430 | 0.403125,0.282257 431 | 0.405469,0.28173 432 | 0.407812,0.281221 433 | 0.410156,0.280692 434 | 0.4125,0.280162 435 | 0.414844,0.279632 436 | 0.417188,0.279058 437 | 0.419531,0.278564 438 | 0.421875,0.278049 439 | 0.424219,0.277561 440 | 0.426563,0.277097 441 | 0.428906,0.276625 442 | 0.43125,0.276196 443 | 0.433594,0.275705 444 | 0.435938,0.27525 445 | 0.438281,0.274752 446 | 0.440625,0.274261 447 | 0.442969,0.273795 448 | 0.445312,0.273324 449 | 0.447656,0.27288 450 | 0.45,0.272428 451 | 0.452344,0.271955 452 | 0.454687,0.271581 453 | 0.457031,0.271164 454 | 0.459375,0.270779 455 | 0.461719,0.270387 456 | 0.464063,0.269988 457 | 0.466406,0.269611 458 | 0.46875,0.269236 459 | 0.471094,0.268846 460 | 0.473438,0.268481 461 | 0.475781,0.268092 462 | 0.478125,0.267725 463 | 0.480469,0.267348 464 | 0.482813,0.266989 465 | 0.485156,0.266619 466 | 0.4875,0.266269 467 | 0.489844,0.26594 468 | 0.492188,0.265606 469 | 0.494531,0.265311 470 | 0.496875,0.264982 471 | 0.499219,0.264668 472 | 0.501562,0.264364 473 | 0.503906,0.264058 474 | 0.50625,0.263719 475 | 0.508594,0.263425 476 | 0.510938,0.263082 477 | 0.513281,0.26281 478 | 0.515625,0.262488 479 | 0.517969,0.262135 480 | 0.520313,0.261761 481 | 0.522656,0.261409 482 | 0.525,0.261105 483 | 0.527344,0.260841 484 | 0.529688,0.260583 485 | 0.532031,0.260326 486 | 0.534375,0.26006 487 | 0.536719,0.259831 488 | 0.539062,0.259563 489 | 0.541406,0.259304 490 | 0.54375,0.259037 491 | 0.546094,0.258764 492 | 0.548437,0.258501 493 | 0.550781,0.258262 494 | 0.553125,0.258013 495 | 0.555469,0.257744 496 | 0.557813,0.25746 497 | 0.560156,0.257206 498 | 0.5625,0.256931 499 | 0.564844,0.2567 500 | 0.567188,0.256486 501 | 0.569531,0.256233 502 | 0.571875,0.256014 503 | 0.574219,0.255742 504 | 0.576563,0.255424 505 | 0.578906,0.255228 506 | 0.58125,0.255004 507 | 0.583594,0.254787 508 | 0.585938,0.254551 509 | 0.588281,0.254322 510 | 0.590625,0.254048 511 | 0.592969,0.253832 512 | 0.595312,0.253599 513 | 0.597656,0.253375 514 | > 0.6,180.689 515 | -------------------------------------------------------------------------------- /tests/csv/data/CumulatedHistogram.csv: -------------------------------------------------------------------------------- 1 | Abs. deviation [mm],Normalized surface [%], 2 | 0.001174168,0.850087218, 3 | 0.002348337,1.712812670, 4 | 0.003522505,2.589375153, 5 | 0.004696674,3.480449319, 6 | 0.005870842,4.383642599, 7 | 0.007045010,5.301519856, 8 | 0.008219179,6.236816943, 9 | 0.009393346,7.190382481, 10 | 0.010567515,8.160080761, 11 | 0.011741684,9.138847888, 12 | 0.012915852,10.142892599, 13 | 0.014090020,11.157120019, 14 | 0.015264189,12.188186496, 15 | 0.016438358,13.220177591, 16 | 0.017612524,14.253802598, 17 | 0.018786693,15.305617452, 18 | 0.019960862,16.370445490, 19 | 0.021135030,17.449319363, 20 | 0.022309199,18.540568650, 21 | 0.023483368,19.645923376, 22 | 0.024657534,20.766735077, 23 | 0.025831703,21.897117794, 24 | 0.027005872,23.046384752, 25 | 0.028180040,24.203044176, 26 | 0.029354209,25.370097160, 27 | 0.030528378,26.553612947, 28 | 0.031702545,27.745500207, 29 | 0.032876715,28.938910365, 30 | 0.034050882,30.145353079, 31 | 0.035225049,31.350904703, 32 | 0.036399219,32.566606998, 33 | 0.037573386,33.788216114, 34 | 0.038747557,35.019856691, 35 | 0.039921723,36.264941096, 36 | 0.041095894,37.515404820, 37 | 0.042270061,38.787707686, 38 | 0.043444227,40.075999498, 39 | 0.044618398,41.366159916, 40 | 0.045792565,42.665281892, 41 | 0.046966735,43.965929747, 42 | 0.048140902,45.282265544, 43 | 0.049315069,46.595153213, 44 | 0.050489239,47.902497649, 45 | 0.051663406,49.191641808, 46 | 0.052837577,50.463205576, 47 | 0.054011744,51.701438427, 48 | 0.055185910,52.894777060, 49 | 0.056360081,54.045236111, 50 | 0.057534248,55.158489943, 51 | 0.058708418,56.231778860, 52 | 0.059882585,57.266777754, 53 | 0.061056755,58.270329237, 54 | 0.062230922,59.242850542, 55 | 0.063405089,60.201680660, 56 | 0.064579256,61.133688688, 57 | 0.065753430,62.058019638, 58 | 0.066927597,62.974274158, 59 | 0.068101764,63.874566555, 60 | 0.069275931,64.767158031, 61 | 0.070450097,65.656721592, 62 | 0.071624272,66.536921263, 63 | 0.072798438,67.421746254, 64 | 0.073972605,68.311476707, 65 | 0.075146772,69.211310148, 66 | 0.076320939,70.112890005, 67 | 0.077495113,71.033751965, 68 | 0.078669280,71.973276138, 69 | 0.079843447,72.926187515, 70 | 0.081017613,73.900020123, 71 | 0.082191788,74.890083075, 72 | 0.083365954,75.886934996, 73 | 0.084540121,76.888716221, 74 | 0.085714288,77.873545885, 75 | 0.086888455,78.834104538, 76 | 0.088062629,79.769921303, 77 | 0.089236796,80.671823025, 78 | 0.090410963,81.540828943, 79 | 0.091585129,82.365190983, 80 | 0.092759296,83.161443472, 81 | 0.093933471,83.915746212, 82 | 0.095107637,84.628838301, 83 | 0.096281804,85.308974981, 84 | 0.097455971,85.954582691, 85 | 0.098630138,86.579447985, 86 | 0.099804312,87.169665098, 87 | 0.100978479,87.730222940, 88 | 0.102152646,88.278704882, 89 | 0.103326812,88.807529211, 90 | 0.104500979,89.309030771, 91 | 0.105675153,89.796549082, 92 | 0.106849320,90.263152122, 93 | 0.108023487,90.709644556, 94 | 0.109197654,91.131585836, 95 | 0.110371821,91.530019045, 96 | 0.111545995,91.908967495, 97 | 0.112720162,92.270261049, 98 | 0.113894328,92.608851194, 99 | 0.115068495,92.930996418, 100 | 0.116242670,93.230342865, 101 | 0.117416836,93.512493372, 102 | 0.118591003,93.781965971, 103 | 0.119765170,94.037300348, 104 | 0.120939337,94.280588627, 105 | 0.122113511,94.516515732, 106 | 0.123287678,94.737839699, 107 | 0.124461845,94.948399067, 108 | 0.125636011,95.145440102, 109 | 0.126810178,95.328295231, 110 | 0.127984345,95.501041412, 111 | 0.129158512,95.671397448, 112 | 0.130332693,95.836204290, 113 | 0.131506860,95.993894339, 114 | 0.132681027,96.144771576, 115 | 0.133855194,96.292334795, 116 | 0.135029361,96.436476707, 117 | 0.136203527,96.576750278, 118 | 0.137377694,96.711659431, 119 | 0.138551861,96.838545799, 120 | 0.139726028,96.958643198, 121 | 0.140900195,97.073030472, 122 | 0.142074376,97.179389000, 123 | 0.143248543,97.278797626, 124 | 0.144422710,97.373503447, 125 | 0.145596877,97.460317612, 126 | 0.146771044,97.542625666, 127 | 0.147945210,97.620105743, 128 | 0.149119377,97.693759203, 129 | 0.150293544,97.762089968, 130 | 0.151467711,97.826510668, 131 | 0.152641878,97.887098789, 132 | 0.153816059,97.945028543, 133 | 0.154990226,97.999238968, 134 | 0.156164393,98.050117493, 135 | 0.157338560,98.098433018, 136 | 0.158512726,98.143929243, 137 | 0.159686893,98.187518120, 138 | 0.160861060,98.228704929, 139 | 0.162035227,98.266649246, 140 | 0.163209394,98.301833868, 141 | 0.164383575,98.335844278, 142 | 0.165557742,98.369115591, 143 | 0.166731909,98.400253057, 144 | 0.167906076,98.429644108, 145 | 0.169080243,98.458266258, 146 | 0.170254409,98.486226797, 147 | 0.171428576,98.513489962, 148 | 0.172602743,98.540550470, 149 | 0.173776910,98.567348719, 150 | 0.174951077,98.594599962, 151 | 0.176125258,98.622393608, 152 | 0.177299425,98.649758101, 153 | 0.178473592,98.676317930, 154 | 0.179647759,98.702639341, 155 | 0.180821925,98.729163408, 156 | 0.181996092,98.754578829, 157 | 0.183170259,98.778545856, 158 | 0.184344426,98.800712824, 159 | 0.185518593,98.821377754, 160 | 0.186692759,98.840701580, 161 | 0.187866941,98.858773708, 162 | 0.189041108,98.875623941, 163 | 0.190215275,98.891466856, 164 | 0.191389441,98.906296492, 165 | 0.192563608,98.920285702, 166 | 0.193737775,98.933547735, 167 | 0.194911942,98.946100473, 168 | 0.196086109,98.958146572, 169 | 0.197260275,98.969858885, 170 | 0.198434457,98.981326818, 171 | 0.199608624,98.992526531, 172 | 0.200782791,99.003458023, 173 | 0.201956958,99.014151096, 174 | 0.203131124,99.024713039, 175 | 0.204305291,99.035084248, 176 | 0.205479458,99.045264721, 177 | 0.206653625,99.055224657, 178 | 0.207827792,99.064975977, 179 | 0.209001958,99.074620008, 180 | 0.210176140,99.084144831, 181 | 0.211350307,99.093610048, 182 | 0.212524474,99.103021622, 183 | 0.213698640,99.112468958, 184 | 0.214872807,99.121993780, 185 | 0.216046974,99.131822586, 186 | 0.217221141,99.141925573, 187 | 0.218395308,99.152237177, 188 | 0.219569474,99.162942171, 189 | 0.220743641,99.174284935, 190 | 0.221917823,99.186444283, 191 | 0.223091990,99.199700356, 192 | 0.224266157,99.213397503, 193 | 0.225440323,99.227601290, 194 | 0.226614490,99.242031574, 195 | 0.227788657,99.257463217, 196 | 0.228962824,99.273687601, 197 | 0.230136991,99.290460348, 198 | 0.231311157,99.307823181, 199 | 0.232485339,99.325370789, 200 | 0.233659506,99.342995882, 201 | 0.234833673,99.359863997, 202 | 0.236007839,99.376505613, 203 | 0.237182006,99.393224716, 204 | 0.238356173,99.408757687, 205 | 0.239530340,99.423128366, 206 | 0.240704507,99.436163902, 207 | 0.241878673,99.448078871, 208 | 0.243052840,99.459606409, 209 | 0.244227022,99.470394850, 210 | 0.245401189,99.480372667, 211 | 0.246575356,99.489355087, 212 | 0.247749522,99.497824907, 213 | 0.248923689,99.506187439, 214 | 0.250097871,99.514180422, 215 | 0.251272023,99.521952868, 216 | 0.252446204,99.529504776, 217 | 0.253620356,99.536782503, 218 | 0.254794538,99.544256926, 219 | 0.255968690,99.551975727, 220 | 0.257142872,99.559956789, 221 | 0.258317024,99.567097425, 222 | 0.259491205,99.573200941, 223 | 0.260665387,99.578565359, 224 | 0.261839539,99.583303928, 225 | 0.263013721,99.587816000, 226 | 0.264187872,99.591827393, 227 | 0.265362054,99.595296383, 228 | 0.266536206,99.598497152, 229 | 0.267710388,99.601626396, 230 | 0.268884540,99.604904652, 231 | 0.270058721,99.608004093, 232 | 0.271232873,99.610841274, 233 | 0.272407055,99.613296986, 234 | 0.273581237,99.615508318, 235 | 0.274755388,99.617511034, 236 | 0.275929570,99.619400501, 237 | 0.277103722,99.621152878, 238 | 0.278277904,99.622786045, 239 | 0.279452056,99.624377489, 240 | 0.280626237,99.625933170, 241 | 0.281800389,99.627441168, 242 | 0.282974571,99.628907442, 243 | 0.284148753,99.630337954, 244 | 0.285322905,99.631732702, 245 | 0.286497086,99.633097649, 246 | 0.287671238,99.634426832, 247 | 0.288845420,99.635708332, 248 | 0.290019572,99.636954069, 249 | 0.291193753,99.638164043, 250 | 0.292367905,99.639350176, 251 | 0.293542087,99.640506506, 252 | 0.294716269,99.641644955, 253 | 0.295890421,99.642759562, 254 | 0.297064602,99.643862247, 255 | 0.298238754,99.644958973, 256 | 0.299412936,99.646079540, 257 | 0.300587088,99.647146463, 258 | 0.301761270,99.648189545, 259 | 0.302935421,99.649208784, 260 | 0.304109603,99.650222063, 261 | 0.305283755,99.651223421, 262 | 0.306457937,99.652212858, 263 | 0.307632118,99.653196335, 264 | 0.308806270,99.654173851, 265 | 0.309980452,99.655145407, 266 | 0.311154604,99.656116962, 267 | 0.312328786,99.657064676, 268 | 0.313502938,99.657994509, 269 | 0.314677119,99.658888578, 270 | 0.315851271,99.659770727, 271 | 0.317025453,99.660623074, 272 | 0.318199635,99.661463499, 273 | 0.319373786,99.662297964, 274 | 0.320547968,99.663114548, 275 | 0.321722120,99.663919210, 276 | 0.322896302,99.664711952, 277 | 0.324070454,99.665486813, 278 | 0.325244635,99.666249752, 279 | 0.326418787,99.666994810, 280 | 0.327592969,99.667727947, 281 | 0.328767151,99.668449163, 282 | 0.329941303,99.669170380, 283 | 0.331115484,99.669861794, 284 | 0.332289636,99.670547247, 285 | 0.333463818,99.671214819, 286 | 0.334637970,99.671888351, 287 | 0.335812151,99.672549963, 288 | 0.336986303,99.673199654, 289 | 0.338160485,99.673849344, 290 | 0.339334637,99.674487114, 291 | 0.340508819,99.675124884, 292 | 0.341683000,99.675756693, 293 | 0.342857152,99.676388502, 294 | 0.344031334,99.677008390, 295 | 0.345205486,99.677628279, 296 | 0.346379668,99.678236246, 297 | 0.347553819,99.678844213, 298 | 0.348728001,99.679440260, 299 | 0.349902153,99.680042267, 300 | 0.351076335,99.680626392, 301 | 0.352250516,99.681204557, 302 | 0.353424668,99.681776762, 303 | 0.354598850,99.682331085, 304 | 0.355773002,99.682891369, 305 | 0.356947184,99.683433771, 306 | 0.358121336,99.683976173, 307 | 0.359295517,99.684512615, 308 | 0.360469669,99.685037136, 309 | 0.361643851,99.685549736, 310 | 0.362818033,99.686068296, 311 | 0.363992184,99.686574936, 312 | 0.365166366,99.687081575, 313 | 0.366340518,99.687576294, 314 | 0.367514700,99.688071012, 315 | 0.368688852,99.688565731, 316 | 0.369863033,99.689054489, 317 | 0.371037185,99.689537287, 318 | 0.372211367,99.690020084, 319 | 0.373385519,99.690496922, 320 | 0.374559700,99.690973759, 321 | 0.375733882,99.691444635, 322 | 0.376908034,99.691903591, 323 | 0.378082216,99.692356586, 324 | 0.379256368,99.692809582, 325 | 0.380430549,99.693262577, 326 | 0.381604701,99.693715572, 327 | 0.382778883,99.694168568, 328 | 0.383953035,99.694627523, 329 | 0.385127217,99.695068598, 330 | 0.386301398,99.695515633, 331 | 0.387475550,99.695956707, 332 | 0.388649732,99.696403742, 333 | 0.389823884,99.696844816, 334 | 0.390998065,99.697297812, 335 | 0.392172217,99.697750807, 336 | 0.393346399,99.698209763, 337 | 0.394520551,99.698662758, 338 | 0.395694733,99.699115753, 339 | 0.396868914,99.699580669, 340 | 0.398043066,99.700045586, 341 | 0.399217248,99.700510502, 342 | 0.400391400,99.700975418, 343 | 0.401565582,99.701446295, 344 | 0.402739733,99.701923132, 345 | 0.403913915,99.702399969, 346 | 0.405088067,99.702882767, 347 | 0.406262249,99.703365564, 348 | 0.407436401,99.703866243, 349 | 0.408610582,99.704360962, 350 | 0.409784764,99.704867601, 351 | 0.410958916,99.705368280, 352 | 0.412133098,99.705886841, 353 | 0.413307250,99.706405401, 354 | 0.414481431,99.706929922, 355 | 0.415655583,99.707466364, 356 | 0.416829765,99.708002806, 357 | 0.418003917,99.708539248, 358 | 0.419178098,99.709075689, 359 | 0.420352280,99.709624052, 360 | 0.421526432,99.710172415, 361 | 0.422700614,99.710714817, 362 | 0.423874766,99.711263180, 363 | 0.425048947,99.711811543, 364 | 0.426223099,99.712359905, 365 | 0.427397281,99.712908268, 366 | 0.428571433,99.713462591, 367 | 0.429745615,99.714022875, 368 | 0.430919796,99.714577198, 369 | 0.432093948,99.715125561, 370 | 0.433268130,99.715679884, 371 | 0.434442282,99.716258049, 372 | 0.435616463,99.716848135, 373 | 0.436790615,99.717527628, 374 | 0.437964797,99.718189240, 375 | 0.439138949,99.718862772, 376 | 0.440313131,99.719601870, 377 | 0.441487283,99.720370770, 378 | 0.442661464,99.721109867, 379 | 0.443835646,99.721854925, 380 | 0.445009798,99.722534418, 381 | 0.446183980,99.723124504, 382 | 0.447358131,99.723702669, 383 | 0.448532313,99.724268913, 384 | 0.449706465,99.724835157, 385 | 0.450880647,99.725407362, 386 | 0.452054799,99.725949764, 387 | 0.453228980,99.726492167, 388 | 0.454403162,99.727016687, 389 | 0.455577314,99.727553129, 390 | 0.456751496,99.728089571, 391 | 0.457925647,99.728626013, 392 | 0.459099829,99.729162455, 393 | 0.460273981,99.729698896, 394 | 0.461448163,99.730223417, 395 | 0.462622315,99.730753899, 396 | 0.463796496,99.731266499, 397 | 0.464970678,99.731796980, 398 | 0.466144830,99.732333422, 399 | 0.467319012,99.732869864, 400 | 0.468493164,99.733406305, 401 | 0.469667345,99.733954668, 402 | 0.470841497,99.734508991, 403 | 0.472015679,99.735087156, 404 | 0.473189831,99.735677242, 405 | 0.474364012,99.736225605, 406 | 0.475538194,99.736738205, 407 | 0.476712346,99.737244844, 408 | 0.477886528,99.737739563, 409 | 0.479060680,99.738228321, 410 | 0.480234861,99.738723040, 411 | 0.481409013,99.739205837, 412 | 0.482583195,99.739670753, 413 | 0.483757347,99.740111828, 414 | 0.484931529,99.740546942, 415 | 0.486105680,99.740970135, 416 | 0.487279862,99.741393328, 417 | 0.488454044,99.741828442, 418 | 0.489628196,99.742263556, 419 | 0.490802377,99.742686749, 420 | 0.491976529,99.743098021, 421 | 0.493150711,99.743497372, 422 | 0.494324863,99.743902683, 423 | 0.495499045,99.744313955, 424 | 0.496673197,99.744719267, 425 | 0.497847378,99.745100737, 426 | 0.499021560,99.745482206, 427 | 0.500195742,99.745851755, 428 | 0.501369894,99.746215343, 429 | 0.502544045,99.746555090, 430 | 0.503718197,99.746876955, 431 | 0.504892409,99.747145176, 432 | 0.506066561,99.747359753, 433 | 0.507240713,99.747556448, 434 | 0.508414865,99.747741222, 435 | 0.509589076,99.747925997, 436 | 0.510763228,99.748116732, 437 | 0.511937380,99.748301506, 438 | 0.513111591,99.748486280, 439 | 0.514285743,99.748671055, 440 | 0.515459895,99.748855829, 441 | 0.516634047,99.749040604, 442 | 0.517808259,99.749231339, 443 | 0.518982410,99.749416113, 444 | 0.520156562,99.749600887, 445 | 0.521330774,99.749785662, 446 | 0.522504926,99.749970436, 447 | 0.523679078,99.750155210, 448 | 0.524853230,99.750345945, 449 | 0.526027441,99.750530720, 450 | 0.527201593,99.750715494, 451 | 0.528375745,99.750900269, 452 | 0.529549897,99.751085043, 453 | 0.530724108,99.751269817, 454 | 0.531898260,99.751448631, 455 | 0.533072412,99.751621485, 456 | 0.534246624,99.751800299, 457 | 0.535420775,99.751973152, 458 | 0.536594927,99.752151966, 459 | 0.537769079,99.752324820, 460 | 0.538943291,99.752503633, 461 | 0.540117443,99.752676487, 462 | 0.541291595,99.752849340, 463 | 0.542465746,99.753028154, 464 | 0.543639958,99.753201008, 465 | 0.544814110,99.753379822, 466 | 0.545988262,99.753552675, 467 | 0.547162473,99.753731489, 468 | 0.548336625,99.753904343, 469 | 0.549510777,99.754077196, 470 | 0.550684929,99.754256010, 471 | 0.551859140,99.754428864, 472 | 0.553033292,99.754607677, 473 | 0.554207444,99.754780531, 474 | 0.555381656,99.754959345, 475 | 0.556555808,99.755132198, 476 | 0.557729959,99.755311012, 477 | 0.558904111,99.755483866, 478 | 0.560078323,99.755656719, 479 | 0.561252475,99.755823612, 480 | 0.562426627,99.755990505, 481 | 0.563600779,99.756157398, 482 | 0.564774990,99.756318331, 483 | 0.565949142,99.756485224, 484 | 0.567123294,99.756652117, 485 | 0.568297505,99.756813049, 486 | 0.569471657,99.756979942, 487 | 0.570645809,99.757146835, 488 | 0.571819961,99.757307768, 489 | 0.572994173,99.757474661, 490 | 0.574168324,99.757641554, 491 | 0.575342476,99.757808447, 492 | 0.576516628,99.757969379, 493 | 0.577690840,99.758136272, 494 | 0.578864992,99.758303165, 495 | 0.580039144,99.758464098, 496 | 0.581213355,99.758630991, 497 | 0.582387507,99.758797884, 498 | 0.583561659,99.758964777, 499 | 0.584735811,99.759125710, 500 | 0.585910022,99.759292603, 501 | 0.587084174,99.759459496, 502 | 0.588258326,99.759620428, 503 | 0.589432538,99.759787321, 504 | 0.590606689,99.759954214, 505 | 0.591780841,99.760121107, 506 | 0.592954993,99.760282040, 507 | 0.594129205,99.760448933, 508 | 0.595303357,99.760615826, 509 | 0.596477509,99.760776758, 510 | 0.597651660,99.760943651, 511 | 0.598825872,99.761110544, 512 | 0.600000024,99.761271477, 513 | 0.601174176,100.000000000, 514 | -------------------------------------------------------------------------------- /src/csv/preprocessing.rs: -------------------------------------------------------------------------------- 1 | use crate::csv; 2 | use crate::csv::value::Value; 3 | use crate::csv::Table; 4 | use schemars_derive::JsonSchema; 5 | use serde::{Deserialize, Serialize}; 6 | use std::cmp::Ordering::Equal; 7 | use tracing::{debug, warn}; 8 | 9 | #[derive(JsonSchema, Deserialize, Serialize, Debug, Clone)] 10 | /// Preprocessor options 11 | pub enum Preprocessor { 12 | /// Try to extract the headers from the first row - fallible if first row contains a number 13 | ExtractHeaders, 14 | /// Replace all fields in column by number by a deleted marker 15 | DeleteColumnByNumber(usize), 16 | /// Replace all fields in column by name by a deleted marker 17 | DeleteColumnByName(String), 18 | /// Sort rows by column with given name. Fails if no headers were extracted or column name is not found, or if any row has no numbers there 19 | SortByColumnName(String), 20 | /// Sort rows by column with given number. Fails if any row has no numbers there or if out of bounds. 21 | SortByColumnNumber(usize), 22 | /// Replace all fields in row with given number by a deleted marker 23 | DeleteRowByNumber(usize), 24 | /// Replace all fields in row where at least a single field matches regex by a deleted marker 25 | DeleteRowByRegex(String), 26 | /// replace found cell using row and column index by a deleted marker 27 | DeleteCellByNumber { 28 | /// column number 29 | column: usize, 30 | /// row number 31 | row: usize, 32 | }, 33 | /// replace found cell using column header and row index by a deleted marker 34 | DeleteCellByName { 35 | /// column with given name 36 | column: String, 37 | /// row number 38 | row: usize, 39 | }, 40 | } 41 | 42 | impl Preprocessor { 43 | pub(crate) fn process(&self, table: &mut Table) -> Result<(), csv::Error> { 44 | match self { 45 | Preprocessor::ExtractHeaders => extract_headers(table), 46 | Preprocessor::DeleteColumnByNumber(id) => delete_column_number(table, *id), 47 | Preprocessor::DeleteColumnByName(name) => delete_column_name(table, name.as_str()), 48 | Preprocessor::SortByColumnName(name) => sort_by_column_name(table, name.as_str()), 49 | Preprocessor::SortByColumnNumber(id) => sort_by_column_id(table, *id), 50 | Preprocessor::DeleteRowByNumber(id) => delete_row_by_number(table, *id), 51 | Preprocessor::DeleteRowByRegex(regex) => delete_row_by_regex(table, regex), 52 | Preprocessor::DeleteCellByNumber { column, row } => { 53 | delete_cell_by_number(table, *column, *row) 54 | } 55 | Preprocessor::DeleteCellByName { column, row } => { 56 | delete_cell_by_column_name_and_row_number(table, column, *row) 57 | } 58 | } 59 | } 60 | } 61 | 62 | fn delete_row_by_regex(table: &mut Table, regex: &str) -> Result<(), csv::Error> { 63 | let regex = regex::Regex::new(regex)?; 64 | table 65 | .rows_mut() 66 | .filter(|row| row.iter().any(|v| regex.is_match(v.to_string().as_str()))) 67 | .for_each(|mut row| row.iter_mut().for_each(|v| **v = Value::deleted())); 68 | Ok(()) 69 | } 70 | 71 | fn delete_row_by_number(table: &mut Table, id: usize) -> Result<(), csv::Error> { 72 | if let Some(mut v) = table.rows_mut().nth(id) { 73 | v.iter_mut().for_each(|v| **v = Value::deleted()) 74 | } 75 | Ok(()) 76 | } 77 | 78 | fn delete_cell_by_number(table: &mut Table, column: usize, row: usize) -> Result<(), csv::Error> { 79 | let value = table 80 | .columns 81 | .get_mut(column) 82 | .ok_or_else(|| { 83 | csv::Error::InvalidAccess(format!("Cell with column number {} not found.", column)) 84 | })? 85 | .rows 86 | .get_mut(row) 87 | .ok_or_else(|| { 88 | csv::Error::InvalidAccess(format!("Cell with row number {} not found.", row)) 89 | })?; 90 | 91 | *value = Value::deleted(); 92 | 93 | Ok(()) 94 | } 95 | 96 | fn delete_cell_by_column_name_and_row_number( 97 | table: &mut Table, 98 | column: &str, 99 | row: usize, 100 | ) -> Result<(), csv::Error> { 101 | let value = table 102 | .columns 103 | .iter_mut() 104 | .find(|col| col.header.as_deref().unwrap_or_default() == column) 105 | .ok_or_else(|| { 106 | csv::Error::InvalidAccess(format!("Cell with column name '{}' not found.", column)) 107 | })? 108 | .rows 109 | .get_mut(row) 110 | .ok_or_else(|| { 111 | csv::Error::InvalidAccess(format!("Cell with row number {} not found.", row)) 112 | })?; 113 | 114 | *value = Value::deleted(); 115 | 116 | Ok(()) 117 | } 118 | 119 | fn get_permutation(rows_to_sort_by: &Vec) -> permutation::Permutation { 120 | permutation::sort_by(rows_to_sort_by, |a, b| b.partial_cmp(a).unwrap_or(Equal)) 121 | } 122 | 123 | fn apply_permutation(table: &mut Table, mut permutation: permutation::Permutation) { 124 | table.columns.iter_mut().for_each(|c| { 125 | permutation.apply_slice_in_place(&mut c.rows); 126 | }); 127 | } 128 | 129 | fn sort_by_column_id(table: &mut Table, id: usize) -> Result<(), csv::Error> { 130 | let sort_master_col = table.columns.get(id).ok_or_else(|| { 131 | csv::Error::InvalidAccess(format!( 132 | "Column number sorting by id {id} requested but column not found." 133 | )) 134 | })?; 135 | let col_floats: Result, csv::Error> = sort_master_col 136 | .rows 137 | .iter() 138 | .map(|v| { 139 | v.get_quantity().map(|q| q.value).ok_or_else(|| { 140 | csv::Error::UnexpectedValue( 141 | v.clone(), 142 | "Expected quantity while trying to sort by column id".to_string(), 143 | ) 144 | }) 145 | }) 146 | .collect(); 147 | let permutation = get_permutation(&col_floats?); 148 | apply_permutation(table, permutation); 149 | Ok(()) 150 | } 151 | 152 | fn sort_by_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> { 153 | let sort_master_col = table 154 | .columns 155 | .iter() 156 | .find(|c| c.header.as_deref().unwrap_or_default() == name) 157 | .ok_or_else(|| { 158 | csv::Error::InvalidAccess(format!( 159 | "Requested format sorting by column'{name}' but column not found." 160 | )) 161 | })?; 162 | let col_floats: Result, csv::Error> = sort_master_col 163 | .rows 164 | .iter() 165 | .map(|v| { 166 | v.get_quantity().map(|q| q.value).ok_or_else(|| { 167 | csv::Error::UnexpectedValue( 168 | v.clone(), 169 | "Expected quantity while trying to sort by column name".to_string(), 170 | ) 171 | }) 172 | }) 173 | .collect(); 174 | let permutation = get_permutation(&col_floats?); 175 | apply_permutation(table, permutation); 176 | Ok(()) 177 | } 178 | 179 | fn delete_column_name(table: &mut Table, name: &str) -> Result<(), csv::Error> { 180 | if let Some(c) = table 181 | .columns 182 | .iter_mut() 183 | .find(|col| col.header.as_deref().unwrap_or_default() == name) 184 | { 185 | c.delete_contents(); 186 | } 187 | Ok(()) 188 | } 189 | 190 | fn delete_column_number(table: &mut Table, id: usize) -> Result<(), csv::Error> { 191 | if let Some(col) = table.columns.get_mut(id) { 192 | col.delete_contents(); 193 | } 194 | Ok(()) 195 | } 196 | 197 | fn extract_headers(table: &mut Table) -> Result<(), csv::Error> { 198 | debug!("Extracting headers..."); 199 | let can_extract = table 200 | .columns 201 | .iter() 202 | .all(|c| matches!(c.rows.first(), Some(Value::String(_)))); 203 | if !can_extract { 204 | warn!("Cannot extract header for this csv!"); 205 | return Ok(()); 206 | } 207 | 208 | for col in table.columns.iter_mut() { 209 | let title = col.rows.drain(0..1).next().ok_or_else(|| { 210 | csv::Error::InvalidAccess("Tried to extract header of empty column!".to_string()) 211 | })?; 212 | if let Value::String(title) = title { 213 | col.header = Some(title); 214 | } 215 | } 216 | Ok(()) 217 | } 218 | 219 | #[cfg(test)] 220 | mod tests { 221 | use super::*; 222 | use crate::csv::{Column, Delimiters, Error}; 223 | use std::fs::File; 224 | 225 | fn setup_table(delimiters: Option) -> Table { 226 | let delimiters = delimiters.unwrap_or_default(); 227 | Table::from_reader( 228 | File::open("tests/csv/data/DeviationHistogram.csv").unwrap(), 229 | &delimiters, 230 | ) 231 | .unwrap() 232 | } 233 | 234 | fn setup_table_two(delimiters: Option) -> Table { 235 | let delimiters = delimiters.unwrap_or_default(); 236 | Table::from_reader( 237 | File::open("tests/csv/data/defects_headers.csv").unwrap(), 238 | &delimiters, 239 | ) 240 | .unwrap() 241 | } 242 | 243 | #[test] 244 | fn test_extract_headers_two() { 245 | let mut table = setup_table_two(None); 246 | extract_headers(&mut table).unwrap(); 247 | assert_eq!( 248 | table.columns.first().unwrap().header.as_deref().unwrap(), 249 | "Entry" 250 | ); 251 | assert_eq!( 252 | table.columns.last().unwrap().header.as_deref().unwrap(), 253 | "Radius" 254 | ); 255 | } 256 | 257 | #[test] 258 | fn test_extract_headers() { 259 | let mut table = setup_table(None); 260 | extract_headers(&mut table).unwrap(); 261 | assert_eq!( 262 | table.columns.first().unwrap().header.as_deref().unwrap(), 263 | "Deviation [mm]" 264 | ); 265 | assert_eq!( 266 | table.columns.last().unwrap().header.as_deref().unwrap(), 267 | "Surface [mm²]" 268 | ); 269 | } 270 | 271 | #[test] 272 | fn test_delete_column_by_id() { 273 | let mut table = setup_table(None); 274 | extract_headers(&mut table).unwrap(); 275 | delete_column_number(&mut table, 0).unwrap(); 276 | assert_eq!( 277 | table.columns.first().unwrap().header.as_deref().unwrap(), 278 | "DELETED" 279 | ); 280 | assert!(table 281 | .columns 282 | .first() 283 | .unwrap() 284 | .rows 285 | .iter() 286 | .all(|v| *v == Value::deleted())); 287 | } 288 | 289 | #[test] 290 | fn test_delete_column_by_name() { 291 | let mut table = setup_table(None); 292 | extract_headers(&mut table).unwrap(); 293 | delete_column_name(&mut table, "Surface [mm²]").unwrap(); 294 | assert_eq!( 295 | table.columns.last().unwrap().header.as_deref().unwrap(), 296 | "DELETED" 297 | ); 298 | assert!(table 299 | .columns 300 | .last() 301 | .unwrap() 302 | .rows 303 | .iter() 304 | .all(|v| *v == Value::deleted())); 305 | } 306 | 307 | #[test] 308 | fn test_delete_row_by_id() { 309 | let mut table = setup_table(None); 310 | delete_row_by_number(&mut table, 0).unwrap(); 311 | assert_eq!( 312 | table 313 | .columns 314 | .first() 315 | .unwrap() 316 | .rows 317 | .first() 318 | .unwrap() 319 | .get_string() 320 | .as_deref() 321 | .unwrap(), 322 | "DELETED" 323 | ); 324 | } 325 | 326 | #[test] 327 | fn test_delete_row_by_regex() { 328 | let mut table = setup_table(None); 329 | delete_row_by_regex(&mut table, "mm").unwrap(); 330 | assert_eq!( 331 | table 332 | .columns 333 | .first() 334 | .unwrap() 335 | .rows 336 | .first() 337 | .unwrap() 338 | .get_string() 339 | .as_deref() 340 | .unwrap(), 341 | "DELETED" 342 | ); 343 | } 344 | 345 | #[test] 346 | fn test_sort_by_name() { 347 | let mut table = setup_table(None); 348 | extract_headers(&mut table).unwrap(); 349 | sort_by_column_name(&mut table, "Surface [mm²]").unwrap(); 350 | let mut peekable_rows = table.rows().peekable(); 351 | while let Some(row) = peekable_rows.next() { 352 | if let Some(next_row) = peekable_rows.peek() { 353 | assert!( 354 | row.get(1).unwrap().get_quantity().unwrap().value 355 | >= next_row.get(1).unwrap().get_quantity().unwrap().value 356 | ); 357 | } 358 | } 359 | } 360 | 361 | #[test] 362 | fn test_sort_by_id() { 363 | let mut table = setup_table(None); 364 | extract_headers(&mut table).unwrap(); 365 | let column = 1; 366 | sort_by_column_id(&mut table, column).unwrap(); 367 | let mut peekable_rows = table.rows().peekable(); 368 | while let Some(row) = peekable_rows.next() { 369 | if let Some(next_row) = peekable_rows.peek() { 370 | assert!( 371 | row.get(column).unwrap().get_quantity().unwrap().value 372 | >= next_row.get(column).unwrap().get_quantity().unwrap().value 373 | ); 374 | } 375 | } 376 | } 377 | 378 | #[test] 379 | fn sorting_by_mixed_column_fails() { 380 | let column = Column { 381 | header: Some("Field".to_string()), 382 | rows: vec![ 383 | Value::from_str("1.0", &None), 384 | Value::String("String-Value".to_string()), 385 | ], 386 | }; 387 | let mut table = Table { 388 | columns: vec![column], 389 | }; 390 | let order_by_name = sort_by_column_name(&mut table, "Field"); 391 | assert!(matches!( 392 | order_by_name.unwrap_err(), 393 | Error::UnexpectedValue(_, _) 394 | )); 395 | 396 | let order_by_id = sort_by_column_id(&mut table, 0); 397 | assert!(matches!( 398 | order_by_id.unwrap_err(), 399 | Error::UnexpectedValue(_, _) 400 | )); 401 | } 402 | 403 | #[test] 404 | fn non_existing_table_fails() { 405 | let mut table = setup_table(None); 406 | let order_by_name = sort_by_column_name(&mut table, "Non-Existing-Field"); 407 | assert!(matches!( 408 | order_by_name.unwrap_err(), 409 | Error::InvalidAccess(_) 410 | )); 411 | 412 | let order_by_id = sort_by_column_id(&mut table, 999); 413 | assert!(matches!(order_by_id.unwrap_err(), Error::InvalidAccess(_))); 414 | } 415 | 416 | #[test] 417 | fn test_delete_cell_by_numb() { 418 | let mut table = setup_table(None); 419 | delete_cell_by_number(&mut table, 1, 2).unwrap(); 420 | 421 | assert_eq!( 422 | table 423 | .columns 424 | .get(1) 425 | .unwrap() 426 | .rows 427 | .get(2) 428 | .unwrap() 429 | .get_string() 430 | .as_deref() 431 | .unwrap(), 432 | "DELETED" 433 | ); 434 | 435 | assert_ne!( 436 | table 437 | .columns 438 | .get(1) 439 | .unwrap() 440 | .rows 441 | .first() 442 | .unwrap() 443 | .get_string() 444 | .as_deref() 445 | .unwrap(), 446 | "DELETED" 447 | ); 448 | 449 | assert_eq!( 450 | table 451 | .columns 452 | .first() 453 | .unwrap() 454 | .rows 455 | .get(1) 456 | .unwrap() 457 | .get_string(), 458 | None 459 | ); 460 | } 461 | 462 | #[test] 463 | fn test_delete_cell_by_name() { 464 | let mut table = setup_table(None); 465 | extract_headers(&mut table).unwrap(); 466 | delete_cell_by_column_name_and_row_number(&mut table, "Surface [mm²]", 1).unwrap(); 467 | 468 | assert_eq!( 469 | table 470 | .columns 471 | .get(1) 472 | .unwrap() 473 | .rows 474 | .get(1) 475 | .unwrap() 476 | .get_string() 477 | .as_deref() 478 | .unwrap(), 479 | "DELETED" 480 | ); 481 | 482 | assert_eq!( 483 | table 484 | .columns 485 | .get(1) 486 | .unwrap() 487 | .rows 488 | .get(3) 489 | .unwrap() 490 | .get_string(), 491 | None 492 | ); 493 | 494 | assert_eq!( 495 | table 496 | .columns 497 | .get(0) 498 | .unwrap() 499 | .rows 500 | .get(1) 501 | .unwrap() 502 | .get_string(), 503 | None 504 | ); 505 | } 506 | } 507 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Havocompare - a folder comparison utility 2 | 3 | [![Crates.io](https://img.shields.io/crates/d/havocompare?style=flat)](https://crates.io/crates/havocompare) 4 | [![Documentation](https://docs.rs/havocompare/badge.svg)](https://docs.rs/havocompare) 5 | ![CI](https://github.com/VolumeGraphics/havocompare/actions/workflows/rust.yml/badge.svg?branch=main "CI") 6 | [![Coverage Status](https://coveralls.io/repos/github/VolumeGraphics/havocompare/badge.svg?branch=main)](https://coveralls.io/github/VolumeGraphics/havocompare?branch=main) 7 | [![License](https://img.shields.io/badge/license-MIT-blue?style=flat)](LICENSE) 8 | 9 | ## Contributors: 10 | 11 | 12 | Contributors 13 | 14 | 15 | ## Quickstart 16 | 17 | ### 0. Install havocompare 18 | 19 | You have rust? cool! try: 20 | `cargo install havocompare` 21 | 22 | You just want a binary: 23 | Check our binary downloads on github-pages 24 | 25 | ### 1. Create a config file 26 | 27 | Havocompare was developed with a few design goals in mind. We wanted a human-readable and easily composable 28 | configuration file format. 29 | After a few tries we ended up with the current format, which is a list of rules inside a yaml file. 30 | See the following example `config.yaml`: 31 | 32 | ```yaml 33 | rules: 34 | - name: "Numerical results csv" 35 | # you can have multiple includes and excludes 36 | pattern_include: 37 | - "**/export_*.csv" 38 | # excludes are optional 39 | pattern_exclude: 40 | - "**/export_1337.csv" 41 | CSV: 42 | comparison_modes: 43 | - Relative: 0.1 44 | - Absolute: 1.0 45 | ``` 46 | 47 | It creates a new rule named rule including all files matching "export_*.csv" in all sub-folders but exclude " 48 | export_1337.csv". 49 | String cells will be checked for perfect identity, numbers (including numbers with units) will be checked for a relative 50 | deviation smaller than `0.1` 51 | AND absolute deviation smaller than `1.0`. 52 | 53 | __Comparison rules__ 54 | 55 | - Relative means validity is checked like: `|nominal - actual| / |nominal| < tolerance` 56 | - Absolute means validity is checked like: `|nominal - actual| < tolerance` 57 | - "nan" and "nan" is equal 58 | - `0` difference with `0` nominal value is valid for any relative difference 59 | 60 | ### 2. Run the compare 61 | 62 | Running the comparison is super easy, just supply nominal, actual and the config: 63 | `./havocompare compare nominal_dir actual_dir config.yaml` 64 | The report of the comparison will be written inside the `./report` folder. Differences will also be printed to the 65 | terminal. 66 | Furthermore, if differences are found, the return code will be `1`, if no differences are found, it will be `0` making 67 | integration of 68 | havocompare into a CI system rather easy. 69 | 70 | ## Details on the config 71 | 72 | ### Validation Scheme 73 | 74 | Writing a valid configuration file can be error-prone without auto-completion. We suggest using json schema to validate 75 | your yaml 76 | and even enable auto-completion in IDEs like pycharm. To generate the schema you can call: 77 | `./havocompare schema > config_scheme.json` and import the resulting scheme into your IDE. 78 | 79 | ### Comparison options 80 | 81 | #### CSV 82 | 83 | The `comparison_modes` option is required and of type 'list'. It can comprise either a relative numerical ('Relative') 84 | maximum deviation or a maximum 85 | deviation ('Absolute'). 86 | You can specify the decimal separator and the field separator. If you don't specify, havocompare will try to guess it 87 | from each csv file. 88 | Note: If delimiters are not specified, even different delimiters between nominal and actual are accepted as long as all 89 | deviations are in bounds. 90 | To ignore specific cells, you can specify an exclusion regex. 91 | 92 | The preprocessing steps are done after the file is parsed using the given delimiters (or guessing) but before anything 93 | else. Processing order is as written in the list. 94 | In the below example, headers will be extracted from the csv-input file, then a column with the title "Column to delete" 95 | will be deleted. 96 | If any of the preprocessing steps fail, havocompare will exit with an error immediately so use them carefully. 97 | 98 | See the following example with all optional parameters set: 99 | 100 | ```yaml 101 | rules: 102 | - name: "CSV - Demo all options" 103 | # what files to include - use as many as make sense to reduce duplication in your rules 104 | pattern_include: 105 | - "**/*.csv" 106 | # optional: of all included files, remove the ones matching any exclude pattern 107 | pattern_exclude: 108 | - "**/ignored.csv" 109 | CSV: 110 | # delimiters are optional, if not given, they will be auto-detected. 111 | # auto-detection allows different delimiters for nominal and actual 112 | decimal_separator: '.' 113 | field_delimiter: ';' 114 | # can have Absolute or Relative or both 115 | comparison_modes: 116 | - Absolute: 1.0 117 | - Relative: 0.1 118 | # optional: exclude fields matching the regex from comparison 119 | exclude_field_regex: "Excluded" 120 | # optional: preprocessing of the csv files 121 | preprocessing: 122 | # extracts the headers to the header-fields, makes reports more legible and allows for further processing "ByName". 123 | # While it may fail, there's no penalty for it, as long as you don't rely on it. 124 | - ExtractHeaders 125 | # Sort the table by column 0, beware that the column must only contain numbers / quantities 126 | - SortByColumnNumber: 0 127 | # Delete a column by name, needs `ExtractHeaders` first - delete sets all values to 'DELETED' 128 | - DeleteColumnByName: "Vertex_Position_Y" 129 | - DeleteColumnByNumber: 1 130 | # Sorts are stable, so a second sort will keep the first sort as sub-order. 131 | - SortByColumnName: "Vertex_Position_X" 132 | # Deletes the first row by setting all values to 'DELETED' - meaning that numbering stays constant 133 | - DeleteRowByNumber: 0 134 | # Deletes rows having any element matching the given regex (may delete different lines in nom / act)! 135 | - DeleteRowByRegex: "Vertex_Count" 136 | # Deletes the cell (column, row) by setting the value to 'DELETED' 137 | - DeleteCellByNumber: 138 | column: 0 139 | row: 0 140 | # Deletes the cell (column name, row) by setting the value to 'DELETED'. This needs `ExtractHeaders` 141 | - DeleteCellByName: 142 | column: "Column to delete" 143 | row: 0 144 | ``` 145 | 146 | #### Image comparison 147 | 148 | Image comparison is done using the `image compare` crate. 149 | Specify loads of options here and then filter on threshold. 150 | 151 | ```yaml 152 | rules: 153 | - name: "JPG comparison" 154 | pattern_include: 155 | - "**/*.jpg" 156 | # exclude can of course also be specified! 157 | Image: 158 | # Compare images in RGBA-mode, can also be RGB and Gray 159 | # Comparison mode set to Hybrid means we want MSSIM on the Y channel and 2 dim vec diff on UV for color information 160 | RGBA: Hybrid 161 | threshold: 0.9 162 | ``` 163 | 164 | #### Plain text comparison 165 | 166 | For plain text comparison the file is read and compared line by line. For each line the normalized Damerau-Levenshtein 167 | distance from the `strsim` 168 | crate is used. You can ignore single lines which you know are different by specifying an arbitrary number of ignored 169 | lines: 170 | 171 | ```yaml 172 | rules: 173 | - name: "HTML-Compare strict" 174 | pattern_exclude: 175 | - "**/*_changed.html" 176 | pattern_include: 177 | - "**/*.html" 178 | PlainText: 179 | # Normalized Damerau-Levenshtein distance 180 | threshold: 1.0 181 | # All lines matching any regex below will be ignored 182 | ignore_lines: 183 | - "stylesheet" 184 | - "next_ignore" 185 | - "[A-Z]*[0-9]" 186 | ``` 187 | 188 | #### PDF text comparison 189 | 190 | For PDF text comparison the text will be extracted and written to temporary files. The files will then be compared using 191 | the Plain text comparison: 192 | 193 | ```yaml 194 | rules: 195 | - name: "PDF-Text-Compare" 196 | pattern_exclude: 197 | - "**/*_changed.pdf" 198 | pattern_include: 199 | - "**/*.pdf" 200 | PDFText: 201 | # Normalized Damerau-Levenshtein distance 202 | threshold: 1.0 203 | # All lines matching any regex below will be ignored 204 | ignore_lines: 205 | - "stylesheet" 206 | - "next_ignore" 207 | - "[A-Z]*[0-9]" 208 | ``` 209 | 210 | #### Hash comparison 211 | 212 | For binary files which cannot otherwise be checked we can also do a simple hash comparison. 213 | Currently, we only support SHA-256 but more checks can be added easily. 214 | 215 | ```yaml 216 | rules: 217 | - name: "Hash comparison strict" 218 | pattern_exclude: 219 | - "**/*.bin" 220 | Hash: 221 | # Currently we only have Sha256 222 | function: Sha256 223 | ``` 224 | 225 | #### File metadata comparison 226 | 227 | For the cases where the pure existence or some metadata are already enough. 228 | 229 | ```yaml 230 | rules: 231 | - name: "Metadata comparison" 232 | pattern_exclude: 233 | - "**/*.bin" 234 | FileProperties: 235 | # nom/act file paths must not contain whitespace 236 | forbid_name_regex: "[\\s]" 237 | # files must have their modification timestamp within 3600 seconds 238 | modification_date_tolerance_secs: 3600 239 | # files sizes must be within 1 kb 240 | file_size_tolerance_bytes: 1024 241 | ``` 242 | 243 | #### Run external comparison tool 244 | 245 | In case you want to run an external comparison tool, you can use this option 246 | 247 | ```yaml 248 | rules: 249 | - name: "External checker" 250 | pattern_include: 251 | - "*.pdf" 252 | External: 253 | # this config will call `/usr/bin/pdf-diff --only-images nominal.pdf actual.pdf` 254 | # return code will decide on comparison result 255 | executable: "/usr/bin/pdf-diff" 256 | # optional: add as many extra params as 257 | extra_params: 258 | - "--only-images" 259 | ``` 260 | 261 | #### JSON comparison 262 | 263 | Compares JSON files for different keys in both files and mismatches in values. 264 | `ignore_keys` is a list of regexes that are matched against the individual key names, the key value pair is excluded 265 | from the comparison if a regex matches. 266 | The values are not affected by this. 267 | 268 | ```yaml 269 | rules: 270 | - name: "Compare JSON files" 271 | pattern_include: 272 | - "**/*.json" 273 | Json: 274 | ignore_keys: 275 | # drop "ignore_this_key" and "ignore_this_keys" with this regex :) 276 | - "ignore_this_key(s?)" 277 | ``` 278 | 279 | #### File Exists / Directory Comparison 280 | 281 | Compares directory structure and file existence in both paths. 282 | 283 | ```yaml 284 | rules: 285 | - name: "Directory Checker" 286 | # to check directory structure only (ignoring the files) use "**/*/" and remove pattern_exclude 287 | # to check all files and directory use "**/*" and remove pattern_exclude 288 | # to check files only uses "**/*.*" but this works only in windows. Or use "**/*" and pattern_exclude "**/*/" 289 | pattern_include: 290 | - "**/*" 291 | pattern_exclude: 292 | - "**/*/" 293 | Directory: 294 | # Mode Identical to check whether both paths are really the same: whether entry is missing in actual, and/or if entry exists in actual but not in nominal 295 | # Mode MissingOnly to check only if entry is missing in actual, ignoring entries that exist in actual but not in nominal 296 | mode: Identical 297 | ``` 298 | 299 | ### Use HavoCompare in your unit-tests 300 | 301 | 1. Add havocompare to your dev-dependencies: 302 | ```toml 303 | [dev-dependencies] 304 | havocompare = "0.5" 305 | ``` 306 | 2. Use it in a unit-test like 307 | ```rust 308 | #[test] 309 | // works in 0.5 series 310 | fn integ_test_dirs() { 311 | let result_dir = process_whatever_test_data(); 312 | // just write the usual yaml file 313 | let result = havocompare::compare_folders("../tests/data/nominal/integ_test/case", &result_dir, "../tests/data/config.yaml", "../tests/out_report").unwrap; 314 | assert!(result); 315 | } 316 | #[test] 317 | // works starting with 0.5.3 only 318 | fn integ_test_file() { 319 | let result_file = process_generate_image(); 320 | // see docs for all options 321 | let compare_mode = ComparisonMode::Image(ImageCompareConfig{threshold: 0.97}); 322 | let result = havocompare::compare_files("../tests/data/nominal.png", &result_file, &compare_mode).unwrap; 323 | assert!(result); 324 | } 325 | ``` 326 | 327 | ## Changelog 328 | 329 | ### 0.8.0 330 | 331 | - Report will always be generated even though compare is failing. 332 | - Small additions: Add error message when files count don't match, use tracing::error instead of println 333 | - return error when: 334 | - folder can't be read in Directory/FileExist Checker 335 | - nominal and/or actual is not a folder instead of returning Ok(false) 336 | 337 | ### 0.7.0 338 | 339 | - add file exist checker 340 | - fix file property checker report that "Creation date" of "nominal" and "actual" columns are switched 341 | - update thiserror, itertools and pdf-extract crates 342 | 343 | ### 0.6.1 344 | 345 | - Add new version of json-diff, featuring a better API and better filtering options 346 | 347 | ### 0.6.0 348 | 349 | - Add new options for image compare module (a lot of options!) 350 | - Bump json-compare to new version fixing bugs in regex field excludes and sorting 351 | 352 | ### 0.5.4 353 | 354 | - Add option to run single file mode from CLI 355 | 356 | ### 0.5.3 357 | 358 | - Add option to sort json arrays (including deep sorting) 359 | - Make single file comparison function to public api 360 | - Update dependencies, fix broken pdf import regarding whitespaces 361 | 362 | ### 0.5.2 363 | 364 | - Preserve white spaces in CSV and PDF report instead of being collapsed by HTML 365 | - Add + and - symbols to the report index 366 | - Display combined file names in report index if the compared file names are different 367 | 368 | ### 0.5.1 369 | 370 | - Fix potential crash in JSON checking 371 | 372 | ### 0.5.0 373 | 374 | - Add JSON checking 375 | 376 | ### 0.4.0 377 | 378 | - Separate reporting logic from comparison logic 379 | - Implement a machine-readable JSON reporting 380 | 381 | ### 0.3.2 382 | 383 | - Allow direct opening of reports after comparison with `--open` 384 | - Parsing failures when running `compare` are now propagated to terminal 385 | 386 | ### 0.3.1 387 | 388 | - Fix swapped actual and nominal for hash and text-compares 389 | 390 | ### 0.3.0 391 | 392 | - Allow RGBA image comparison 393 | - Add file metadata comparison 394 | - Add external checking 395 | - Add and adjust reports for file metadata comparison and external checking 396 | - Add csv header comparison. Previously, the report only marks the differences but not returning any error. 397 | - Added config yaml validation command to check whether file can be loaded or not 398 | 399 | ### 0.2.4 400 | 401 | - add check for row lines of both compared csv files, and throw error if they are unequal 402 | - Add deletion by cell 403 | - Simplify report sub-folders creation: sub-folders are now created temporarily in the temp folder instead of in the 404 | current working folder 405 | - Change report row numbering to always start with 0, so row deletion is more understandable 406 | - fix floating point value comparison of non-displayable diff values 407 | 408 | ### 0.2.3 409 | 410 | - bump pdf-extract crate to 0.6.4 to fix "'attempted to leave 411 | type `linked_hash_map::Node, object::Object>` uninitialized'" 412 | 413 | ### 0.2.2 414 | 415 | - Include files which has error and can't be compared to the report 416 | - Fixed a bug which caused the program exited early out of rules-loop, and not processing all 417 | 418 | ### 0.2.0 419 | 420 | - Deletion of columns will no longer really delete them but replace every value with "DELETED" 421 | - Expose config struct to library API 422 | - Fixed a bug regarding wrong handling of multiple empty lines 423 | - Reworked CSV reporting to have an interleaved and more compact view 424 | - Display the relative path of compared files instead of file name in the report index.html 425 | - Made header-extraction fallible but uncritical - can now always be enabled 426 | - Wrote a completely new csv parser: 427 | - Respects escaping with '\' 428 | - Allows string-literals containing unescaped field separators (field1, "field2, but as literal", field3) 429 | - Allows multi-line string literals with quotes 430 | - CSVs with non-rectangular format will now fail 431 | 432 | ### 0.1.4 433 | 434 | - Add multiple includes and excludes - warning, this will break rules-files from 0.1.3 and earlier 435 | - Remove all `unwrap` and `expect` in the library code in favor of correct error propagation 436 | - Add preprocessing options for CSV files 437 | - Refined readme.md 438 | - fix unique key creation in the report generation 439 | - Add PDF-Text compare 440 | 441 | ### 0.1.3: 442 | 443 | - Add optional cli argument to configure the folder to store the report 444 | 445 | ### 0.1.2: 446 | 447 | - Add SHA-256 comparison mode 448 | - Fix BOM on windows for CSV comparison 449 | 450 | ### 0.1.1: 451 | 452 | - Better error message on folder not found 453 | - Better test coverage 454 | - Fix colors on Windows terminal 455 | - Extend CI to Windows and mac 456 | -------------------------------------------------------------------------------- /src/csv/tokenizer/mod.rs: -------------------------------------------------------------------------------- 1 | use super::Error; 2 | use crate::csv::tokenizer::guess_format::guess_format_from_reader; 3 | use crate::csv::value::Value; 4 | use crate::csv::Delimiters; 5 | use std::cmp::Ordering; 6 | use std::io::{Read, Seek}; 7 | use tracing::debug; 8 | 9 | mod guess_format; 10 | const BOM: char = '\u{feff}'; 11 | const DEFAULT_FIELD_SEPARATOR: char = ','; 12 | const ESCAPE_BYTE: u8 = b'\\'; 13 | const ESCAPE_QUOTE_BYTE: u8 = b'"'; 14 | const QUOTE: char = '\"'; 15 | const NEW_LINE: char = '\n'; 16 | const CARRIAGE_RETURN: char = '\r'; 17 | 18 | #[derive(PartialEq, Eq, Debug)] 19 | pub enum Token<'a> { 20 | Field(&'a str), 21 | LineBreak, 22 | } 23 | 24 | #[derive(PartialEq, Eq, Debug, Clone, Copy)] 25 | enum LiteralTerminator { 26 | Quote, 27 | } 28 | 29 | impl LiteralTerminator { 30 | pub fn get_char(&self) -> char { 31 | match self { 32 | LiteralTerminator::Quote => QUOTE, 33 | } 34 | } 35 | } 36 | 37 | #[derive(PartialEq, Eq, Debug)] 38 | enum SpecialCharacter { 39 | NewLine(usize), 40 | LiteralMarker(usize, LiteralTerminator), 41 | FieldStop(usize, char), 42 | } 43 | 44 | impl SpecialCharacter { 45 | pub fn get_position(&self) -> usize { 46 | match self { 47 | SpecialCharacter::NewLine(pos) => *pos, 48 | SpecialCharacter::LiteralMarker(pos, _) => *pos, 49 | SpecialCharacter::FieldStop(pos, _) => *pos, 50 | } 51 | } 52 | 53 | pub fn len(&self) -> usize { 54 | match self { 55 | SpecialCharacter::NewLine(_) => NEW_LINE.len_utf8(), 56 | SpecialCharacter::FieldStop(_, pat) => pat.len_utf8(), 57 | SpecialCharacter::LiteralMarker(_, marker) => marker.get_char().len_utf8(), 58 | } 59 | } 60 | 61 | #[cfg(test)] 62 | pub fn quote(pos: usize) -> SpecialCharacter { 63 | SpecialCharacter::LiteralMarker(pos, LiteralTerminator::Quote) 64 | } 65 | } 66 | 67 | impl PartialOrd for SpecialCharacter { 68 | fn partial_cmp(&self, other: &Self) -> Option { 69 | Some(self.cmp(other)) 70 | } 71 | } 72 | 73 | impl Ord for SpecialCharacter { 74 | fn cmp(&self, other: &Self) -> Ordering { 75 | self.get_position().cmp(&other.get_position()) 76 | } 77 | } 78 | 79 | fn find_next_unescaped(string: &str, pat: char) -> Option { 80 | let pos = string.find(pat); 81 | if let Some(pos) = pos { 82 | if pos > 0 { 83 | if let Some(byte_before) = string.as_bytes().get(pos - 1) { 84 | if *byte_before == ESCAPE_BYTE { 85 | let remainder = &string[pos + pat.len_utf8()..]; 86 | return find_next_unescaped(remainder, pat) 87 | .map(|ipos| ipos + pos + pat.len_utf8()); 88 | } 89 | } 90 | } 91 | if pos < string.len() - 1 && pat == QUOTE { 92 | if let Some(byte_after) = string.as_bytes().get(pos + 1) { 93 | if *byte_after == ESCAPE_QUOTE_BYTE { 94 | let new_start_offset = pos + pat.len_utf8() + 1; 95 | let remainder = &string[new_start_offset..]; 96 | return find_next_unescaped(remainder, pat).map(|ipos| ipos + new_start_offset); 97 | } 98 | } 99 | } 100 | Some(pos) 101 | } else { 102 | None 103 | } 104 | } 105 | 106 | fn find_literal(string: &str, terminator: LiteralTerminator) -> Option { 107 | find_next_unescaped(string, terminator.get_char()) 108 | .map(|p| SpecialCharacter::LiteralMarker(p, terminator)) 109 | } 110 | 111 | fn find_new_line(string: &str) -> Option { 112 | find_next_unescaped(string, NEW_LINE).map(SpecialCharacter::NewLine) 113 | } 114 | 115 | fn find_field_stop(string: &str, field_sep: char) -> Option { 116 | find_next_unescaped(string, field_sep).map(|p| SpecialCharacter::FieldStop(p, field_sep)) 117 | } 118 | 119 | fn find_special_char(string: &str, field_sep: char) -> Option { 120 | for (pos, chr) in string.char_indices() { 121 | if pos > 0 && string.as_bytes()[pos - 1] == ESCAPE_BYTE { 122 | continue; 123 | } 124 | match chr { 125 | QUOTE => { 126 | return Some(SpecialCharacter::LiteralMarker( 127 | pos, 128 | LiteralTerminator::Quote, 129 | )); 130 | } 131 | NEW_LINE => { 132 | return Some(SpecialCharacter::NewLine(pos)); 133 | } 134 | other if other == field_sep => { 135 | return Some(SpecialCharacter::FieldStop(pos, field_sep)); 136 | } 137 | _ => continue, 138 | } 139 | } 140 | None 141 | } 142 | 143 | struct RowBuffer(Vec>); 144 | impl RowBuffer { 145 | pub fn new() -> RowBuffer { 146 | RowBuffer(vec![Vec::new()]) 147 | } 148 | 149 | pub fn push_field(&mut self, value: Value) { 150 | if let Some(current_row) = self.0.last_mut() { 151 | current_row.push(value); 152 | } 153 | } 154 | 155 | pub fn new_row(&mut self) { 156 | self.0.push(Vec::new()); 157 | } 158 | 159 | pub fn into_iter(mut self) -> std::vec::IntoIter> { 160 | self.trim_end(); 161 | self.0.into_iter() 162 | } 163 | 164 | fn trim_end(&mut self) { 165 | 'PopEmpty: loop { 166 | if let Some(back) = self.0.last() { 167 | if back.len() <= 1 { 168 | if let Some(first) = back.first() { 169 | if first.as_str().is_empty() { 170 | self.0.pop(); 171 | } else { 172 | break 'PopEmpty; 173 | } 174 | } else { 175 | self.0.pop(); 176 | } 177 | } else { 178 | break 'PopEmpty; 179 | } 180 | } 181 | } 182 | } 183 | } 184 | 185 | pub(crate) struct Parser { 186 | reader: R, 187 | delimiters: Delimiters, 188 | } 189 | 190 | fn tokenize(input: &str, field_sep: char) -> Result, Error> { 191 | let mut tokens = Vec::new(); 192 | let mut pos = 0; 193 | while let Some(remainder) = &input.get(pos..) { 194 | if let Some(special_char) = find_special_char(remainder, field_sep) { 195 | let mut end_pos = special_char.get_position(); 196 | match &special_char { 197 | SpecialCharacter::FieldStop(_, _) => { 198 | tokens.push(Token::Field(&remainder[..end_pos])); 199 | } 200 | SpecialCharacter::NewLine(_) => { 201 | let field_value = &remainder[..end_pos].trim(); 202 | tokens.push(Token::Field(field_value)); 203 | tokens.push(Token::LineBreak); 204 | } 205 | SpecialCharacter::LiteralMarker(_, terminator) => { 206 | let (literal_end_pos, token, break_line) = 207 | parse_literal(field_sep, remainder, *terminator)?; 208 | tokens.push(token); 209 | if break_line { 210 | tokens.push(Token::LineBreak); 211 | } 212 | end_pos += literal_end_pos; 213 | } 214 | }; 215 | pos += end_pos + special_char.len(); 216 | } else { 217 | break; 218 | } 219 | } 220 | 221 | if pos <= input.len() { 222 | tokens.push(Token::Field(&input[pos..])); 223 | } 224 | Ok(tokens) 225 | } 226 | 227 | fn parse_literal( 228 | field_sep: char, 229 | remainder: &str, 230 | literal_type: LiteralTerminator, 231 | ) -> Result<(usize, Token, bool), Error> { 232 | let terminator_len = literal_type.get_char().len_utf8(); 233 | let after_first_quote = &remainder[terminator_len..]; 234 | let quote_end = 235 | find_literal(after_first_quote, literal_type).ok_or(Error::UnterminatedLiteral)?; 236 | let after_second_quote_in_remainder = quote_end.get_position() + 2 * terminator_len; 237 | let inner_remainder = &remainder[after_second_quote_in_remainder..]; 238 | let field_end = find_field_stop(inner_remainder, field_sep) 239 | .map(|sc| sc.get_position()) 240 | .unwrap_or(inner_remainder.len()); 241 | let line_end = find_new_line(inner_remainder) 242 | .map(|sc| sc.get_position()) 243 | .unwrap_or(inner_remainder.len()); 244 | if line_end < field_end { 245 | let token = Token::Field(&remainder[..after_second_quote_in_remainder]); 246 | Ok((after_second_quote_in_remainder, token, true)) 247 | } else { 248 | let token = Token::Field(&remainder[..after_second_quote_in_remainder + field_end]); 249 | Ok((after_second_quote_in_remainder + field_end, token, false)) 250 | } 251 | } 252 | 253 | impl Parser { 254 | pub fn new_guess_format(mut reader: R) -> Result { 255 | guess_format_from_reader(&mut reader).map(|delimiters| Parser { reader, delimiters }) 256 | } 257 | 258 | pub fn new(reader: R, delimiters: Delimiters) -> Option { 259 | delimiters.field_delimiter?; 260 | Some(Parser { reader, delimiters }) 261 | } 262 | 263 | pub(crate) fn parse_to_rows(&mut self) -> Result>, Error> { 264 | debug!( 265 | "Generating tokens with field delimiter: {:?}", 266 | self.delimiters.field_delimiter 267 | ); 268 | 269 | let mut string_buffer = String::new(); 270 | self.reader.read_to_string(&mut string_buffer)?; 271 | // remove BoM & windows line endings to linux line endings 272 | string_buffer.retain(|c| ![BOM, CARRIAGE_RETURN].contains(&c)); 273 | let field_sep = self 274 | .delimiters 275 | .field_delimiter 276 | .unwrap_or(DEFAULT_FIELD_SEPARATOR); 277 | 278 | let mut buffer = RowBuffer::new(); 279 | 280 | tokenize(string_buffer.as_str(), field_sep)? 281 | .into_iter() 282 | .for_each(|t| match t { 283 | Token::Field(input_str) => { 284 | buffer.push_field(Value::from_str( 285 | input_str, 286 | &self.delimiters.decimal_separator, 287 | )); 288 | } 289 | Token::LineBreak => buffer.new_row(), 290 | }); 291 | 292 | Ok(buffer.into_iter()) 293 | } 294 | } 295 | 296 | #[cfg(test)] 297 | mod tokenizer_tests { 298 | use super::*; 299 | use std::fs::File; 300 | use std::io::Cursor; 301 | 302 | #[test] 303 | fn unescaped() { 304 | let str = "...\\,...,"; 305 | let next = find_next_unescaped(str, ',').unwrap(); 306 | assert_eq!(next, 8); 307 | } 308 | 309 | #[test] 310 | fn next_special_char_finds_first_quote() { 311 | let str = ".....\"..',."; 312 | let next = find_special_char(str, ',').unwrap(); 313 | assert_eq!(next, SpecialCharacter::quote(5)); 314 | } 315 | 316 | #[test] 317 | fn next_special_char_finds_first_unescaped_quote() { 318 | let str = r#"..\"."..',."#; 319 | let next = find_special_char(str, ',').unwrap(); 320 | assert_eq!(next, SpecialCharacter::quote(5)); 321 | } 322 | 323 | #[test] 324 | fn tokenization_simple() { 325 | let str = "bla,blubb,2.0"; 326 | let mut tokens = tokenize(str, ',').unwrap(); 327 | assert_eq!(tokens.len(), 3); 328 | assert_eq!(tokens.pop().unwrap(), Token::Field("2.0")); 329 | assert_eq!(tokens.pop().unwrap(), Token::Field("blubb")); 330 | assert_eq!(tokens.pop().unwrap(), Token::Field("bla")); 331 | } 332 | 333 | #[test] 334 | fn tokenization_simple_last_field_empty() { 335 | let str = "bla,\nblubb,"; 336 | let mut tokens = tokenize(str, ',').unwrap(); 337 | assert_eq!(tokens.len(), 5); 338 | assert_eq!(tokens.pop().unwrap(), Token::Field("")); 339 | assert_eq!(tokens.pop().unwrap(), Token::Field("blubb")); 340 | assert_eq!(tokens.pop().unwrap(), Token::LineBreak); 341 | assert_eq!(tokens.pop().unwrap(), Token::Field("")); 342 | assert_eq!(tokens.pop().unwrap(), Token::Field("bla")); 343 | } 344 | 345 | #[test] 346 | fn tokenization_with_literals() { 347 | let str = r#"bla,"bla,bla",2.0"#; 348 | let mut tokens = tokenize(str, ',').unwrap(); 349 | assert_eq!(tokens.len(), 3); 350 | assert_eq!(tokens.pop().unwrap(), Token::Field("2.0")); 351 | assert_eq!(tokens.pop().unwrap(), Token::Field("\"bla,bla\"")); 352 | assert_eq!(tokens.pop().unwrap(), Token::Field("bla")); 353 | } 354 | 355 | #[test] 356 | fn tokenization_of_unterminated_literal_errors() { 357 | let str = r#"bla,"There is no termination"#; 358 | let tokens = tokenize(str, ','); 359 | assert!(matches!(tokens.unwrap_err(), Error::UnterminatedLiteral)); 360 | } 361 | 362 | #[test] 363 | fn tokenization_of_literals_and_spaces() { 364 | let str = r#"bla, "literally""#; 365 | let mut tokens = tokenize(str, ',').unwrap(); 366 | assert_eq!(tokens.len(), 2); 367 | assert_eq!(tokens.pop().unwrap(), Token::Field(" \"literally\"")); 368 | assert_eq!(tokens.pop().unwrap(), Token::Field("bla")); 369 | } 370 | 371 | #[test] 372 | fn tokenization_literals_at_line_end() { 373 | let str = r#"bla,"bla,bla" 374 | bla,bla"#; 375 | let mut tokens = tokenize(str, ',').unwrap(); 376 | assert_eq!(tokens.pop().unwrap(), Token::Field("bla")); 377 | assert_eq!(tokens.pop().unwrap(), Token::Field("bla")); 378 | assert_eq!(tokens.pop().unwrap(), Token::LineBreak); 379 | assert_eq!(tokens.pop().unwrap(), Token::Field("\"bla,bla\"")); 380 | assert_eq!(tokens.pop().unwrap(), Token::Field("bla")); 381 | } 382 | 383 | #[test] 384 | fn tokenization_with_multi_line_literals() { 385 | let str = "bla,\"bla\nbla\",2.0"; 386 | let mut tokens = tokenize(str, ',').unwrap(); 387 | assert_eq!(tokens.len(), 3); 388 | assert_eq!(tokens.pop().unwrap(), Token::Field("2.0")); 389 | assert_eq!(tokens.pop().unwrap(), Token::Field("\"bla\nbla\"")); 390 | assert_eq!(tokens.pop().unwrap(), Token::Field("bla")); 391 | } 392 | 393 | #[test] 394 | fn tokenize_to_values_cuts_last_nl() { 395 | let str = "bla\n2.0\n\n"; 396 | let mut parser = Parser::new_guess_format(Cursor::new(str)).unwrap(); 397 | let lines = parser.parse_to_rows().unwrap(); 398 | assert_eq!(lines.len(), 2); 399 | } 400 | 401 | #[test] 402 | fn tokenization_with_multi_line_with_escape_break_literals() { 403 | let str = "\\\"bla,\"'bla\\\"\nbla'\",2.0"; 404 | let mut tokens = tokenize(str, ',').unwrap(); 405 | assert_eq!(tokens.len(), 3); 406 | assert_eq!(tokens.pop().unwrap(), Token::Field("2.0")); 407 | assert_eq!(tokens.pop().unwrap(), Token::Field("\"'bla\\\"\nbla'\"")); 408 | assert_eq!(tokens.pop().unwrap(), Token::Field("\\\"bla")); 409 | } 410 | 411 | #[test] 412 | fn find_chars_unicode_with_utf8() { 413 | let str = r#"mm²,Area"#; 414 | let pos = find_next_unescaped(str, ',').unwrap(); 415 | assert_eq!(pos, 4); 416 | } 417 | 418 | #[test] 419 | fn find_next_unescaped_field_after_utf8_multibyte_char() { 420 | let str = r#"mm²,Area"#; 421 | let pos = find_next_unescaped(str, ',').unwrap(); 422 | assert_eq!(pos, 4); 423 | } 424 | 425 | #[test] 426 | fn tokenization_windows_newlines() { 427 | let str = "bla\n\rbla"; 428 | let mut tokens = Parser::new( 429 | Cursor::new(str), 430 | Delimiters { 431 | field_delimiter: Some(','), 432 | decimal_separator: None, 433 | }, 434 | ) 435 | .unwrap() 436 | .parse_to_rows() 437 | .unwrap(); 438 | assert_eq!(tokens.len(), 2); 439 | assert_eq!( 440 | *tokens.next().unwrap().first().unwrap(), 441 | Value::from_str("bla", &None) 442 | ); 443 | assert_eq!( 444 | *tokens.next().unwrap().first().unwrap(), 445 | Value::from_str("bla", &None) 446 | ); 447 | } 448 | 449 | #[test] 450 | fn tokenization_new_lines() { 451 | let str = "bla,bla\nbla,bla"; 452 | let mut tokens = tokenize(str, ',').unwrap(); 453 | assert_eq!(tokens.len(), 5); 454 | assert_eq!(tokens.pop().unwrap(), Token::Field("bla")); 455 | assert_eq!(tokens.pop().unwrap(), Token::Field("bla")); 456 | assert_eq!(tokens.pop().unwrap(), Token::LineBreak); 457 | assert_eq!(tokens.pop().unwrap(), Token::Field("bla")); 458 | assert_eq!(tokens.pop().unwrap(), Token::Field("bla")); 459 | } 460 | 461 | #[test] 462 | fn tokenizer_smoke() { 463 | let actual = File::open( 464 | "tests/integ/data/display_of_status_message_in_cm_tables/actual/Volume1.csv", 465 | ) 466 | .unwrap(); 467 | let mut parser = Parser::new_guess_format(actual).unwrap(); 468 | parser.parse_to_rows().unwrap(); 469 | let nominal = File::open( 470 | "tests/integ/data/display_of_status_message_in_cm_tables/expected/Volume1.csv", 471 | ) 472 | .unwrap(); 473 | let mut parser = Parser::new_guess_format(nominal).unwrap(); 474 | parser.parse_to_rows().unwrap(); 475 | } 476 | 477 | #[test] 478 | fn tokenizer_semicolon_test() { 479 | let nominal = 480 | File::open("tests/csv/data/easy_pore_export_annoration_table_result.csv").unwrap(); 481 | let mut parser = Parser::new_guess_format(nominal).unwrap(); 482 | for line in parser.parse_to_rows().unwrap() { 483 | assert_eq!(line.len(), 5); 484 | } 485 | } 486 | 487 | #[test] 488 | fn special_quote_escape_works() { 489 | let str = r#"""..""#; 490 | let quote = find_next_unescaped(str, '\"').unwrap(); 491 | assert_eq!(quote, 4); 492 | } 493 | 494 | #[test] 495 | fn special_quote_escape_works_complicated() { 496 | let str = r#"""Scene""=>""Mesh 1""""#; 497 | let (pos, _, _) = parse_literal(',', str, LiteralTerminator::Quote).unwrap(); 498 | assert_eq!(pos, 22); 499 | } 500 | 501 | #[test] 502 | fn tokenize_complicated_literal_smoke() { 503 | let str = r#"Deviation interval A [mm],-0.6 504 | Deviation interval B [mm],0.6 505 | Actual object,"""Scene""=>""Volume 1""=>""Merged region""" 506 | Nominal object,"""Scene""=>""Mesh 1""""#; 507 | let lines = Parser::new_guess_format(Cursor::new(str)) 508 | .unwrap() 509 | .parse_to_rows() 510 | .unwrap(); 511 | for line in lines { 512 | assert_eq!(line.len(), 2); 513 | } 514 | } 515 | } 516 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![crate_name = "havocompare"] 2 | //! # Comparing folders and files by rules 3 | //! Havocompare allows to compare folders (or to be more exact: the files inside the folders) following user definable rules. 4 | //! A self contained html report is generated. To use it without the CLI, the main method is: [`compare_folders`]. 5 | //! 6 | #![warn(missing_docs)] 7 | #![warn(unused_qualifications)] 8 | #![deny(deprecated)] 9 | #![deny(clippy::unwrap_used)] 10 | #![deny(clippy::expect_used)] 11 | 12 | use schemars::schema_for; 13 | use schemars_derive::JsonSchema; 14 | use serde::{Deserialize, Serialize}; 15 | use std::borrow::Cow; 16 | use std::fs::File; 17 | use std::io::{BufReader, Read}; 18 | use std::path::{Path, PathBuf}; 19 | use thiserror::Error; 20 | use tracing::{debug, error, info, span}; 21 | use vg_errortools::{fat_io_wrap_std, FatIOError}; 22 | 23 | pub use csv::CSVCompareConfig; 24 | pub use hash::HashConfig; 25 | 26 | use crate::directory::DirectoryConfig; 27 | use crate::external::ExternalConfig; 28 | pub use crate::html::HTMLCompareConfig; 29 | pub use crate::image::ImageCompareConfig; 30 | pub use crate::json::JsonConfig; 31 | use crate::properties::PropertiesConfig; 32 | use crate::report::{get_relative_path, DiffDetail, Difference}; 33 | 34 | /// comparison module for csv comparison 35 | pub mod csv; 36 | 37 | mod directory; 38 | mod external; 39 | mod hash; 40 | mod html; 41 | mod image; 42 | mod pdf; 43 | mod properties; 44 | mod report; 45 | 46 | mod json; 47 | 48 | #[derive(Error, Debug)] 49 | /// Top-Level Error class for all errors that can happen during havocompare-running 50 | pub enum Error { 51 | /// Pattern used for globbing was invalid 52 | #[error("Failed to evaluate globbing pattern! {0}")] 53 | IllegalGlobbingPattern(#[from] glob::PatternError), 54 | /// Regex pattern requested could not be compiled 55 | #[error("Failed to compile regex! {0}")] 56 | RegexCompilationError(#[from] regex::Error), 57 | /// An error occurred in the csv rule checker 58 | #[error("CSV module error")] 59 | CSVModuleError(#[from] csv::Error), 60 | /// An error occurred in the image rule checker 61 | #[error("Image module error")] 62 | ImageModuleError(#[from] image::Error), 63 | /// An error occurred in the directory/file exists rule checker 64 | #[error("Image module error")] 65 | DirectoryModuleError(#[from] directory::Error), 66 | 67 | /// An error occurred in the reporting module 68 | #[error("Error occurred during report creation {0}")] 69 | ReportingError(#[from] report::Error), 70 | /// An error occurred during reading yaml 71 | #[error("Serde error, loading a yaml: {0}")] 72 | SerdeYamlFail(#[from] serde_yaml::Error), 73 | /// An error occurred during writing json 74 | #[error("Serde error, writing json: {0}")] 75 | SerdeJsonFail(#[from] serde_json::Error), 76 | /// A problem happened while accessing a file 77 | #[error("File access failed {0}")] 78 | FileAccessError(#[from] FatIOError), 79 | 80 | /// could not extract filename from path 81 | #[error("File path parsing failed")] 82 | FilePathParsingFails(String), 83 | 84 | /// Different number of files matched pattern in actual and nominal 85 | #[error("Different number of files matched pattern in actual {0} and nominal {1}")] 86 | DifferentNumberOfFiles(usize, usize), 87 | 88 | /// Different number of files matched pattern in actual and nominal 89 | #[error("{0} is not a directory")] 90 | NotDirectory(String), 91 | } 92 | 93 | #[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)] 94 | #[allow(clippy::upper_case_acronyms)] 95 | /// Representing the comparison mode 96 | pub enum ComparisonMode { 97 | /// smart CSV compare 98 | CSV(CSVCompareConfig), 99 | /// thresholds comparison 100 | Image(ImageCompareConfig), 101 | /// plain text compare 102 | PlainText(HTMLCompareConfig), 103 | /// Compare using file hashes 104 | Hash(HashConfig), 105 | /// PDF text compare 106 | PDFText(HTMLCompareConfig), 107 | /// Compare file-properties 108 | FileProperties(PropertiesConfig), 109 | 110 | /// Compare JSON files 111 | Json(JsonConfig), 112 | 113 | /// Run external comparison executable 114 | External(ExternalConfig), 115 | 116 | /// File exists / directory structure checker 117 | Directory(DirectoryConfig), 118 | } 119 | 120 | fn get_file_name(path: &Path) -> Option> { 121 | path.file_name().map(|f| f.to_string_lossy()) 122 | } 123 | 124 | #[derive(Debug, Deserialize, Serialize, JsonSchema)] 125 | /// Represents a whole configuration file consisting of several comparison rules 126 | pub struct ConfigurationFile { 127 | /// A list of all rules to be checked on run 128 | pub rules: Vec, 129 | } 130 | 131 | impl ConfigurationFile { 132 | /// creates a [`ConfigurationFile`] file struct from anything implementing `Read` 133 | pub fn from_reader(reader: impl Read) -> Result { 134 | let config: ConfigurationFile = serde_yaml::from_reader(reader)?; 135 | Ok(config) 136 | } 137 | 138 | /// creates a [`ConfigurationFile`] from anything path-convertible 139 | pub fn from_file(file: impl AsRef) -> Result { 140 | let config_reader = fat_io_wrap_std(file, &File::open)?; 141 | Self::from_reader(BufReader::new(config_reader)) 142 | } 143 | } 144 | 145 | #[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)] 146 | /// Representing a single comparison rule 147 | pub struct Rule { 148 | /// The name of the rule - will be displayed in logs 149 | pub name: String, 150 | /// A list of glob-patterns to include 151 | pub pattern_include: Vec, 152 | /// A list of glob-patterns to exclude - optional 153 | pub pattern_exclude: Option>, 154 | /// How these files shall be compared 155 | #[serde(flatten)] 156 | pub file_type: ComparisonMode, 157 | } 158 | 159 | fn glob_files( 160 | path: impl AsRef, 161 | patterns: &[impl AsRef], 162 | ) -> Result, glob::PatternError> { 163 | let mut files = Vec::new(); 164 | for pattern in patterns { 165 | let path_prefix = path.as_ref().join(pattern.as_ref()); 166 | let path_pattern = path_prefix.to_string_lossy(); 167 | debug!("Globbing: {}", path_pattern); 168 | files.extend(glob::glob(path_pattern.as_ref())?.filter_map(|p| p.ok())); 169 | } 170 | Ok(files) 171 | } 172 | 173 | fn filter_exclude(paths: Vec, excludes: Vec) -> Vec { 174 | debug!( 175 | "Filtering paths {:#?} with exclusion list {:#?}", 176 | &paths, &excludes 177 | ); 178 | paths 179 | .into_iter() 180 | .filter_map(|p| if excludes.contains(&p) { None } else { Some(p) }) 181 | .collect() 182 | } 183 | 184 | /// Use this to compare a single file against another file using a given rule 185 | pub fn compare_files( 186 | nominal: impl AsRef, 187 | actual: impl AsRef, 188 | comparison_mode: &ComparisonMode, 189 | ) -> Difference { 190 | let file_name_nominal = nominal.as_ref().to_string_lossy(); 191 | let file_name_actual = actual.as_ref().to_string_lossy(); 192 | let _file_span = span!(tracing::Level::INFO, "Processing").entered(); 193 | 194 | info!("File: {file_name_nominal} | {file_name_actual}"); 195 | 196 | let compare_result: Result> = { 197 | match comparison_mode { 198 | ComparisonMode::CSV(conf) => { 199 | csv::compare_paths(nominal.as_ref(), actual.as_ref(), conf).map_err(|e| e.into()) 200 | } 201 | ComparisonMode::Image(conf) => { 202 | image::compare_paths(nominal.as_ref(), actual.as_ref(), conf).map_err(|e| e.into()) 203 | } 204 | ComparisonMode::PlainText(conf) => { 205 | html::compare_files(nominal.as_ref(), actual.as_ref(), conf).map_err(|e| e.into()) 206 | } 207 | ComparisonMode::Hash(conf) => { 208 | hash::compare_files(nominal.as_ref(), actual.as_ref(), conf).map_err(|e| e.into()) 209 | } 210 | ComparisonMode::PDFText(conf) => { 211 | pdf::compare_files(nominal.as_ref(), actual.as_ref(), conf).map_err(|e| e.into()) 212 | } 213 | ComparisonMode::FileProperties(conf) => { 214 | properties::compare_files(nominal.as_ref(), actual.as_ref(), conf) 215 | .map_err(|e| e.into()) 216 | } 217 | ComparisonMode::External(conf) => { 218 | external::compare_files(nominal.as_ref(), actual.as_ref(), conf) 219 | .map_err(|e| e.into()) 220 | } 221 | ComparisonMode::Json(conf) => { 222 | json::compare_files(nominal.as_ref(), actual.as_ref(), conf).map_err(|e| e.into()) 223 | } 224 | ComparisonMode::Directory(conf) => { 225 | let pattern = ["**/*"]; 226 | let exclude_pattern: Vec = Vec::new(); 227 | match get_files(nominal.as_ref(), &pattern, &exclude_pattern) { 228 | Ok(n) => match get_files(actual.as_ref(), &pattern, &exclude_pattern) { 229 | Ok(a) => directory::compare_paths( 230 | nominal.as_ref(), 231 | actual.as_ref(), 232 | &n, 233 | &a, 234 | conf, 235 | ) 236 | .map_err(|e| e.into()), 237 | Err(e) => Err(e.into()), 238 | }, 239 | Err(e) => Err(e.into()), 240 | } 241 | } 242 | } 243 | }; 244 | let compare_result = match compare_result { 245 | Ok(r) => r, 246 | Err(e) => { 247 | let e = e.to_string(); 248 | error!("Problem comparing the files {}", &e); 249 | let mut d = Difference::new_for_file(nominal, actual); 250 | d.error(); 251 | d.push_detail(DiffDetail::Error(e)); 252 | d 253 | } 254 | }; 255 | 256 | if compare_result.is_error { 257 | error!("Files didn't match"); 258 | } else { 259 | debug!("Files matched"); 260 | } 261 | 262 | compare_result 263 | } 264 | 265 | pub(crate) fn get_files( 266 | path: impl AsRef, 267 | patterns_include: &[impl AsRef], 268 | patterns_exclude: &[impl AsRef], 269 | ) -> Result, glob::PatternError> { 270 | let files_exclude = glob_files(path.as_ref(), patterns_exclude)?; 271 | let files_include: Vec<_> = glob_files(path.as_ref(), patterns_include)?; 272 | Ok(filter_exclude(files_include, files_exclude)) 273 | } 274 | 275 | fn process_rule( 276 | nominal: impl AsRef, 277 | actual: impl AsRef, 278 | rule: &Rule, 279 | compare_results: &mut Vec, 280 | ) -> Result { 281 | let _file_span = span!(tracing::Level::INFO, "Rule").entered(); 282 | info!("Name: {}", rule.name.as_str()); 283 | if !nominal.as_ref().is_dir() { 284 | error!( 285 | "Nominal folder {} is not a folder", 286 | nominal.as_ref().to_string_lossy() 287 | ); 288 | return Err(Error::NotDirectory( 289 | nominal.as_ref().to_string_lossy().to_string(), 290 | )); 291 | } 292 | if !actual.as_ref().is_dir() { 293 | error!( 294 | "Actual folder {} is not a folder", 295 | actual.as_ref().to_string_lossy() 296 | ); 297 | return Err(Error::NotDirectory( 298 | actual.as_ref().to_string_lossy().to_string(), 299 | )); 300 | } 301 | 302 | let exclude_patterns = rule.pattern_exclude.as_deref().unwrap_or_default(); 303 | 304 | let nominal_cleaned_paths = 305 | get_files(nominal.as_ref(), &rule.pattern_include, exclude_patterns)?; 306 | let actual_cleaned_paths = get_files(actual.as_ref(), &rule.pattern_include, exclude_patterns)?; 307 | 308 | let mut all_okay = true; 309 | match &rule.file_type { 310 | ComparisonMode::Directory(config) => { 311 | match directory::compare_paths( 312 | nominal.as_ref(), 313 | actual.as_ref(), 314 | &nominal_cleaned_paths, 315 | &actual_cleaned_paths, 316 | config, 317 | ) { 318 | Ok(diff) => { 319 | all_okay = !diff.is_error; 320 | compare_results.push(diff); 321 | } 322 | Err(e) => { 323 | error!("Problem comparing the files {}", &e); 324 | return Err(e.into()); 325 | } 326 | } 327 | } 328 | _ => { 329 | info!( 330 | "Found {} files matching includes in actual, {} files in nominal", 331 | actual_cleaned_paths.len(), 332 | nominal_cleaned_paths.len() 333 | ); 334 | let actual_files = actual_cleaned_paths.len(); 335 | let nominal_files = nominal_cleaned_paths.len(); 336 | 337 | if actual_files != nominal_files { 338 | error!( 339 | "Different number of files matched pattern in actual {} and nominal {}", 340 | actual_files, nominal_files 341 | ); 342 | return Err(Error::DifferentNumberOfFiles(actual_files, nominal_files)); 343 | } 344 | 345 | nominal_cleaned_paths 346 | .into_iter() 347 | .zip(actual_cleaned_paths) 348 | .for_each(|(n, a)| { 349 | let compare_result = compare_files(n, a, &rule.file_type); 350 | all_okay &= !compare_result.is_error; 351 | compare_results.push(compare_result); 352 | }); 353 | } 354 | } 355 | 356 | Ok(all_okay) 357 | } 358 | 359 | /// Use this function if you don't want this crate to load and parse a config file but provide a custom rules struct yourself 360 | pub fn compare_folders_cfg( 361 | nominal: impl AsRef, 362 | actual: impl AsRef, 363 | config_struct: ConfigurationFile, 364 | report_path: impl AsRef, 365 | ) -> Result { 366 | let mut rule_results: Vec = Vec::new(); 367 | 368 | let results: Vec = config_struct 369 | .rules 370 | .into_iter() 371 | .map(|rule| { 372 | let mut compare_results: Vec = Vec::new(); 373 | let okay = process_rule( 374 | nominal.as_ref(), 375 | actual.as_ref(), 376 | &rule, 377 | &mut compare_results, 378 | ); 379 | 380 | let rule_name = rule.name.as_str(); 381 | 382 | let result = match okay { 383 | Ok(res) => res, 384 | Err(e) => { 385 | compare_results.push(Difference { 386 | nominal_file: nominal.as_ref().to_path_buf(), 387 | actual_file: actual.as_ref().to_path_buf(), 388 | relative_file_path: get_relative_path(actual.as_ref(), nominal.as_ref()) 389 | .to_string_lossy() 390 | .to_string(), 391 | is_error: true, 392 | detail: vec![DiffDetail::Error(e.to_string())], 393 | }); 394 | error!("Error occurred during rule-processing for rule {rule_name}: {e}"); 395 | false 396 | } 397 | }; 398 | 399 | rule_results.push(report::RuleDifferences { 400 | rule, 401 | diffs: compare_results, 402 | }); 403 | 404 | result 405 | }) 406 | .collect(); 407 | 408 | let all_okay = results.iter().all(|result| *result); 409 | report::create_reports(&rule_results, &report_path)?; 410 | Ok(all_okay) 411 | } 412 | 413 | /// The main function for comparing folders. It will parse a config file in yaml format, create a report in report_path and compare the folders nominal and actual. 414 | pub fn compare_folders( 415 | nominal: impl AsRef, 416 | actual: impl AsRef, 417 | config_file: impl AsRef, 418 | report_path: impl AsRef, 419 | ) -> Result { 420 | let config = ConfigurationFile::from_file(config_file)?; 421 | compare_folders_cfg(nominal, actual, config, report_path) 422 | } 423 | 424 | /// Create the jsonschema for the current configuration file format 425 | pub fn get_schema() -> Result { 426 | let schema = schema_for!(ConfigurationFile); 427 | Ok(serde_json::to_string_pretty(&schema)?) 428 | } 429 | 430 | /// Try to load config yaml and check whether it is a valid one. Returns true if file can be loaded, otherwise false 431 | pub fn validate_config(config_file: impl AsRef) -> bool { 432 | let config_file = config_file.as_ref(); 433 | let config_file_string = config_file.to_string_lossy(); 434 | if !config_file.exists() { 435 | error!("Could not find config file: {config_file_string}"); 436 | return false; 437 | } 438 | 439 | match ConfigurationFile::from_file(config_file) { 440 | Ok(_) => { 441 | info!("Config file {config_file_string} loaded successfully"); 442 | true 443 | } 444 | Err(e) => { 445 | error!( 446 | "Could not load config file {config_file_string}: {}", 447 | e.to_string() 448 | ); 449 | false 450 | } 451 | } 452 | } 453 | 454 | #[cfg(test)] 455 | mod tests { 456 | use crate::image::{CompareMode, RGBCompareMode}; 457 | 458 | use super::*; 459 | 460 | #[test] 461 | fn folder_not_found_is_error() { 462 | let rule = Rule { 463 | name: "test rule".to_string(), 464 | file_type: ComparisonMode::Image(ImageCompareConfig { 465 | threshold: 1.0, 466 | mode: CompareMode::RGB(RGBCompareMode::Hybrid), 467 | }), 468 | pattern_include: vec!["*.".to_string()], 469 | pattern_exclude: None, 470 | }; 471 | let mut result = Vec::new(); 472 | assert!(process_rule("NOT_EXISTING", ".", &rule, &mut result).is_err()); 473 | assert!(process_rule(".", "NOT_EXISTING", &rule, &mut result).is_err()); 474 | } 475 | 476 | #[test] 477 | fn multiple_include_exclude_works() { 478 | let pattern_include = vec![ 479 | "**/Components.csv".to_string(), 480 | "**/CumulatedHistogram.csv".to_string(), 481 | ]; 482 | let empty = vec![""]; 483 | let result = 484 | get_files("tests/csv/data/", &pattern_include, &empty).expect("could not glob"); 485 | assert_eq!(result.len(), 2); 486 | let excludes = vec!["**/Components.csv".to_string()]; 487 | let result = 488 | get_files("tests/csv/data/", &pattern_include, &excludes).expect("could not glob"); 489 | assert_eq!(result.len(), 1); 490 | let excludes = vec![ 491 | "**/Components.csv".to_string(), 492 | "**/CumulatedHistogram.csv".to_string(), 493 | ]; 494 | let result = 495 | get_files("tests/csv/data/", &pattern_include, &excludes).expect("could not glob"); 496 | assert!(result.is_empty()); 497 | } 498 | } 499 | --------------------------------------------------------------------------------