├── .gitignore ├── .pre-commit-config.yaml ├── CHANGES ├── COPYING ├── Cargo.lock ├── Cargo.toml ├── Makefile ├── README.md ├── bin └── txt2man ├── doc ├── TODO.md ├── bm-journal.patch └── thinp-version-2 │ └── notes.md ├── examples └── custom-emitter.rs ├── flake.lock ├── flake.nix ├── man8 ├── cache_check.txt ├── cache_dump.txt ├── cache_metadata_size.txt ├── cache_repair.txt ├── cache_restore.txt ├── cache_writeback.txt ├── era_check.txt ├── era_dump.txt ├── era_invalidate.txt ├── era_restore.txt ├── thin_check.txt ├── thin_delta.txt ├── thin_dump.txt ├── thin_ls.txt ├── thin_metadata_pack.txt ├── thin_metadata_size.txt ├── thin_metadata_unpack.txt ├── thin_migrate.txt ├── thin_repair.txt ├── thin_restore.txt ├── thin_rmap.txt └── thin_trim.txt ├── mk_release ├── rustfmt.toml ├── src ├── bin │ ├── pdata_tools.rs │ └── pdata_tools_dev.rs ├── cache │ ├── check.rs │ ├── damage_generator.rs │ ├── dump.rs │ ├── hint.rs │ ├── ir.rs │ ├── mapping.rs │ ├── metadata_generator.rs │ ├── metadata_size.rs │ ├── mod.rs │ ├── repair.rs │ ├── restore.rs │ ├── superblock.rs │ ├── writeback.rs │ └── xml.rs ├── checksum.rs ├── commands │ ├── cache_check.rs │ ├── cache_dump.rs │ ├── cache_generate_damage.rs │ ├── cache_generate_metadata.rs │ ├── cache_metadata_size.rs │ ├── cache_repair.rs │ ├── cache_restore.rs │ ├── cache_writeback.rs │ ├── engine.rs │ ├── era_check.rs │ ├── era_dump.rs │ ├── era_generate_metadata.rs │ ├── era_invalidate.rs │ ├── era_repair.rs │ ├── era_restore.rs │ ├── mod.rs │ ├── thin_check.rs │ ├── thin_delta.rs │ ├── thin_dump.rs │ ├── thin_explore.rs │ ├── thin_generate_damage.rs │ ├── thin_generate_metadata.rs │ ├── thin_ls.rs │ ├── thin_metadata_pack.rs │ ├── thin_metadata_size.rs │ ├── thin_metadata_unpack.rs │ ├── thin_migrate.rs │ ├── thin_repair.rs │ ├── thin_restore.rs │ ├── thin_rmap.rs │ ├── thin_shrink.rs │ ├── thin_stat.rs │ ├── thin_trim.rs │ ├── utils.rs │ └── utils │ │ └── range_parsing_tests.rs ├── copier │ ├── base.rs │ ├── batcher.rs │ ├── mod.rs │ ├── report.rs │ ├── rescue_copier.rs │ ├── rescue_copier │ │ └── tests.rs │ ├── sync_copier.rs │ ├── sync_copier │ │ └── tests.rs │ ├── test_utils.rs │ └── wrapper.rs ├── devtools │ ├── damage_generator.rs │ └── mod.rs ├── dump_utils.rs ├── era │ ├── check.rs │ ├── dump.rs │ ├── invalidate.rs │ ├── ir.rs │ ├── metadata_generator.rs │ ├── mod.rs │ ├── repair.rs │ ├── restore.rs │ ├── superblock.rs │ ├── writeset.rs │ └── xml.rs ├── file_utils.rs ├── grid_layout.rs ├── io_engine │ ├── async_.rs │ ├── base.rs │ ├── buffer.rs │ ├── core.rs │ ├── gaps.rs │ ├── mod.rs │ ├── ramdisk.rs │ ├── spindle.rs │ ├── sync.rs │ ├── sync │ │ └── tests.rs │ ├── utils.rs │ └── utils │ │ └── tests.rs ├── ioctl.rs ├── ioctl │ └── tests.rs ├── lib.rs ├── math.rs ├── pack │ ├── delta_list.rs │ ├── mod.rs │ ├── node_encode.rs │ ├── toplevel.rs │ └── vm.rs ├── pdata │ ├── array.rs │ ├── array │ │ └── tests.rs │ ├── array_builder.rs │ ├── array_builder │ │ ├── test_utils.rs │ │ └── tests.rs │ ├── array_walker.rs │ ├── array_walker │ │ └── tests.rs │ ├── bitset.rs │ ├── btree.rs │ ├── btree │ │ └── tests.rs │ ├── btree_builder.rs │ ├── btree_builder │ │ ├── test_utils.rs │ │ └── tests.rs │ ├── btree_error.rs │ ├── btree_iterator.rs │ ├── btree_leaf_walker.rs │ ├── btree_lookup.rs │ ├── btree_merge.rs │ ├── btree_walker.rs │ ├── btree_walker │ │ └── tests.rs │ ├── mod.rs │ ├── space_map │ │ ├── allocated_blocks.rs │ │ ├── base.rs │ │ ├── checker.rs │ │ ├── common.rs │ │ ├── disk.rs │ │ ├── metadata.rs │ │ ├── mod.rs │ │ └── tests.rs │ └── unpack.rs ├── random.rs ├── report.rs ├── run_iter.rs ├── shrink │ ├── mod.rs │ └── toplevel.rs ├── thin │ ├── block_time.rs │ ├── check.rs │ ├── damage_generator.rs │ ├── delta.rs │ ├── delta │ │ └── tests.rs │ ├── delta_visitor.rs │ ├── device_detail.rs │ ├── dump.rs │ ├── human_readable_format.rs │ ├── ir.rs │ ├── ls.rs │ ├── metadata.rs │ ├── metadata_generator.rs │ ├── metadata_repair.rs │ ├── metadata_repair │ │ └── sorting_roots_tests.rs │ ├── metadata_size.rs │ ├── migrate │ │ ├── base.rs │ │ ├── devices.rs │ │ ├── metadata.rs │ │ ├── mod.rs │ │ └── stream.rs │ ├── mod.rs │ ├── repair.rs │ ├── restore.rs │ ├── rmap.rs │ ├── runs.rs │ ├── shrink.rs │ ├── stat.rs │ ├── superblock.rs │ ├── trim.rs │ └── xml.rs ├── units.rs ├── utils │ ├── hashvec.rs │ └── mod.rs ├── version.rs ├── write_batcher.rs ├── write_batcher │ └── tests.rs └── xml.rs ├── tests ├── cache_check.rs ├── cache_dump.rs ├── cache_metadata_size.rs ├── cache_repair.rs ├── cache_restore.rs ├── cache_writeback.rs ├── common │ ├── cache.rs │ ├── cache_xml_generator.rs │ ├── common_args.rs │ ├── era.rs │ ├── era_xml_generator.rs │ ├── fixture.rs │ ├── input_arg.rs │ ├── mod.rs │ ├── output_option.rs │ ├── piping.rs │ ├── process.rs │ ├── program.rs │ ├── target.rs │ ├── test_dir.rs │ ├── thin.rs │ └── thin_xml_generator.rs ├── era_check.rs ├── era_dump.rs ├── era_invalidate.rs ├── era_restore.rs ├── testdata │ ├── corrupted_tmeta_with_metadata_snap.pack │ ├── readme.md │ ├── scripts │ │ ├── corrupted_tmeta_with_metadata_snap.sh │ │ ├── tmeta.sh │ │ ├── tmeta_device_id_reuse.sh │ │ ├── tmeta_with_corrupted_metadata_snap.sh │ │ ├── tmeta_with_deleted_snapshot.sh │ │ └── tmeta_with_empty_roots.sh │ ├── tmeta.pack │ ├── tmeta_device_id_reuse.pack │ ├── tmeta_device_id_reuse_with_corrupted_thins.pack │ ├── tmeta_with_corrupted_metadata_snap.pack │ ├── tmeta_with_deleted_snapshot.pack │ ├── tmeta_with_empty_roots.pack │ └── tmeta_with_metadata_snap.pack ├── thin_check.rs ├── thin_delta.rs ├── thin_dump.rs ├── thin_ls.rs ├── thin_metadata_pack.rs ├── thin_metadata_size.rs ├── thin_metadata_unpack.rs ├── thin_repair.rs ├── thin_restore.rs ├── thin_rmap.rs └── thin_shrink.rs └── xml_metadata ├── basic_tests_dd.xml └── basic_tests_dt.xml /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.tmp 3 | *.swp 4 | *.swo 5 | *.o 6 | *.so 7 | *.a 8 | *.gmo 9 | *_t 10 | *.d 11 | *.8 12 | *.d.[0-9]* 13 | test.data 14 | cachegrind.* 15 | \#*\# 16 | core 17 | /target 18 | **/*.rs.bk 19 | 20 | googletest/ 21 | 22 | bin/pdata_tools 23 | bin/pdata_tools_dev 24 | thin_check 25 | thin_dump 26 | thin_restore 27 | thin_repair 28 | thin_rmap 29 | thin_metadata_size 30 | thin_show_blocks 31 | 32 | cache_check 33 | cache_dump 34 | cache_restore 35 | cache_repair 36 | cache_metadata_size 37 | 38 | era_check 39 | era_dump 40 | era_invalidate 41 | 42 | *.metadata 43 | bad-metadata 44 | 45 | unit-tests/Makefile 46 | unit-tests/unit_tests 47 | 48 | gmock-1.6.0.zip 49 | gmock-1.6.0/ 50 | 51 | autom4te.cache/ 52 | 53 | *.xml 54 | *.bin 55 | *.patch 56 | *.orig 57 | *.rej 58 | 59 | version.h 60 | src/version.rs 61 | config.cache 62 | config.log 63 | config.status 64 | configure 65 | 66 | callgrind.* 67 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.5.0 4 | hooks: 5 | - id: check-merge-conflict 6 | - id: end-of-file-fixer 7 | - id: mixed-line-ending 8 | - id: trailing-whitespace 9 | - repo: https://github.com/doublify/pre-commit-rust 10 | rev: v1.0 11 | hooks: 12 | - id: fmt 13 | - repo: local 14 | hooks: 15 | - id: cargo-check-locked 16 | name: cargo check 17 | description: Check the package for errors. 18 | entry: cargo check --locked 19 | language: system 20 | files: Cargo.(lock|toml)$ 21 | pass_filenames: false 22 | - id: cargo-clippy-locked 23 | name: cargo clippy 24 | description: Run the Clippy linter on all the targets. 25 | entry: cargo clippy --locked --all-targets --all-features -- -D warnings 26 | language: system 27 | files: \.rs$ 28 | pass_filenames: false 29 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "thinp" 3 | version = "1.1.0" 4 | authors = ["Joe Thornber "] 5 | edition = "2021" 6 | license = "GPL-3.0-only" 7 | 8 | [dependencies] 9 | anyhow = "1.0" 10 | base64 = "0.22" 11 | byteorder = "1.4" 12 | clap = { version = "4.5", default-features = false, features = [ 13 | "std", 14 | "help", 15 | "usage", 16 | "error-context", 17 | "suggestions", 18 | ] } 19 | crc32c = "0.6" 20 | data-encoding = "2.5" 21 | devicemapper = "0.34.3" 22 | exitcode = "1.1.2" 23 | fixedbitset = "0.4" 24 | flate2 = "1.0" 25 | iovec = "0.1" 26 | indicatif = "0.17" 27 | libc = "0.2" 28 | nom = "7.1" 29 | num_cpus = "1.16" 30 | num-derive = "0.4" 31 | num-traits = "0.2" 32 | quick-xml = "0.36" 33 | rand = "0.8" 34 | rangemap = "1.5" 35 | roaring = "0.10" 36 | rio = { git = "https://github.com/jthornber/rio", branch = "master", optional = true } 37 | thiserror = "1.0" 38 | tui = { version = "0.19", default-features = false, features = [ 39 | "termion", 40 | ], optional = true } 41 | termion = { version = "1.5", optional = true } 42 | udev = "0.7" 43 | 44 | [dev-dependencies] 45 | duct = "0.13" 46 | mockall = "0.13" 47 | quickcheck = "1.0" 48 | quickcheck_macros = "1.0" 49 | rand = { version = "0.8", features = ["small_rng"] } 50 | tempfile = "3.6" 51 | thinp = { path = ".", features = ["devtools"] } 52 | 53 | [features] 54 | devtools = ["tui", "termion"] 55 | io_uring = ["dep:rio"] 56 | no_cleanup = [] 57 | 58 | [profile.release] 59 | debug = true 60 | 61 | [[bin]] 62 | name = "pdata_tools_dev" 63 | required-features = ["devtools"] 64 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | # run 'nix develop' to launch the dev environment 2 | { 3 | description = "Development environment for thin-provisioning-tools"; 4 | 5 | inputs = { 6 | nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; 7 | rust-overlay.url = "github:oxalica/rust-overlay"; 8 | flake-utils.url = "github:numtide/flake-utils"; 9 | }; 10 | 11 | outputs = { self, nixpkgs, rust-overlay, flake-utils, ... }: 12 | flake-utils.lib.eachDefaultSystem (system: 13 | let 14 | overlays = [ (import rust-overlay) ]; 15 | pkgs = import nixpkgs { 16 | inherit system overlays; 17 | }; 18 | in 19 | with pkgs; 20 | { 21 | devShells.default = mkShell { 22 | buildInputs = [ 23 | openssl 24 | pkg-config 25 | rust-bin.beta.latest.default 26 | ]; 27 | }; 28 | } 29 | ); 30 | } 31 | -------------------------------------------------------------------------------- /man8/cache_check.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | cache_check - validates cache metadata on a device or file. 3 | 4 | SYNOPSIS 5 | cache_check [options] {device|file} 6 | 7 | DESCRIPTION 8 | cache_check checks cache metadata created by the device-mapper cache target 9 | on a device or file. 10 | 11 | This tool cannot be run on live metadata. 12 | 13 | OPTIONS 14 | -h, --help Print help and exit. 15 | -V, --version Output version information and exit. 16 | -q, --quiet Suppress output messages, return only exit code. 17 | --super-block-only Only check the superblock. 18 | --skip-hints Skip checking of the policy hint values metadata. 19 | --skip-discards Skip checking of the discard bits in the metadata. 20 | --clear-needs-check-flag Clears the 'needs_check' flag in the superblock. 21 | 22 | The kernel may set a flag to force the pool to be checked before it's next 23 | activated. Set this switch to clear the flag if the check is successful. 24 | If the metadata check failed, the flag is not cleared and a cache_repair run 25 | is needed to fix any issues. After cache_repair succeeded, you may run 26 | cache_check again. 27 | 28 | EXAMPLE 29 | Analyses and repairs cache metadata on logical volume /dev/vg/metadata: 30 | 31 | $ cache_check /dev/vg/metadata 32 | 33 | The device may not be actively used by the target when running. 34 | 35 | DIAGNOSTICS 36 | cache_check returns an exit code of 0 for success or 1 for error. 37 | 38 | SEE ALSO 39 | cache_dump(8), cache_repair(8), cache_restore(8) 40 | 41 | AUTHOR 42 | Joe Thornber , Heinz Mauelshagen 43 | 44 | -------------------------------------------------------------------------------- /man8/cache_dump.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | cache_dump - dump cache metadata from device or file to standard output. 3 | 4 | SYNOPSIS 5 | cache_dump [options] {device|file} 6 | 7 | DESCRIPTION 8 | cache_dump dumps binary cache metadata created by the device-mapper cache 9 | target on a device or file to standard output for analysis or postprocessing 10 | in XML format. XML formatted metadata can be fed into cache_restore in order 11 | to put it back onto a metadata device (to process by the device-mapper 12 | target), or file. 13 | 14 | This tool cannot be run on live metadata. 15 | 16 | OPTIONS 17 | -h, --help Print help and exit. 18 | -V, --version Print version information and exit. 19 | -r, --repair Repair the metadata whilst dumping it. 20 | -o {xml file} Specify an output file for the xml, rather than printing to stdout. 21 | 22 | EXAMPLES 23 | Dumps the cache metadata on logical volume /dev/vg/metadata to standard 24 | output in XML format: 25 | 26 | $ cache_dump /dev/vg/metadata 27 | 28 | Dumps the cache metadata on logical volume /dev/vg/metadata whilst repairing 29 | it to standard output in XML format: 30 | 31 | $ cache_dump --repair /dev/vg/metadata 32 | 33 | DIAGNOSTICS 34 | cache_dump returns an exit code of 0 for success or 1 for error. 35 | 36 | SEE ALSO 37 | cache_check(8), cache_repair(8), cache_restore(8) 38 | 39 | AUTHOR 40 | Joe Thornber , Heinz Mauelshagen 41 | 42 | -------------------------------------------------------------------------------- /man8/cache_metadata_size.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | cache_metadata_size - Estimate the size of the metadata device needed for a given configuration. 3 | 4 | SYNOPSIS 5 | cache_metadata_size [options] 6 | 7 | DESCRIPTION 8 | cache_metadata_size estimates the size of the metadata. Either --nr-blocks, 9 | or --block-size and --device-size must be specified. 10 | 11 | OPTIONS 12 | --nr-blocks {natural} Specify the number of cache blocks. 13 | --block-size {sectors} Specify the size of each cache block in 512 byte sectors. 14 | --device-size {sectors} Specify total size of the fast device used in the cache. In 512 byte sectors. 15 | --max-hint-width {nr bytes} Specify hint width. 16 | 17 | Cache policies use a per block 'hint' to record extra info (for instance 18 | hit counts). At the moment all policies use a 4 byte hint 19 | width. If you want to use a different hint width specify it with this 20 | switch. 21 | 22 | EXAMPLE 23 | $ cache_metadata_size --nr-blocks 10240 24 | 25 | $ cache_metadata_size --block-size 128 --device-size 1024000 26 | 27 | SEE ALSO 28 | cache_dump(8), cache_repair(8), cache_restore(8) 29 | 30 | AUTHOR 31 | Joe Thornber , Heinz Mauelshagen 32 | -------------------------------------------------------------------------------- /man8/cache_repair.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | cache_repair - repair cache binary metadata from device/file to device/file. 3 | 4 | SYNOPSIS 5 | cache_repair [options] -i {device|file} -o {device|file} 6 | 7 | DESCRIPTION 8 | cache_repair reads binary cache metadata created by the respective 9 | device-mapper target from one device or file, repairs it and writes it to 10 | another device or file. If written to a metadata device, the metadata can 11 | be processed by the device-mapper target. 12 | 13 | This tool cannot be run on live metadata. 14 | 15 | OPTIONS 16 | -h, --help Print help and exit. 17 | -V, --version Print version information and exit. 18 | -i, --input {device|file} Input file or device containing binary metadata. 19 | -o, --output {device|file} Output file or device for repaired binary metadata. 20 | 21 | If a file is then it must be preallocated, and large enough to hold the 22 | metadata. 23 | 24 | EXAMPLE 25 | Reads the binary cache metadata from file metadata, repairs it and writes it 26 | to logical volume /dev/vg/metadata for further processing by the respective 27 | device-mapper target: 28 | 29 | $ cache_repair -i metadata -o /dev/vg/metadata 30 | 31 | DIAGNOSTICS 32 | cache_repair returns an exit code of 0 for success or 1 for error. 33 | 34 | SEE ALSO 35 | cache_dump(8), cache_check(8), cache_restore(8) 36 | 37 | AUTHOR 38 | Joe Thornber , Heinz Mauelshagen 39 | -------------------------------------------------------------------------------- /man8/cache_restore.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | cache_restore - restore cache metadata file to device or file. 3 | 4 | SYNOPSIS 5 | cache_restore [options] -i {xml file} -o {device|file} 6 | 7 | DESCRIPTION 8 | cache_restore restores cache metadata created by the respective device-mapper 9 | target dumped into an XML formatted (see cache_dump(8)) file, which 10 | optionally can be preprocessed before the restore to another device or file. 11 | If restored to a metadata device, the metadata can be processed by the 12 | device-mapper target. 13 | 14 | This tool cannot be run on live metadata. 15 | 16 | OPTIONS 17 | -h, --help Print help and exit. 18 | -V, --version Print version information and exit. 19 | -q, --quiet Don't print any output. Check the exit code to test for success. 20 | -i, --input {xml file} Input xml. 21 | -o, --output {device|file} Output file or device for restored binary metadata. 22 | 23 | If a file is used thin it must be preallocated, and large enough to hold 24 | the metadata. 25 | 26 | --metadata-version {1|2} Choose a metadata version. 27 | 28 | DEBUGGING OPTIONS 29 | --debug-override-metadata-version {integer} Override the version stored in the metadata. 30 | --omit-clean-shutdown Don't set the clean shutdown flag. 31 | 32 | EXAMPLE 33 | Restores the XML formatted cache metadata on file metadata to logical 34 | volume /dev/vg/metadata for further processing by the respective 35 | device-mapper target: 36 | 37 | $ cache_restore -i metadata -o /dev/vg/metadata 38 | 39 | DIAGNOSTICS 40 | cache_restore returns an exit code of 0 for success or 1 for error. 41 | 42 | SEE ALSO 43 | cache_dump(8), cache_check(8), cache_repair(8) 44 | 45 | AUTHOR 46 | Joe Thornber , Heinz Mauelshagen 47 | 48 | -------------------------------------------------------------------------------- /man8/cache_writeback.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | cache_writeback - writeback dirty blocks to the origin device. 3 | 4 | SYNOPSIS 5 | cache_writeback [options] --metadata-device {device|file} --origin-device {device|file} --fast-device {device|file} 6 | 7 | DESCRIPTION 8 | cache_writeback is an offline tool that writes back dirty data to the data 9 | device (origin). Intended for use in recovery scenarios when the SSD is 10 | giving IO errors. 11 | 12 | This tool cannot be run on a live cache. 13 | 14 | OPTIONS 15 | -h, --help Print help and exit. 16 | -V, --version Print version information and exit. 17 | --metadata-device {device|file} Location of cache metadata. 18 | --origin-device {device|file} Slow device being cached. 19 | --fast-device {device|file} Fast device containing the data that needs to be written back. 20 | --no-metadata-update Do not update the metadata to clear the dirty flags. 21 | 22 | You may want to use this flag if you're decommissioning the cache. 23 | 24 | --buffer-size-meg {size} Specify the size for the data cache, in megabytes. 25 | 26 | Defaults to 64 MiB, a larger size may improve performance. 27 | 28 | --list-failed-blocks List any blocks that failed the writeback process. 29 | 30 | SEE ALSO 31 | cache_dump(8), cache_check(8), cache_repair(8), cache_restore(8) 32 | 33 | AUTHOR 34 | Joe Thornber 35 | -------------------------------------------------------------------------------- /man8/era_check.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | era_check - validate era metadata on device or file. 3 | 4 | SYNOPSIS 5 | era_check [options] {device|file} 6 | 7 | DESCRIPTION 8 | era_check checks era metadata created by the device-mapper era target on a 9 | device or file. 10 | 11 | This tool cannot be run on live metadata. 12 | 13 | OPTIONS 14 | -h, --help Print help and exit. 15 | -V, --version Print version information and exit. 16 | -q, --quiet Suppress output messages, return only exit code. 17 | --super-block-only Only check the superblock is present. 18 | 19 | EXAMPLE 20 | Analyse thin provisioning metadata on logical volume /dev/vg/metadata: 21 | 22 | $ era_check /dev/vg/metadata 23 | 24 | The device may not be actively used by the target when running. 25 | 26 | DIAGNOSTICS 27 | era_check returns an exit code of 0 for success or 1 for error. 28 | 29 | SEE ALSO 30 | era_dump(8), era_restore(8), era_invalidate(8) 31 | 32 | AUTHOR 33 | Joe Thornber 34 | -------------------------------------------------------------------------------- /man8/era_dump.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | era_dump - dump era metadata from device or file to standard output. 3 | 4 | SYNOPSIS 5 | era_dump [options] {device|file} 6 | 7 | DESCRIPTION 8 | era_dump dumps binary era metadata created by the device-mapper era target on 9 | a device or file to standard output for analysis or postprocessing in 10 | XML format. XML formatted metadata can be fed into era_restore (see 11 | era_restore(8)) in order to put it back onto a metadata device (to process by 12 | the device-mapper target) or file. 13 | 14 | This tool cannot be run on live metadata. 15 | 16 | OPTIONS 17 | -h, --help Print help and exit. 18 | -V, --version Print version information and exit. 19 | --repair Repair the metadata whilst dumping it. 20 | --logical Fold any unprocessed write sets into the final era array. 21 | 22 | You probably want to do this if you're intending to process the results as 23 | it simplifies the XML. 24 | 25 | -o {xml file} Specify a file for the output rather than writing to stdout. 26 | 27 | EXAMPLES 28 | Dumps era metadata on logical volume /dev/vg/metadata to standard output in 29 | XML format: 30 | 31 | $ era_dump /dev/vg/metadata 32 | 33 | DIAGNOSTICS 34 | era_dump returns an exit code of 0 for success or 1 for error. 35 | 36 | SEE ALSO 37 | era_check(8), era_restore(8), era_invalidate(8) 38 | 39 | AUTHOR 40 | Joe Thornber 41 | -------------------------------------------------------------------------------- /man8/era_invalidate.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | era_invalidate - Provide a list of blocks that have changed since a particular era. 3 | 4 | SYNOPSIS 5 | era_invalidate [options] {device|file} 6 | 7 | DESCRIPTION 8 | era_invalidate examines era metadata and lists blocks that may have changed 9 | since a given era. 10 | 11 | This tool cannot be run on live metadata unless the --metadata-snap option is 12 | used. 13 | 14 | OPTIONS 15 | -h, --help Print help and exit. 16 | -V, --version Print version information and exit. 17 | -o {output file} Write output to a file rather than stdout. 18 | --metadata-snapshot Use the metadata snapshot rather than the current superblock. 19 | --written-since {era nr} Blocks written since the given era will be listed. 20 | 21 | EXAMPLE 22 | List the blocks that may have been written since the beginning of era 13 on the 23 | metadata device /dev/vg/metadata: 24 | 25 | $ era_invalidate --written-since 13 /dev/vg/metadata 26 | 27 | The device may not be actively used by the target when running. 28 | 29 | DIAGNOSTICS 30 | era_invalidate returns an exit code of 0 for success or 1 for error (eg, 31 | metadata corruption). 32 | 33 | SEE ALSO 34 | era_check(8), era_dump(8), era_restore(8) 35 | 36 | AUTHOR 37 | Joe Thornber 38 | -------------------------------------------------------------------------------- /man8/era_restore.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | era_restore - restore era metadata file to device or file. 3 | 4 | SYNOPSIS 5 | era_restore [options] -i {xml file} -o {device|file} 6 | 7 | DESCRIPTION 8 | era_restore restores era metadata created by the respective device-mapper 9 | target dumped into an XML formatted (see era_dump(8)) file, which 10 | optionally can be preprocessed before the restore to another device or 11 | file. If restored to a metadata device, the metadata can be processed by 12 | the device-mapper target. 13 | 14 | This tool cannot be run on live metadata. 15 | 16 | OPTIONS 17 | -h, --help Print help and exit. 18 | -V, --version Print version information and exit. 19 | -q, --quiet Don't print any output. Check the exit code to test for success. 20 | -i, --input {xml file} Specify input file containing xml metadata. 21 | -o, --output {device|file} Output device or file for restored binary metadata. 22 | 23 | If a file is used, then it must be preallocated, and large enough to hold 24 | the metadata. 25 | 26 | EXAMPLE 27 | Restores the XML formatted era metadata on file metadata to logical volume 28 | /dev/vg/metadata for further processing by the respective device-mapper 29 | target: 30 | 31 | $ era_restore -i metadata -o /dev/vg/metadata 32 | 33 | DIAGNOSTICS 34 | era_restore returns an exit code of 0 for success or 1 for error. 35 | 36 | SEE ALSO 37 | era_dump(8), era_check(8) 38 | 39 | AUTHOR 40 | Joe Thornber , Heinz Mauelshagen 41 | -------------------------------------------------------------------------------- /man8/thin_check.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | thin_check - validates thin provisioning metadata on a device or file 3 | 4 | SYNOPSIS 5 | thin_check [options] {device|file} 6 | 7 | DESCRIPTION 8 | thin_check checks thin provisioning metadata created by the device-mapper 9 | thin provisioning target on a device or file. 10 | 11 | The tool cannot be run on live metadata unless the --metadata-snapshot 12 | option is used. 13 | 14 | OPTIONS 15 | -q, --quiet Suppress output messages, return only exit code. 16 | -h, --help Print help and exit. 17 | -V, --version Output version information and exit. 18 | --super-block-only Only check the superblock. 19 | 20 | --skip-mappings Skip checking of the block mappings which make up the bulk of the metadata. 21 | 22 | --ignore-non-fatal-errors Will only return a non-zero exit code if it finds a fatal error. 23 | 24 | An example of a nonfatal error is an incorrect data block reference count 25 | causing a block to be considered allocated when it in fact isn't. Ignoring 26 | errors for a long time is not advised, you really should be using 27 | thin_repair to fix them. 28 | 29 | --clear-needs-check-flag Clears the 'needs_check' flag in the superblock. 30 | 31 | The kernel may set a flag to force the pool to be checked before it's next 32 | activated. Set this switch to clear the flag if the check is successful. 33 | If the metadata check failed, the flag is not cleared and a thin_repair run 34 | is needed to fix any issues. After thin_repair succeeded, you may run 35 | thin_check again. 36 | 37 | --metadata-snapshot, -m Check the metadata snapshot. 38 | 39 | This will check the devices tree and mappings in a metadata snapshot. 40 | The snap does not contain space maps, so these will not be checked. This 41 | may be used on live metadata. 42 | 43 | --auto-repair Automatically repair any trivial issues found with the metadata. 44 | 45 | Currently only fixes metadata leaks. 46 | 47 | --override-mapping-root Specify a mapping root to use. 48 | 49 | Don't use this. This overrides what's specified in the superblock. Only 50 | use this if you really understand the metadata format and are trying to 51 | recover damaged metadata. 52 | 53 | EXAMPLE 54 | Analyses thin provisioning metadata on logical volume /dev/vg/metadata: 55 | 56 | $ thin_check /dev/vg/metadata 57 | 58 | The device must not be actively used by the target when running. 59 | 60 | DIAGNOSTICS 61 | thin_check returns an exit code of 0 for success or 1 for error. 62 | 63 | SEE ALSO 64 | thin_dump(8), thin_repair(8), thin_restore(8), thin_rmap(8), thin_metadata_size(8) 65 | 66 | AUTHOR 67 | Joe Thornber , Heinz Mauelshagen 68 | 69 | -------------------------------------------------------------------------------- /man8/thin_delta.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | thin_delta - Print the differences in the mappings between two thin devices. 3 | 4 | SYNOPSIS 5 | thin_delta [options] {device|file} 6 | 7 | DESCRIPTION 8 | thin_delta allows you to compare the mappings in two thin volumes (snapshots 9 | allow common blocks between thin volumes). 10 | 11 | This tool cannot be run on live metadata unless the --metadata-snap option is 12 | used. 13 | 14 | OPTIONS 15 | --thin1, --snap1 {natural} The numeric identifier for the first thin volume to diff. 16 | --thin2, --snap2 {natural} The numeric identifier for the second thin volume to diff. 17 | --metadata-snap [block nr] Use a metadata snapshot. 18 | 19 | If you want to get information out of a live pool then you will need to 20 | take a metadata snapshot and use this switch. In order for the information 21 | to be meaningful, you need to ensure the thin volumes you're examining are 22 | not changing (ie, do not activate those thins). 23 | 24 | --verbose Provide extra information on the mappings. 25 | -h, --help Print help and exit. 26 | -V, --version Output version information and exit. 27 | 28 | SEE ALSO 29 | thin_dump(8), thin_repair(8), thin_restore(8), thin_rmap(8), thin_metadata_size(8) 30 | 31 | AUTHOR 32 | Joe Thornber , Heinz Mauelshagen 33 | 34 | -------------------------------------------------------------------------------- /man8/thin_dump.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | thin_dump - dump thin provisioning metadata from device or file to standard output. 3 | 4 | SYNOPSIS 5 | thin_dump [options] {device|file} 6 | 7 | DESCRIPTION 8 | thin_dump dumps binary thin provisioning metadata (optionally from alternate 9 | block; see option --metadata-snap) created by the 10 | device-mapper thin provisioning target on a device or file to standard 11 | output for analysis or postprocessing in either XML or human readable format. 12 | XML formatted metadata can be fed into thin_restore (see thin_restore(8)) 13 | in order to put it back onto a metadata device (to process by the 14 | device-mapper target) or file. 15 | 16 | This tool cannot be run on live metadata unless the --metadata-snap 17 | option is used. 18 | 19 | OPTIONS 20 | -h, --help Print help and exit. 21 | -V, --version Print version information and exit. 22 | -f, --format {xml|human_readable} Choose output format. 23 | -r, --repair Repair the metadata whilst dumping it. 24 | -m, --metadata-snap[=] Dump metadata snapshot. 25 | 26 | If block is not provided, access the default metadata snapshot created by 27 | the thin provisioning device-mapper target, else try the one at block nr. 28 | See the thin provisioning target documentation on how to create or release 29 | a metadata snapshot and retrieve the block number from the kernel. 30 | 31 | --dev-id {natural} Dump the specified device. 32 | 33 | This option may be specified multiple times to select more than one thin 34 | device. 35 | 36 | --transaction-id {natural} Override the transaction id given in the input xml. 37 | --data-block-size {sectors} Override the data block size given in the input xml. 38 | --nr-data-blocks {natural} Override the nr data blocks given in the input xml. 39 | 40 | --skip-mappings Do not dump the mappings. 41 | -o {xml file} Specify a file for the output rather than writing to stdout. 42 | 43 | EXAMPLES 44 | Dumps the thin provisioning metadata on logical volume /dev/vg/metadata to 45 | standard output in human readable format: 46 | 47 | $ thin_dump -f human_readable /dev/vg/metadata 48 | 49 | Dumps the thin provisioning metadata on logical volume /dev/vg/metadata to 50 | standard output in XML format: 51 | 52 | $ thin_dump /dev/vg/metadata 53 | 54 | Dumps the thin provisioning metadata snapshot on logical volume /dev/vg/metadata 55 | to standard output in human readable format (not processable by thin_restore(8)): 56 | 57 | $ thin_dump --format human_readable --metadata-snap /dev/vg/metadata 58 | 59 | DIAGNOSTICS 60 | thin_dump returns an exit code of 0 for success or 1 for error. 61 | 62 | SEE ALSO 63 | thin_check(8), thin_repair(8), thin_restore(8), thin_rmap(8), thin_metadata_size(8) 64 | 65 | AUTHOR 66 | Joe Thornber , Heinz Mauelshagen 67 | -------------------------------------------------------------------------------- /man8/thin_ls.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | thin_ls - List thin volumes within a pool. 3 | 4 | SYNOPSIS 5 | thin_ls [options] {device|file} 6 | 7 | DESCRIPTION 8 | thin_ls displays information about thin volumes in a pool. Pass the metadata 9 | device on the command line, not the pool device. 10 | 11 | This tool cannot be run on live metadata unless the --metadata-snap 12 | option is used. 13 | 14 | OPTIONS 15 | -h, --help Print help and exit. 16 | -V, --version Print version information and exit. 17 | -o, --format Give a comma separated list of fields to be output. 18 | 19 | Valid fields are: 20 | DEV, MAPPED_BLOCKS, EXCLUSIVE_BLOCKS, SHARED_BLOCKS, HIGHEST_BLOCK, 21 | MAPPED_SECTORS, EXCLUSIVE_SECTORS, SHARED_SECTORS, HIGHEST_SECTOR, 22 | MAPPED_BYTES, EXCLUSIVE_BYTES, SHARED_BYTES, HIGHEST_BYTE, MAPPED, 23 | EXCLUSIVE, SHARED, HIGHEST_MAPPED, TRANSACTION, CREATE_TIME, SNAP_TIME 24 | 25 | --no-headers Don't output headers. 26 | -m, --metadata-snap Use metadata snapshot. 27 | 28 | If you want to get information out of a live pool then you will need to 29 | take a metadata snapshot and use this switch. 30 | 31 | SEE ALSO 32 | thin_dump(8), thin_repair(8), thin_restore(8), thin_rmap(8), thin_trim(8), 33 | thin_metadata_size(8) 34 | 35 | AUTHOR 36 | Joe Thornber 37 | -------------------------------------------------------------------------------- /man8/thin_metadata_pack.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | thin_metadata_pack - pack thin provisioning binary metadata. 3 | 4 | SYNOPSIS 5 | thin_metadata_pack [options] -i {device|file} -o {device|file} 6 | 7 | DESCRIPTION 8 | thin_metadata_pack and thin_metadata_unpack are used to compress 9 | binary metadata. Useful for support. 10 | 11 | thin_metadata_pack compresses the metadata, omitting any metadata blocks that are unused. 12 | 13 | This tool cannot be run on live metadata. 14 | 15 | OPTIONS 16 | -h, --help Print help and exit. 17 | -V, --version Print version information and exit. 18 | -i, --input {device|file} Input file or device with binary data. 19 | -o, --output {device|file} Output file or device for binary data. 20 | 21 | SEE ALSO 22 | thin_dump(8), thin_check(8), thin_restore(8), thin_rmap(8), thin_metadata_size(8) 23 | 24 | AUTHOR 25 | Joe Thornber 26 | -------------------------------------------------------------------------------- /man8/thin_metadata_size.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | thin_metadata_size - thin provisioning metadata device/file size calculator. 3 | 4 | SYNOPSIS 5 | thin_metadata_size [options] 6 | 7 | DESCRIPTION 8 | thin_metadata_size calculates the size of the thin provisioning metadata 9 | based on the block size of the thin provisioned devices, the size of the thin 10 | provisioning pool and the maximum number of all thin provisioned devices and 11 | snapshots. Because thin provisioning pools are holding widely variable 12 | contents, this tool is needed to provide sensible initial default size. 13 | 14 | OPTIONS 15 | -h, --help Print help and exit. 16 | -V, --version Print version information and exit. 17 | -b, --block-size {BLOCKSIZE[bskKmMgGtTpPeEzZyY]} Set block size. 18 | 19 | Block size of thin provisioned devices in units of bytes, sectors, 20 | kibibytes, kilobytes, ... respectively. Default is in sectors without a 21 | block size unit specifier. Size/number option arguments can be followed by 22 | unit specifiers in short one character and long form (eg. -b1m or 23 | -b1mebibytes). 24 | 25 | -s, --pool-size {POOLSIZE[bskKmMgGtTpPeEzZyY]} Set pool size. 26 | 27 | Thin provisioning pool size in units of bytes, sectors, kibibytes, 28 | kilobytes, ... respectively. Default is in sectors without a pool size 29 | unit specifier. 30 | 31 | -m, --max-thins {count[bskKmMgGtTpPeEzZyY]} Set max thins. 32 | 33 | Maximum sum of all thin provisioned devices and snapshots. Unit identifier 34 | supported to allow for convenient entry of large quantities, eg. 1000000 = 35 | 1M. Default is absolute quantity without a number unit specifier. 36 | 37 | -u, --unit {bskKmMgGtTpPeEzZyY} 38 | 39 | Output unit specifier in units of bytes, sectors, kibibytes, kilobytes, ... 40 | respectively. Default is in sectors without an output unit specifier. 41 | 42 | -n, --numeric-only {short|long} Limit output to just the size number. 43 | 44 | EXAMPLES 45 | Calculates the thin provisioning metadata device size for block size 64 46 | kibibytes, pool size 1 tebibytes and maximum number of thin provisioned 47 | devices and snapshots of 1000 in units of sectors with long output: 48 | 49 | $ thin_metadata_size -b64k -s1t -m1000 50 | 51 | Or (using the long options instead) for block size 1 gibibyte, pool size 1 52 | petabyte and maximum number of thin provisioned devices and snapshots of 1 53 | million with numeric-only output in units of gigabytes: 54 | 55 | $ thin_metadata_size --block-size=1g --pool-size=1P --max-thins=1M --unit=G --numeric-only 56 | 57 | Same as before (1g, 1P, 1M, numeric-only) but with unit specifier character 58 | appended: 59 | 60 | $ thin_metadata_size --block-size=1gibi --pool-size=1petabytes --max-thins=1mega --unit=G --numeric-only=short 61 | 62 | Or with unit specifier string appended: 63 | 64 | $ thin_metadata_size --block-size=1gibi --pool-size=1petabytes --max-thins=1mega --unit=G -nlong 65 | 66 | DIAGNOSTICS 67 | 68 | thin_metadata_size returns an exit code of 0 for success or 1 for error. 69 | 70 | SEE ALSO 71 | thin_dump(8), thin_check(8), thin_repair(8), thin_restore(8), thin_rmap(8) 72 | 73 | AUTHOR 74 | Joe Thornber , Heinz Mauelshagen 75 | 76 | -------------------------------------------------------------------------------- /man8/thin_metadata_unpack.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | thin_metadata_unpack - unpack thin provisioning binary metadata. 3 | 4 | SYNOPSIS 5 | thin_metadata_unpack [options] -i {device|file} -o {device|file} 6 | 7 | DESCRIPTION 8 | thin_metadata_pack and thin_metadata_unpack are used to compress 9 | binary metadata. Useful for support. 10 | 11 | thin_metadata_unpack expands metadata that has previously been packed with 12 | thin_metadata_pack. It outputs a binary file that the rest of the thin 13 | tools can use. 14 | 15 | This tool cannot be run on live metadata. 16 | 17 | OPTIONS 18 | -h, --help Print help and exit. 19 | -V, --version Print version information and exit. 20 | -i, --input {device|file} Input file or device with binary data. 21 | -o, --output {device|file} Output file or device for binary data. 22 | 23 | SEE ALSO 24 | thin_dump(8), thin_check(8), thin_restore(8), thin_rmap(8), thin_metadata_size(8) 25 | 26 | AUTHOR 27 | Joe Thornber 28 | -------------------------------------------------------------------------------- /man8/thin_migrate.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | thin_migrate 3 | 4 | SYNOPSIS 5 | thin_migrate [options] --source-dev {device} --dest-dev {device} 6 | thin_migrate [options] --source-dev {device} --dest-file {file} 7 | 8 | DESCRIPTION 9 | thin_migrate copies provisioned data blocks from a thin device to another 10 | device or file. It is designed for cloning a read-only snapshot or thin 11 | volume within a live thin-pool. A metadata snapshot is required to read the 12 | data mappings. 13 | 14 | OPTIONS 15 | -h, --help Print help and exit. 16 | -V, --version Print version information and exit. 17 | --source-dev {device} The input read-only thin device to copy. 18 | --dest-dev {device} The output device as the copy destination. 19 | --dest-file {file} The output file as the copy destination. 20 | -q, --quit Suppress output messages, return only exit code. 21 | --buffer-size-meg {size} Specify the size of the data buffers, in megabytes. 22 | 23 | EXAMPLE 24 | 25 | Assuming that there's a thin snapshot 'vg/snap' to copy, we must first set 26 | the snapshot to read-only: 27 | 28 | $ lvchange vg/snap --permission r 29 | 30 | Next, create a metadata snapshot for running thin_migrate: 31 | 32 | $ dmsetup message vg-mythinpool-tpool 0 reserve_metadata_snap 33 | 34 | Once the metadata snapshot is created, copy the snapshot into another linear 35 | or thin volume. The destination volume must have the same size as the source 36 | device. 37 | 38 | $ thin_migrate --src-device /dev/vg/snap --dest-device /dev/vg2/dest 39 | 40 | Alternatively, copy the snapshot into a file. thin_migrate can optionally 41 | create the destination file if it is not present, or the size of the file 42 | will be truncated to match that of the source device. 43 | 44 | $ thin_migrate --src-device /dev/vg/snap --dest-file dest.bin 45 | 46 | Finally, release the metadata snapshot after thin_migrate finishes its job: 47 | 48 | $ dmsetup message vg-mythinpool-tpool 0 release_metadata_snap 49 | 50 | DIAGNOSTICS 51 | thin_migrate returns an exit code of 0 for success or 1 for error. 52 | 53 | AUTHOR 54 | Joe Thornber 55 | -------------------------------------------------------------------------------- /man8/thin_repair.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | thin_repair - repair thin provisioning binary metadata. 3 | 4 | SYNOPSIS 5 | thin_repair [options] -i {device|file} -o {device|file} 6 | 7 | DESCRIPTION 8 | thin_repair reads binary thin provisioning metadata created by the respective 9 | device-mapper target from one device or file, repairs it and writes it 10 | to different device or file. If written to a metadata device, the metadata 11 | can be processed by the device-mapper target. 12 | 13 | This tool cannot be run on live metadata. 14 | 15 | OPTIONS 16 | -h, --help Print help and exit. 17 | -V, --version Print version information and exit. 18 | -i, --input {device|file} Input file or device with binary data. 19 | -o, --output {device|file} Output file or device for binary data. 20 | 21 | If a file is used for output, then it must be preallocated, and large 22 | enough to hold the metadata. 23 | 24 | --transaction-id {natural} Override the transaction id given in the input xml. 25 | --data-block-size {natural} Override the data block size given in the input xml. 26 | --nr-data-blocks {natural} Override the nr data blocks given in the input xml. 27 | 28 | EXAMPLE 29 | 30 | Reads the binary thin provisioning metadata from file metadata, repairs 31 | it and writes it to logical volume /dev/vg/metadata for further processing by 32 | the respective device-mapper target: 33 | 34 | $ thin_repair -i metadata -o /dev/vg/metadata 35 | 36 | DIAGNOSTICS 37 | thin_repair returns an exit code of 0 for success or 1 for error. 38 | 39 | SEE ALSO 40 | thin_dump(8), thin_check(8), thin_restore(8), thin_rmap(8), thin_metadata_size(8) 41 | 42 | AUTHOR 43 | Joe Thornber , Heinz Mauelshagen 44 | 45 | -------------------------------------------------------------------------------- /man8/thin_restore.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | thin_restore - restore thin provisioning metadata file to device or file. 3 | 4 | SYNOPSIS 5 | thin_restore [options] -i {xml file} -o {device|file} 6 | 7 | DESCRIPTION 8 | thin_restore restores thin provisioning metadata created by the respective 9 | device-mapper target dumped into an XML formatted (see thin_dump(8)) file, 10 | which optionally can be preprocessed before the restore to another device or 11 | file. If restored to a metadata device, the metadata can be processed by the 12 | device-mapper target. 13 | 14 | This tool cannot be run on live metadata. 15 | 16 | OPTIONS 17 | -h, --help Print help and exit. 18 | -V, --version Print version information and exit. 19 | -q, --quiet Suppress output messages, return only exit code. 20 | -i, --input {xml file} Input file containing XML metadata. 21 | -o, --output {device|file} Output file or device for restored binary metadata. 22 | 23 | If a file is used for output, then it must be preallocated, and large 24 | enough to hold the metadata. 25 | 26 | --transaction-id {natural} Override the transaction id given in the input xml. 27 | --data-block-size {natural} Override the data block size given in the input xml. 28 | --nr-data-blocks {natural} Override the nr data blocks given in the input xml. 29 | 30 | EXAMPLE 31 | 32 | Restores the XML formatted thin provisioning metadata on file metadata to 33 | logical volume /dev/vg/metadata for further processing by the respective 34 | device-mapper target: 35 | 36 | $ thin_restore -i metadata -o /dev/vg/metadata 37 | 38 | DIAGNOSTICS 39 | 40 | thin_restore returns an exit code of 0 for success or 1 for error. 41 | 42 | SEE ALSO 43 | thin_dump(8), thin_check(8), thin_repair(8), thin_rmap(8), thin_metadata_size(8) 44 | 45 | AUTHOR 46 | Joe Thornber , Heinz Mauelshagen 47 | -------------------------------------------------------------------------------- /man8/thin_rmap.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | thin_rmap - output reverse map of a thin provisioned region of blocks from 3 | metadata device or file. 4 | 5 | SYNOPSIS 6 | thin_rmap [options] {device|file} 7 | 8 | DESCRIPTION 9 | thin_rmap outputs the reverse mapping stored in the metadata on a device or 10 | file between a region of thin provisioned pool blocks and the associated thin 11 | provisioned devices. 12 | 13 | This tool cannot be run on live metadata. 14 | 15 | OPTIONS 16 | -h, --help Print help and exit. 17 | -V, --version Print version information and exit. 18 | --region {block range} Specify range of blocks on the data device. 19 | 20 | At least one region must be specified. Multiple regions may be specified. 21 | 22 | The range takes the format ... For example, 23 | "5..45" specifies data blocks 5 to 44 inclusive, but not 45. 24 | 25 | EXAMPLES 26 | 27 | $ thin_rmap --region 5..45 /dev/pool-metadata 28 | 29 | DIAGNOSTICS 30 | thin_rmap returns an exit code of 0 for success or 1 for error. 31 | 32 | SEE ALSO 33 | thin_check(8), thin_dump(8), thin_repair(8), thin_restore(8), thin_metadata_size(8) 34 | 35 | AUTHOR 36 | Joe Thornber , Heinz Mauelshagen 37 | 38 | -------------------------------------------------------------------------------- /man8/thin_trim.txt: -------------------------------------------------------------------------------- 1 | NAME 2 | thin_trim - Issue discard requests for free pool space (offline tool). 3 | 4 | SYNOPSIS 5 | thin_trim [options] --metadata-dev {device|file} --data-dev {device|file} 6 | 7 | DESCRIPTION 8 | thin_trim sends discard requests to the pool device for unprovisioned areas. 9 | 10 | This tool cannot be run on live metadata. 11 | 12 | OPTIONS 13 | -h, --help Print help and exit. 14 | -V, --version Print version information and exit. 15 | 16 | SEE ALSO 17 | thin_dump(8), thin_repair(8), thin_restore(8), thin_rmap(8), thin_metadata_size(8) 18 | 19 | AUTHOR 20 | Joe Thornber 21 | -------------------------------------------------------------------------------- /mk_release: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | orig_dir=$(pwd) 6 | 7 | # usage: mk_release 8 | tag=$1 9 | echo "creating release tarball for tag '"$tag"'" 10 | 11 | tmp=$(mktemp -d) 12 | dir=$tmp/thin-provisioning-tools-$tag 13 | 14 | mkdir $dir 15 | 16 | git clone . $dir 17 | cd $dir 18 | git checkout $tag 19 | autoreconf 20 | rm -rf .git 21 | cd $tmp 22 | tar jcvf $orig_dir/thin-provisioning-tools-$tag.tar.bz2 thin-provisioning-tools-$tag 23 | cd $orig_dir 24 | 25 | rm -rf $tmp 26 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | reorder_imports = true 2 | -------------------------------------------------------------------------------- /src/bin/pdata_tools.rs: -------------------------------------------------------------------------------- 1 | use std::ffi::OsStr; 2 | use std::path::Path; 3 | use std::process::exit; 4 | 5 | use thinp::commands::*; 6 | 7 | fn get_basename(path: &OsStr) -> &Path { 8 | let p = Path::new(path); 9 | Path::new(p.file_name().unwrap()) 10 | } 11 | 12 | fn register_commands<'a>() -> Vec>> { 13 | vec![ 14 | Box::new(cache_check::CacheCheckCommand), 15 | Box::new(cache_dump::CacheDumpCommand), 16 | Box::new(cache_metadata_size::CacheMetadataSizeCommand), 17 | Box::new(cache_repair::CacheRepairCommand), 18 | Box::new(cache_restore::CacheRestoreCommand), 19 | Box::new(cache_writeback::CacheWritebackCommand), 20 | Box::new(era_check::EraCheckCommand), 21 | Box::new(era_dump::EraDumpCommand), 22 | Box::new(era_invalidate::EraInvalidateCommand), 23 | Box::new(era_repair::EraRepairCommand), 24 | Box::new(era_restore::EraRestoreCommand), 25 | Box::new(thin_check::ThinCheckCommand), 26 | Box::new(thin_delta::ThinDeltaCommand), 27 | Box::new(thin_dump::ThinDumpCommand), 28 | Box::new(thin_ls::ThinLsCommand), 29 | Box::new(thin_metadata_pack::ThinMetadataPackCommand), 30 | Box::new(thin_metadata_size::ThinMetadataSizeCommand), 31 | Box::new(thin_metadata_unpack::ThinMetadataUnpackCommand), 32 | Box::new(thin_migrate::ThinMigrateCommand), 33 | Box::new(thin_repair::ThinRepairCommand), 34 | Box::new(thin_restore::ThinRestoreCommand), 35 | Box::new(thin_rmap::ThinRmapCommand), 36 | Box::new(thin_shrink::ThinShrinkCommand), 37 | Box::new(thin_trim::ThinTrimCommand), 38 | ] 39 | } 40 | 41 | fn usage(commands: &[Box]) { 42 | eprintln!("Usage: "); 43 | eprintln!("commands:"); 44 | commands.iter().for_each(|c| eprintln!(" {}", c.name())); 45 | } 46 | 47 | fn main_() -> exitcode::ExitCode { 48 | let commands = register_commands(); 49 | let mut args = std::env::args_os().peekable(); 50 | 51 | args.next_if(|path| get_basename(path) == Path::new("pdata_tools")); 52 | let cmd = args.peek(); 53 | 54 | if cmd.is_none() { 55 | usage(&commands); 56 | return exitcode::USAGE; 57 | }; 58 | 59 | if let Some(c) = commands 60 | .iter() 61 | .find(|c| get_basename(cmd.unwrap()) == Path::new(c.name())) 62 | { 63 | c.run(&mut args) 64 | } else { 65 | eprintln!("unrecognised command"); 66 | usage(&commands); 67 | exitcode::USAGE 68 | } 69 | } 70 | 71 | fn main() { 72 | exit(main_()) 73 | } 74 | -------------------------------------------------------------------------------- /src/bin/pdata_tools_dev.rs: -------------------------------------------------------------------------------- 1 | use std::ffi::OsStr; 2 | use std::path::Path; 3 | use std::process::exit; 4 | 5 | use thinp::commands::*; 6 | 7 | fn get_basename(path: &OsStr) -> &Path { 8 | let p = Path::new(path); 9 | Path::new(p.file_name().unwrap()) 10 | } 11 | 12 | fn register_commands<'a>() -> Vec>> { 13 | vec![ 14 | Box::new(era_generate_metadata::EraGenerateMetadataCommand), 15 | Box::new(cache_generate_metadata::CacheGenerateMetadataCommand), 16 | Box::new(cache_generate_damage::CacheGenerateDamageCommand), 17 | Box::new(thin_explore::ThinExploreCommand), 18 | Box::new(thin_generate_metadata::ThinGenerateMetadataCommand), 19 | Box::new(thin_generate_damage::ThinGenerateDamageCommand), 20 | Box::new(thin_stat::ThinStatCommand), 21 | ] 22 | } 23 | 24 | fn usage(commands: &[Box]) { 25 | eprintln!("Usage: "); 26 | eprintln!("commands:"); 27 | commands.iter().for_each(|c| eprintln!(" {}", c.name())); 28 | } 29 | 30 | fn main_() -> exitcode::ExitCode { 31 | let commands = register_commands(); 32 | let mut args = std::env::args_os().peekable(); 33 | 34 | args.next_if(|path| get_basename(path) == Path::new("pdata_tools_dev")); 35 | let cmd = args.peek(); 36 | 37 | if cmd.is_none() { 38 | usage(&commands); 39 | return exitcode::USAGE; 40 | }; 41 | 42 | if let Some(c) = commands 43 | .iter() 44 | .find(|c| get_basename(cmd.unwrap()) == Path::new(c.name())) 45 | { 46 | c.run(&mut args) 47 | } else { 48 | eprintln!("unrecognised command"); 49 | usage(&commands); 50 | exitcode::USAGE 51 | } 52 | } 53 | 54 | fn main() { 55 | exit(main_()) 56 | } 57 | -------------------------------------------------------------------------------- /src/cache/damage_generator.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::path::Path; 3 | 4 | use crate::cache::superblock::*; 5 | use crate::commands::engine::*; 6 | use crate::devtools::damage_generator::*; 7 | use crate::pdata::space_map::common::*; 8 | use crate::pdata::unpack::unpack; 9 | 10 | //------------------------------------------ 11 | 12 | pub enum DamageOp { 13 | CreateMetadataLeaks { 14 | nr_blocks: usize, 15 | expected_rc: u32, 16 | actual_rc: u32, 17 | }, 18 | } 19 | 20 | pub struct CacheDamageOpts<'a> { 21 | pub engine_opts: EngineOptions, 22 | pub op: DamageOp, 23 | pub output: &'a Path, 24 | } 25 | 26 | pub fn damage_metadata(opts: CacheDamageOpts) -> Result<()> { 27 | let engine = EngineBuilder::new(opts.output, &opts.engine_opts) 28 | .write(true) 29 | .build()?; 30 | let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; 31 | let sm_root = unpack::(&sb.metadata_sm_root)?; 32 | 33 | match opts.op { 34 | DamageOp::CreateMetadataLeaks { 35 | nr_blocks, 36 | expected_rc, 37 | actual_rc, 38 | } => create_metadata_leaks(engine, sm_root, nr_blocks, expected_rc, actual_rc), 39 | } 40 | } 41 | 42 | //------------------------------------------ 43 | -------------------------------------------------------------------------------- /src/cache/hint.rs: -------------------------------------------------------------------------------- 1 | use byteorder::WriteBytesExt; 2 | use nom::IResult; 3 | use std::convert::TryInto; 4 | use std::io; 5 | 6 | use crate::pdata::unpack::*; 7 | 8 | //------------------------------------------ 9 | 10 | #[derive(Clone, Copy, Default)] 11 | pub struct Hint { 12 | pub hint: [u8; 4], 13 | } 14 | 15 | impl Unpack for Hint { 16 | fn disk_size() -> u32 { 17 | 4 18 | } 19 | 20 | fn unpack(i: &[u8]) -> IResult<&[u8], Hint> { 21 | let size = 4; 22 | Ok(( 23 | &i[size..], 24 | Hint { 25 | hint: i[0..size].try_into().unwrap(), 26 | }, 27 | )) 28 | } 29 | } 30 | 31 | impl Pack for Hint { 32 | fn pack(&self, data: &mut W) -> io::Result<()> { 33 | for v in &self.hint { 34 | data.write_u8(*v)?; 35 | } 36 | Ok(()) 37 | } 38 | } 39 | 40 | //------------------------------------------ 41 | -------------------------------------------------------------------------------- /src/cache/ir.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | 3 | //------------------------------------------ 4 | 5 | #[derive(Clone)] 6 | pub struct Superblock { 7 | pub uuid: String, 8 | pub block_size: u32, 9 | pub nr_cache_blocks: u32, 10 | pub policy: String, 11 | pub hint_width: u32, 12 | } 13 | 14 | #[derive(Clone)] 15 | pub struct Map { 16 | pub cblock: u32, 17 | pub oblock: u64, 18 | pub dirty: bool, 19 | } 20 | 21 | #[derive(Clone)] 22 | pub struct Hint { 23 | pub cblock: u32, 24 | pub data: Vec, 25 | } 26 | 27 | #[derive(Clone)] 28 | pub struct Discard { 29 | pub begin: u64, 30 | pub end: u64, 31 | } 32 | 33 | //------------------------------------------ 34 | 35 | #[derive(Clone)] 36 | pub enum Visit { 37 | Continue, 38 | Stop, 39 | } 40 | 41 | pub trait MetadataVisitor { 42 | fn superblock_b(&mut self, sb: &Superblock) -> Result; 43 | fn superblock_e(&mut self) -> Result; 44 | 45 | fn mappings_b(&mut self) -> Result; 46 | fn mappings_e(&mut self) -> Result; 47 | fn mapping(&mut self, m: &Map) -> Result; 48 | 49 | fn hints_b(&mut self) -> Result; 50 | fn hints_e(&mut self) -> Result; 51 | fn hint(&mut self, h: &Hint) -> Result; 52 | 53 | fn discards_b(&mut self) -> Result; 54 | fn discards_e(&mut self) -> Result; 55 | fn discard(&mut self, d: &Discard) -> Result; 56 | 57 | fn eof(&mut self) -> Result; 58 | } 59 | 60 | //------------------------------------------ 61 | -------------------------------------------------------------------------------- /src/cache/mapping.rs: -------------------------------------------------------------------------------- 1 | use byteorder::WriteBytesExt; 2 | use nom::number::complete::*; 3 | use nom::IResult; 4 | use std::io; 5 | 6 | use crate::pdata::unpack::*; 7 | 8 | //------------------------------------------ 9 | 10 | pub const MAX_ORIGIN_BLOCKS: u64 = 1 << 48; 11 | const FLAGS_MASK: u64 = (1 << 16) - 1; 12 | 13 | //------------------------------------------ 14 | 15 | pub enum MappingFlags { 16 | Valid = 1, 17 | Dirty = 2, 18 | } 19 | 20 | #[derive(Clone, Copy, Default)] 21 | pub struct Mapping { 22 | pub oblock: u64, 23 | pub flags: u32, 24 | } 25 | 26 | impl Mapping { 27 | pub fn is_valid(&self) -> bool { 28 | (self.flags & MappingFlags::Valid as u32) != 0 29 | } 30 | 31 | pub fn is_dirty(&self) -> bool { 32 | (self.flags & MappingFlags::Dirty as u32) != 0 33 | } 34 | 35 | pub fn set_dirty(&mut self, dirty: bool) { 36 | if dirty { 37 | self.flags |= MappingFlags::Dirty as u32; 38 | } else { 39 | self.flags &= !(MappingFlags::Dirty as u32); 40 | } 41 | } 42 | } 43 | 44 | impl Unpack for Mapping { 45 | fn disk_size() -> u32 { 46 | 8 47 | } 48 | 49 | fn unpack(i: &[u8]) -> IResult<&[u8], Mapping> { 50 | let (i, n) = le_u64(i)?; 51 | let oblock = n >> 16; 52 | let flags = n & FLAGS_MASK; 53 | 54 | Ok(( 55 | i, 56 | Mapping { 57 | oblock, 58 | flags: flags as u32, 59 | }, 60 | )) 61 | } 62 | } 63 | 64 | impl Pack for Mapping { 65 | fn pack(&self, data: &mut W) -> io::Result<()> { 66 | let m: u64 = (self.oblock << 16) | self.flags as u64; 67 | m.pack(data) 68 | } 69 | } 70 | 71 | //------------------------------------------ 72 | -------------------------------------------------------------------------------- /src/cache/metadata_size.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{anyhow, Result}; 2 | 3 | use crate::io_engine::BLOCK_SIZE; 4 | use crate::pdata::space_map::metadata::MAX_METADATA_BLOCKS; 5 | 6 | //------------------------------------------ 7 | 8 | const MIN_CACHE_BLOCK_SIZE: u64 = 32768; 9 | const MAX_CACHE_BLOCK_SIZE: u64 = 1073741824; 10 | 11 | pub struct CacheMetadataSizeOptions { 12 | pub nr_blocks: u64, 13 | pub max_hint_width: u32, // bytes 14 | } 15 | 16 | pub fn check_cache_block_size(block_size: u64) -> Result<()> { 17 | if block_size == 0 || (block_size & (MIN_CACHE_BLOCK_SIZE - 1)) != 0 { 18 | return Err(anyhow!("block size must be a multiple of 32 KiB")); 19 | } 20 | 21 | if block_size > MAX_CACHE_BLOCK_SIZE { 22 | return Err(anyhow!("maximum block size is 1 GiB")); 23 | } 24 | 25 | Ok(()) 26 | } 27 | 28 | // Returns estimated size in bytes 29 | pub fn metadata_size(opts: &CacheMetadataSizeOptions) -> Result { 30 | const BYTES_PER_BLOCK_SHIFT: u64 = 4; // 16 bytes for key and value 31 | const TRANSACTION_OVERHEAD: u64 = 4 * 1024 * 1024; // 4 MB 32 | const HINT_OVERHEAD_PER_BLOCK: u64 = 8; // 8 bytes for the key 33 | 34 | let mapping_size = opts.nr_blocks << BYTES_PER_BLOCK_SHIFT; 35 | let hint_size = opts.nr_blocks * (opts.max_hint_width as u64 + HINT_OVERHEAD_PER_BLOCK); 36 | 37 | let mut size = TRANSACTION_OVERHEAD + mapping_size + hint_size; 38 | size = std::cmp::min(size, MAX_METADATA_BLOCKS as u64 * BLOCK_SIZE as u64); 39 | 40 | Ok(size) 41 | } 42 | 43 | //------------------------------------------ 44 | -------------------------------------------------------------------------------- /src/cache/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod check; 2 | pub mod dump; 3 | pub mod hint; 4 | pub mod ir; 5 | pub mod mapping; 6 | pub mod metadata_size; 7 | pub mod repair; 8 | pub mod restore; 9 | pub mod superblock; 10 | pub mod writeback; 11 | pub mod xml; 12 | 13 | #[cfg(feature = "devtools")] 14 | pub mod metadata_generator; 15 | 16 | #[cfg(feature = "devtools")] 17 | pub mod damage_generator; 18 | -------------------------------------------------------------------------------- /src/cache/repair.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::path::Path; 3 | use std::sync::Arc; 4 | 5 | use crate::cache::dump::*; 6 | use crate::cache::restore::*; 7 | use crate::cache::superblock::*; 8 | use crate::commands::engine::*; 9 | use crate::io_engine::*; 10 | use crate::pdata::space_map::metadata::*; 11 | use crate::report::*; 12 | use crate::write_batcher::*; 13 | 14 | //------------------------------------------ 15 | 16 | pub struct CacheRepairOptions<'a> { 17 | pub input: &'a Path, 18 | pub output: &'a Path, 19 | pub engine_opts: EngineOptions, 20 | pub report: Arc, 21 | } 22 | 23 | struct Context { 24 | _report: Arc, 25 | engine_in: Arc, 26 | engine_out: Arc, 27 | } 28 | 29 | fn new_context(opts: &CacheRepairOptions) -> Result { 30 | let engine_in = EngineBuilder::new(opts.input, &opts.engine_opts).build()?; 31 | let engine_out = EngineBuilder::new(opts.output, &opts.engine_opts) 32 | .write(true) 33 | .build()?; 34 | 35 | Ok(Context { 36 | _report: opts.report.clone(), 37 | engine_in, 38 | engine_out, 39 | }) 40 | } 41 | 42 | //------------------------------------------ 43 | 44 | pub fn repair(opts: CacheRepairOptions) -> Result<()> { 45 | let ctx = new_context(&opts)?; 46 | 47 | let sb = read_superblock(ctx.engine_in.as_ref(), SUPERBLOCK_LOCATION)?; 48 | 49 | let sm = core_metadata_sm(ctx.engine_out.get_nr_blocks(), u32::MAX); 50 | let batch_size = ctx.engine_out.get_batch_size(); 51 | let mut w = WriteBatcher::new(ctx.engine_out, sm.clone(), batch_size); 52 | let mut restorer = Restorer::new(&mut w, sb.version as u8); 53 | 54 | dump_metadata(ctx.engine_in, &mut restorer, &sb, true) 55 | } 56 | 57 | //------------------------------------------ 58 | -------------------------------------------------------------------------------- /src/checksum.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{anyhow, Result}; 2 | use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; 3 | use crc32c::crc32c; 4 | 5 | use std::io::Cursor; 6 | 7 | const BLOCK_SIZE: u64 = 4096; 8 | const THIN_SUPERBLOCK_CSUM_XOR: u32 = 160774; 9 | const CACHE_SUPERBLOCK_CSUM_XOR: u32 = 9031977; 10 | const ERA_SUPERBLOCK_CSUM_XOR: u32 = 146538381; 11 | const BITMAP_CSUM_XOR: u32 = 240779; 12 | const INDEX_CSUM_XOR: u32 = 160478; 13 | const BTREE_CSUM_XOR: u32 = 121107; 14 | const ARRAY_CSUM_XOR: u32 = 595846735; 15 | 16 | fn checksum(buf: &[u8]) -> u32 { 17 | crc32c(&buf[4..]) ^ 0xffffffff 18 | } 19 | 20 | #[derive(Debug, PartialEq, Eq)] 21 | #[allow(clippy::upper_case_acronyms)] 22 | #[allow(non_camel_case_types)] 23 | pub enum BT { 24 | THIN_SUPERBLOCK, 25 | CACHE_SUPERBLOCK, 26 | ERA_SUPERBLOCK, 27 | NODE, 28 | INDEX, 29 | BITMAP, 30 | ARRAY, 31 | UNKNOWN, 32 | } 33 | 34 | pub fn metadata_block_type(buf: &[u8]) -> BT { 35 | if buf.len() != BLOCK_SIZE as usize { 36 | return BT::UNKNOWN; 37 | } 38 | 39 | // The checksum is always stored in the first u32 of the buffer. 40 | let mut rdr = Cursor::new(buf); 41 | let sum_on_disk = rdr.read_u32::().unwrap(); 42 | let csum = checksum(buf); 43 | let btype = csum ^ sum_on_disk; 44 | 45 | match btype { 46 | THIN_SUPERBLOCK_CSUM_XOR => BT::THIN_SUPERBLOCK, 47 | CACHE_SUPERBLOCK_CSUM_XOR => BT::CACHE_SUPERBLOCK, 48 | ERA_SUPERBLOCK_CSUM_XOR => BT::ERA_SUPERBLOCK, 49 | BTREE_CSUM_XOR => BT::NODE, 50 | BITMAP_CSUM_XOR => BT::BITMAP, 51 | INDEX_CSUM_XOR => BT::INDEX, 52 | ARRAY_CSUM_XOR => BT::ARRAY, 53 | _ => BT::UNKNOWN, 54 | } 55 | } 56 | 57 | pub fn write_checksum(buf: &mut [u8], kind: BT) -> Result<()> { 58 | if buf.len() != BLOCK_SIZE as usize { 59 | return Err(anyhow!("block is wrong size")); 60 | } 61 | 62 | use BT::*; 63 | let salt = match kind { 64 | THIN_SUPERBLOCK => THIN_SUPERBLOCK_CSUM_XOR, 65 | CACHE_SUPERBLOCK => CACHE_SUPERBLOCK_CSUM_XOR, 66 | ERA_SUPERBLOCK => ERA_SUPERBLOCK_CSUM_XOR, 67 | NODE => BTREE_CSUM_XOR, 68 | BITMAP => BITMAP_CSUM_XOR, 69 | INDEX => INDEX_CSUM_XOR, 70 | ARRAY => ARRAY_CSUM_XOR, 71 | UNKNOWN => { 72 | return Err(anyhow!("Invalid block type")); 73 | } 74 | }; 75 | 76 | let csum = checksum(buf) ^ salt; 77 | let mut out = std::io::Cursor::new(buf); 78 | out.write_u32::(csum)?; 79 | Ok(()) 80 | } 81 | -------------------------------------------------------------------------------- /src/commands/cache_dump.rs: -------------------------------------------------------------------------------- 1 | extern crate clap; 2 | 3 | use clap::{Arg, ArgAction}; 4 | use std::path::Path; 5 | 6 | use crate::cache::dump::{dump, CacheDumpOptions}; 7 | use crate::commands::engine::*; 8 | use crate::commands::utils::*; 9 | use crate::commands::Command; 10 | use crate::version::*; 11 | 12 | //------------------------------------------ 13 | 14 | pub struct CacheDumpCommand; 15 | 16 | impl CacheDumpCommand { 17 | fn cli(&self) -> clap::Command { 18 | let cmd = clap::Command::new(self.name()) 19 | .next_display_order(None) 20 | .version(crate::tools_version!()) 21 | .disable_version_flag(true) 22 | .about("Dump the cache metadata to stdout in XML format") 23 | .arg( 24 | Arg::new("REPAIR") 25 | .help("Repair the metadata whilst dumping it") 26 | .short('r') 27 | .long("repair") 28 | .action(ArgAction::SetTrue), 29 | ) 30 | // options 31 | .arg( 32 | Arg::new("OUTPUT") 33 | .help("Specify the output file rather than stdout") 34 | .short('o') 35 | .long("output") 36 | .value_name("FILE"), 37 | ) 38 | // arguments 39 | .arg( 40 | Arg::new("INPUT") 41 | .help("Specify the input device to dump") 42 | .required(true) 43 | .index(1), 44 | ); 45 | engine_args(version_args(cmd)) 46 | } 47 | } 48 | 49 | impl<'a> Command<'a> for CacheDumpCommand { 50 | fn name(&self) -> &'a str { 51 | "cache_dump" 52 | } 53 | 54 | fn run(&self, args: &mut dyn Iterator) -> exitcode::ExitCode { 55 | let matches = self.cli().get_matches_from(args); 56 | display_version(&matches); 57 | 58 | let input_file = Path::new(matches.get_one::("INPUT").unwrap()); 59 | let output_file = matches.get_one::("OUTPUT").map(Path::new); 60 | 61 | // Create a temporary report just in case these checks 62 | // need to report anything. 63 | let report = std::sync::Arc::new(crate::report::mk_simple_report()); 64 | 65 | if let Err(e) = check_input_file(input_file).and_then(check_file_not_tiny) { 66 | return to_exit_code::<()>(&report, Err(e)); 67 | } 68 | 69 | let engine_opts = parse_engine_opts(ToolType::Cache, &matches); 70 | if engine_opts.is_err() { 71 | return to_exit_code(&report, engine_opts); 72 | } 73 | let engine_opts = engine_opts.unwrap(); 74 | 75 | let opts = CacheDumpOptions { 76 | input: input_file, 77 | output: output_file, 78 | engine_opts, 79 | repair: matches.get_flag("REPAIR"), 80 | }; 81 | 82 | to_exit_code(&report, dump(opts)) 83 | } 84 | } 85 | 86 | //------------------------------------------ 87 | -------------------------------------------------------------------------------- /src/commands/cache_repair.rs: -------------------------------------------------------------------------------- 1 | extern crate clap; 2 | 3 | use clap::{Arg, ArgAction}; 4 | use std::path::Path; 5 | 6 | use crate::cache::repair::{repair, CacheRepairOptions}; 7 | use crate::commands::engine::*; 8 | use crate::commands::utils::*; 9 | use crate::commands::Command; 10 | use crate::report::*; 11 | use crate::version::*; 12 | 13 | pub struct CacheRepairCommand; 14 | 15 | impl CacheRepairCommand { 16 | fn cli(&self) -> clap::Command { 17 | let cmd = clap::Command::new(self.name()) 18 | .next_display_order(None) 19 | .version(crate::tools_version!()) 20 | .disable_version_flag(true) 21 | .about("Repair binary cache metadata, and write it to a different device or file") 22 | .arg( 23 | Arg::new("QUIET") 24 | .help("Suppress output messages, return only exit code.") 25 | .short('q') 26 | .long("quiet") 27 | .action(ArgAction::SetTrue), 28 | ) 29 | // options 30 | .arg( 31 | Arg::new("INPUT") 32 | .help("Specify the input device") 33 | .short('i') 34 | .long("input") 35 | .value_name("FILE") 36 | .required(true), 37 | ) 38 | .arg( 39 | Arg::new("OUTPUT") 40 | .help("Specify the output device") 41 | .short('o') 42 | .long("output") 43 | .value_name("FILE") 44 | .required(true), 45 | ) 46 | // a dummy argument for compatibility with lvconvert 47 | .arg(Arg::new("DUMMY").required(false).hide(true).index(1)); 48 | 49 | verbose_args(engine_args(version_args(cmd))) 50 | } 51 | } 52 | 53 | impl<'a> Command<'a> for CacheRepairCommand { 54 | fn name(&self) -> &'a str { 55 | "cache_repair" 56 | } 57 | 58 | fn run(&self, args: &mut dyn Iterator) -> exitcode::ExitCode { 59 | let matches = self.cli().get_matches_from(args); 60 | display_version(&matches); 61 | 62 | let report = mk_report(matches.get_flag("QUIET")); 63 | let log_level = match parse_log_level(&matches) { 64 | Ok(level) => level, 65 | Err(e) => return to_exit_code::<()>(&report, Err(anyhow::Error::msg(e))), 66 | }; 67 | report.set_level(log_level); 68 | 69 | let input_file = Path::new(matches.get_one::("INPUT").unwrap()); 70 | let output_file = Path::new(matches.get_one::("OUTPUT").unwrap()); 71 | 72 | if let Err(e) = check_input_file(input_file) 73 | .and_then(check_file_not_tiny) 74 | .and_then(|_| check_output_file(output_file)) 75 | { 76 | return to_exit_code::<()>(&report, Err(e)); 77 | } 78 | 79 | let engine_opts = parse_engine_opts(ToolType::Cache, &matches); 80 | if engine_opts.is_err() { 81 | return to_exit_code(&report, engine_opts); 82 | } 83 | let engine_opts = engine_opts.unwrap(); 84 | 85 | let opts = CacheRepairOptions { 86 | input: input_file, 87 | output: output_file, 88 | engine_opts, 89 | report: report.clone(), 90 | }; 91 | 92 | to_exit_code(&report, repair(opts)) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/commands/era_check.rs: -------------------------------------------------------------------------------- 1 | extern crate clap; 2 | 3 | use clap::{Arg, ArgAction}; 4 | use std::path::Path; 5 | 6 | use crate::commands::engine::*; 7 | use crate::commands::utils::*; 8 | use crate::commands::Command; 9 | use crate::era::check::{check, EraCheckOptions}; 10 | use crate::report::*; 11 | use crate::version::*; 12 | 13 | //------------------------------------------ 14 | 15 | pub struct EraCheckCommand; 16 | 17 | impl EraCheckCommand { 18 | fn cli(&self) -> clap::Command { 19 | let cmd = clap::Command::new(self.name()) 20 | .next_display_order(None) 21 | .version(crate::tools_version!()) 22 | .disable_version_flag(true) 23 | .about("Validate era metadata on device or file.") 24 | // flags 25 | .arg( 26 | Arg::new("IGNORE_NON_FATAL") 27 | .help("Only return a non-zero exit code if a fatal error is found.") 28 | .long("ignore-non-fatal-errors") 29 | .action(ArgAction::SetTrue), 30 | ) 31 | .arg( 32 | Arg::new("QUIET") 33 | .help("Suppress output messages, return only exit code.") 34 | .short('q') 35 | .long("quiet") 36 | .action(ArgAction::SetTrue), 37 | ) 38 | .arg( 39 | Arg::new("SB_ONLY") 40 | .help("Only check the superblock.") 41 | .long("super-block-only") 42 | .action(ArgAction::SetTrue), 43 | ) 44 | // arguments 45 | .arg( 46 | Arg::new("INPUT") 47 | .help("Specify the input device to check") 48 | .required(true) 49 | .index(1), 50 | ); 51 | verbose_args(engine_args(version_args(cmd))) 52 | } 53 | } 54 | 55 | impl<'a> Command<'a> for EraCheckCommand { 56 | fn name(&self) -> &'a str { 57 | "era_check" 58 | } 59 | 60 | fn run(&self, args: &mut dyn Iterator) -> exitcode::ExitCode { 61 | let matches = self.cli().get_matches_from(args); 62 | display_version(&matches); 63 | 64 | let input_file = Path::new(matches.get_one::("INPUT").unwrap()); 65 | 66 | let report = mk_report(matches.get_flag("QUIET")); 67 | let log_level = match parse_log_level(&matches) { 68 | Ok(level) => level, 69 | Err(e) => return to_exit_code::<()>(&report, Err(anyhow::Error::msg(e))), 70 | }; 71 | report.set_level(log_level); 72 | 73 | if let Err(e) = check_input_file(input_file) 74 | .and_then(check_file_not_tiny) 75 | .and_then(check_not_xml) 76 | { 77 | return to_exit_code::<()>(&report, Err(e)); 78 | } 79 | 80 | let engine_opts = parse_engine_opts(ToolType::Era, &matches); 81 | if engine_opts.is_err() { 82 | return to_exit_code(&report, engine_opts); 83 | } 84 | 85 | let opts = EraCheckOptions { 86 | dev: input_file, 87 | engine_opts: engine_opts.unwrap(), 88 | sb_only: matches.get_flag("SB_ONLY"), 89 | ignore_non_fatal: matches.get_flag("IGNORE_NON_FATAL"), 90 | report: report.clone(), 91 | }; 92 | 93 | to_exit_code(&report, check(&opts)) 94 | } 95 | } 96 | 97 | //------------------------------------------ 98 | -------------------------------------------------------------------------------- /src/commands/era_dump.rs: -------------------------------------------------------------------------------- 1 | extern crate clap; 2 | 3 | use clap::{Arg, ArgAction}; 4 | use std::path::Path; 5 | 6 | use crate::commands::engine::*; 7 | use crate::commands::utils::*; 8 | use crate::commands::Command; 9 | use crate::era::dump::{dump, EraDumpOptions}; 10 | use crate::version::*; 11 | 12 | //------------------------------------------ 13 | 14 | pub struct EraDumpCommand; 15 | 16 | impl EraDumpCommand { 17 | fn cli(&self) -> clap::Command { 18 | let cmd = clap::Command::new(self.name()) 19 | .next_display_order(None) 20 | .version(crate::tools_version!()) 21 | .disable_version_flag(true) 22 | .about("Dump the era metadata to stdout in XML format") 23 | .arg( 24 | Arg::new("LOGICAL") 25 | .help("Fold any unprocessed write sets into the final era array") 26 | .long("logical") 27 | .action(ArgAction::SetTrue), 28 | ) 29 | .arg( 30 | Arg::new("REPAIR") 31 | .help("Repair the metadata whilst dumping it") 32 | .short('r') 33 | .long("repair") 34 | .action(ArgAction::SetTrue), 35 | ) 36 | // options 37 | .arg( 38 | Arg::new("OUTPUT") 39 | .help("Specify the output file rather than stdout") 40 | .short('o') 41 | .long("output") 42 | .value_name("FILE"), 43 | ) 44 | // arguments 45 | .arg( 46 | Arg::new("INPUT") 47 | .help("Specify the input device to dump") 48 | .required(true) 49 | .index(1), 50 | ); 51 | engine_args(version_args(cmd)) 52 | } 53 | } 54 | 55 | impl<'a> Command<'a> for EraDumpCommand { 56 | fn name(&self) -> &'a str { 57 | "era_dump" 58 | } 59 | 60 | fn run(&self, args: &mut dyn Iterator) -> exitcode::ExitCode { 61 | let matches = self.cli().get_matches_from(args); 62 | display_version(&matches); 63 | 64 | let input_file = Path::new(matches.get_one::("INPUT").unwrap()); 65 | let output_file = matches.get_one::("OUTPUT").map(Path::new); 66 | 67 | let report = std::sync::Arc::new(crate::report::mk_simple_report()); 68 | 69 | if let Err(e) = check_input_file(input_file).and_then(check_file_not_tiny) { 70 | return to_exit_code::<()>(&report, Err(e)); 71 | } 72 | 73 | let engine_opts = parse_engine_opts(ToolType::Era, &matches); 74 | if engine_opts.is_err() { 75 | return to_exit_code(&report, engine_opts); 76 | } 77 | 78 | let opts = EraDumpOptions { 79 | input: input_file, 80 | output: output_file, 81 | engine_opts: engine_opts.unwrap(), 82 | logical: matches.get_flag("LOGICAL"), 83 | repair: matches.get_flag("REPAIR"), 84 | }; 85 | 86 | to_exit_code(&report, dump(opts)) 87 | } 88 | } 89 | 90 | //------------------------------------------ 91 | -------------------------------------------------------------------------------- /src/commands/era_invalidate.rs: -------------------------------------------------------------------------------- 1 | use clap::{value_parser, Arg}; 2 | use std::path::Path; 3 | 4 | use crate::commands::engine::*; 5 | use crate::commands::utils::*; 6 | use crate::commands::Command; 7 | use crate::era::invalidate::{invalidate, EraInvalidateOptions}; 8 | use crate::version::*; 9 | 10 | //------------------------------------------ 11 | 12 | pub struct EraInvalidateCommand; 13 | 14 | impl EraInvalidateCommand { 15 | fn cli(&self) -> clap::Command { 16 | let cmd = clap::Command::new(self.name()) 17 | .next_display_order(None) 18 | .version(crate::tools_version!()) 19 | .disable_version_flag(true) 20 | .about("List blocks that may have changed since a given era") 21 | .arg( 22 | Arg::new("METADATA_SNAPSHOT") 23 | .help("Use the metadata snapshot rather than the current superblock") 24 | .long("metadata-snapshot"), 25 | ) 26 | // options 27 | .arg( 28 | Arg::new("OUTPUT") 29 | .help("Specify the output file rather than stdout") 30 | .short('o') 31 | .long("output") 32 | .value_name("FILE"), 33 | ) 34 | .arg( 35 | Arg::new("WRITTEN_SINCE") 36 | .help("Blocks written since the given era will be listed") 37 | .long("written-since") 38 | .required(true) 39 | .value_name("ERA") 40 | .value_parser(value_parser!(u32)), 41 | ) 42 | // arguments 43 | .arg( 44 | Arg::new("INPUT") 45 | .help("Specify the input device") 46 | .required(true) 47 | .index(1), 48 | ); 49 | engine_args(version_args(cmd)) 50 | } 51 | } 52 | 53 | impl<'a> Command<'a> for EraInvalidateCommand { 54 | fn name(&self) -> &'a str { 55 | "era_invalidate" 56 | } 57 | 58 | fn run(&self, args: &mut dyn Iterator) -> exitcode::ExitCode { 59 | let matches = self.cli().get_matches_from(args); 60 | display_version(&matches); 61 | 62 | let input_file = Path::new(matches.get_one::("INPUT").unwrap()); 63 | let output_file = matches.get_one::("OUTPUT").map(Path::new); 64 | 65 | // Create a temporary report just in case these checks 66 | // need to report anything. 67 | let report = std::sync::Arc::new(crate::report::mk_simple_report()); 68 | 69 | if let Err(e) = check_input_file(input_file).and_then(check_file_not_tiny) { 70 | return to_exit_code::<()>(&report, Err(e)); 71 | } 72 | 73 | let engine_opts = parse_engine_opts(ToolType::Era, &matches); 74 | if engine_opts.is_err() { 75 | return to_exit_code(&report, engine_opts); 76 | } 77 | 78 | let opts = EraInvalidateOptions { 79 | input: input_file, 80 | output: output_file, 81 | engine_opts: engine_opts.unwrap(), 82 | threshold: matches.get_one::("WRITTEN_SINCE").map_or(0, |v| *v), 83 | }; 84 | 85 | to_exit_code(&report, invalidate(&opts)) 86 | } 87 | } 88 | 89 | //------------------------------------------ 90 | -------------------------------------------------------------------------------- /src/commands/era_repair.rs: -------------------------------------------------------------------------------- 1 | extern crate clap; 2 | 3 | use clap::{Arg, ArgAction}; 4 | use std::path::Path; 5 | 6 | use crate::commands::engine::*; 7 | use crate::commands::utils::*; 8 | use crate::commands::Command; 9 | use crate::era::repair::{repair, EraRepairOptions}; 10 | use crate::report::*; 11 | use crate::version::*; 12 | 13 | pub struct EraRepairCommand; 14 | 15 | impl EraRepairCommand { 16 | fn cli(&self) -> clap::Command { 17 | let cmd = clap::Command::new(self.name()) 18 | .next_display_order(None) 19 | .version(crate::tools_version!()) 20 | .disable_version_flag(true) 21 | .about("Repair binary era metadata, and write it to a different device or file") 22 | .arg( 23 | Arg::new("QUIET") 24 | .help("Suppress output messages, return only exit code.") 25 | .short('q') 26 | .long("quiet") 27 | .action(ArgAction::SetTrue), 28 | ) 29 | // options 30 | .arg( 31 | Arg::new("INPUT") 32 | .help("Specify the input device") 33 | .short('i') 34 | .long("input") 35 | .value_name("FILE") 36 | .required(true), 37 | ) 38 | .arg( 39 | Arg::new("OUTPUT") 40 | .help("Specify the output device") 41 | .short('o') 42 | .long("output") 43 | .value_name("FILE") 44 | .required(true), 45 | ); 46 | verbose_args(engine_args(version_args(cmd))) 47 | } 48 | } 49 | 50 | impl<'a> Command<'a> for EraRepairCommand { 51 | fn name(&self) -> &'a str { 52 | "era_repair" 53 | } 54 | 55 | fn run(&self, args: &mut dyn Iterator) -> exitcode::ExitCode { 56 | let matches = self.cli().get_matches_from(args); 57 | display_version(&matches); 58 | 59 | let input_file = Path::new(matches.get_one::("INPUT").unwrap()); 60 | let output_file = Path::new(matches.get_one::("OUTPUT").unwrap()); 61 | 62 | let report = mk_report(matches.get_flag("QUIET")); 63 | let log_level = match parse_log_level(&matches) { 64 | Ok(level) => level, 65 | Err(e) => return to_exit_code::<()>(&report, Err(anyhow::Error::msg(e))), 66 | }; 67 | report.set_level(log_level); 68 | 69 | if let Err(e) = check_input_file(input_file) 70 | .and_then(check_file_not_tiny) 71 | .and_then(|_| check_output_file(output_file)) 72 | { 73 | return to_exit_code::<()>(&report, Err(e)); 74 | } 75 | 76 | let engine_opts = parse_engine_opts(ToolType::Era, &matches); 77 | if engine_opts.is_err() { 78 | return to_exit_code(&report, engine_opts); 79 | } 80 | 81 | let opts = EraRepairOptions { 82 | input: input_file, 83 | output: output_file, 84 | engine_opts: engine_opts.unwrap(), 85 | report: report.clone(), 86 | }; 87 | 88 | to_exit_code(&report, repair(opts)) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/commands/era_restore.rs: -------------------------------------------------------------------------------- 1 | extern crate clap; 2 | 3 | use clap::{Arg, ArgAction}; 4 | use std::path::Path; 5 | 6 | use crate::commands::engine::*; 7 | use crate::commands::utils::*; 8 | use crate::commands::Command; 9 | use crate::era::restore::{restore, EraRestoreOptions}; 10 | use crate::report::{parse_log_level, verbose_args}; 11 | use crate::version::*; 12 | 13 | pub struct EraRestoreCommand; 14 | 15 | impl EraRestoreCommand { 16 | fn cli(&self) -> clap::Command { 17 | let cmd = clap::Command::new(self.name()) 18 | .next_display_order(None) 19 | .version(crate::tools_version!()) 20 | .disable_version_flag(true) 21 | .about("Convert XML format metadata to binary.") 22 | // flags 23 | .arg( 24 | Arg::new("QUIET") 25 | .help("Suppress output messages, return only exit code.") 26 | .short('q') 27 | .long("quiet") 28 | .action(ArgAction::SetTrue), 29 | ) 30 | // options 31 | .arg( 32 | Arg::new("INPUT") 33 | .help("Specify the input xml") 34 | .short('i') 35 | .long("input") 36 | .value_name("FILE") 37 | .required(true), 38 | ) 39 | .arg( 40 | Arg::new("OUTPUT") 41 | .help("Specify the output device") 42 | .short('o') 43 | .long("output") 44 | .value_name("FILE") 45 | .required(true), 46 | ); 47 | verbose_args(engine_args(version_args(cmd))) 48 | } 49 | } 50 | 51 | impl<'a> Command<'a> for EraRestoreCommand { 52 | fn name(&self) -> &'a str { 53 | "era_restore" 54 | } 55 | 56 | fn run(&self, args: &mut dyn Iterator) -> exitcode::ExitCode { 57 | let matches = self.cli().get_matches_from(args); 58 | display_version(&matches); 59 | 60 | let input_file = Path::new(matches.get_one::("INPUT").unwrap()); 61 | let output_file = Path::new(matches.get_one::("OUTPUT").unwrap()); 62 | 63 | let report = mk_report(matches.get_flag("QUIET")); 64 | let log_level = match parse_log_level(&matches) { 65 | Ok(level) => level, 66 | Err(e) => return to_exit_code::<()>(&report, Err(anyhow::Error::msg(e))), 67 | }; 68 | report.set_level(log_level); 69 | 70 | if let Err(e) = check_input_file(input_file).and_then(|_| check_output_file(output_file)) { 71 | return to_exit_code::<()>(&report, Err(e)); 72 | } 73 | 74 | let engine_opts = parse_engine_opts(ToolType::Era, &matches); 75 | if engine_opts.is_err() { 76 | return to_exit_code(&report, engine_opts); 77 | } 78 | 79 | let opts = EraRestoreOptions { 80 | input: input_file, 81 | output: output_file, 82 | engine_opts: engine_opts.unwrap(), 83 | report: report.clone(), 84 | }; 85 | 86 | to_exit_code(&report, restore(opts)) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/commands/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod cache_check; 2 | pub mod cache_dump; 3 | pub mod cache_metadata_size; 4 | pub mod cache_repair; 5 | pub mod cache_restore; 6 | pub mod cache_writeback; 7 | pub mod engine; 8 | pub mod era_check; 9 | pub mod era_dump; 10 | pub mod era_invalidate; 11 | pub mod era_repair; 12 | pub mod era_restore; 13 | pub mod thin_check; 14 | pub mod thin_delta; 15 | pub mod thin_dump; 16 | pub mod thin_ls; 17 | pub mod thin_metadata_pack; 18 | pub mod thin_metadata_size; 19 | pub mod thin_metadata_unpack; 20 | pub mod thin_migrate; 21 | pub mod thin_repair; 22 | pub mod thin_restore; 23 | pub mod thin_rmap; 24 | pub mod thin_shrink; 25 | pub mod thin_trim; 26 | pub mod utils; 27 | 28 | #[cfg(feature = "devtools")] 29 | pub mod cache_generate_damage; 30 | #[cfg(feature = "devtools")] 31 | pub mod cache_generate_metadata; 32 | #[cfg(feature = "devtools")] 33 | pub mod era_generate_metadata; 34 | #[cfg(feature = "devtools")] 35 | pub mod thin_explore; 36 | #[cfg(feature = "devtools")] 37 | pub mod thin_generate_damage; 38 | #[cfg(feature = "devtools")] 39 | pub mod thin_generate_metadata; 40 | #[cfg(feature = "devtools")] 41 | pub mod thin_stat; 42 | 43 | pub trait Command<'a> { 44 | fn name(&self) -> &'a str; 45 | fn run(&self, args: &mut dyn Iterator) -> exitcode::ExitCode; 46 | } 47 | -------------------------------------------------------------------------------- /src/commands/thin_metadata_pack.rs: -------------------------------------------------------------------------------- 1 | extern crate clap; 2 | 3 | use clap::{Arg, ArgAction}; 4 | use std::path::Path; 5 | 6 | use crate::commands::utils::*; 7 | use crate::commands::Command; 8 | use crate::report::*; 9 | use crate::version::*; 10 | 11 | pub struct ThinMetadataPackCommand; 12 | 13 | impl ThinMetadataPackCommand { 14 | fn cli(&self) -> clap::Command { 15 | let cmd = clap::Command::new(self.name()) 16 | .next_display_order(None) 17 | .version(crate::tools_version!()) 18 | .disable_version_flag(true) 19 | .about("Produces a compressed file of thin metadata. Only packs metadata blocks that are actually used.") 20 | // flags 21 | .arg(Arg::new("FORCE") 22 | .help("Force overwrite the output file") 23 | .short('f') 24 | .long("force") 25 | .action(ArgAction::SetTrue)) 26 | // options 27 | .arg(Arg::new("INPUT") 28 | .help("Specify thinp metadata binary device/file") 29 | .required(true) 30 | .short('i') 31 | .long("input") 32 | .value_name("DEV")) 33 | .arg(Arg::new("OUTPUT") 34 | .help("Specify packed output file") 35 | .required(true) 36 | .short('o') 37 | .long("output") 38 | .value_name("FILE")); 39 | 40 | version_args(cmd) 41 | } 42 | } 43 | 44 | impl<'a> Command<'a> for ThinMetadataPackCommand { 45 | fn name(&self) -> &'a str { 46 | "thin_metadata_pack" 47 | } 48 | 49 | fn run(&self, args: &mut dyn Iterator) -> exitcode::ExitCode { 50 | let matches = self.cli().get_matches_from(args); 51 | display_version(&matches); 52 | 53 | let input_file = Path::new(matches.get_one::("INPUT").unwrap()); 54 | let output_file = Path::new(matches.get_one::("OUTPUT").unwrap()); 55 | 56 | let report = mk_simple_report(); 57 | 58 | if let Err(e) = check_input_file(input_file) 59 | .and_then(check_file_not_tiny) 60 | .and_then(check_not_xml) 61 | { 62 | return to_exit_code::<()>(&report, Err(e)); 63 | } 64 | 65 | if !matches.get_flag("FORCE") { 66 | if let Err(e) = check_overwrite_metadata(&report, output_file) { 67 | return to_exit_code::<()>(&report, Err(e)); 68 | } 69 | } 70 | 71 | let report = std::sync::Arc::new(report); 72 | to_exit_code( 73 | &report, 74 | crate::pack::toplevel::pack(input_file, output_file), 75 | ) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/commands/thin_metadata_unpack.rs: -------------------------------------------------------------------------------- 1 | extern crate clap; 2 | 3 | use clap::{Arg, ArgAction}; 4 | use std::path::Path; 5 | 6 | use crate::commands::engine::*; 7 | use crate::commands::utils::*; 8 | use crate::commands::Command; 9 | use crate::pack::toplevel::unpack; 10 | use crate::report::mk_simple_report; 11 | use crate::version::*; 12 | 13 | pub struct ThinMetadataUnpackCommand; 14 | 15 | impl ThinMetadataUnpackCommand { 16 | fn cli(&self) -> clap::Command { 17 | let cmd = clap::Command::new(self.name()) 18 | .next_display_order(None) 19 | .version(crate::tools_version!()) 20 | .disable_version_flag(true) 21 | .about("Unpack a compressed file of thin metadata.") 22 | // flags 23 | .arg( 24 | Arg::new("FORCE") 25 | .help("Force overwrite the output file") 26 | .short('f') 27 | .long("force") 28 | .action(ArgAction::SetTrue), 29 | ) 30 | // options 31 | .arg( 32 | Arg::new("INPUT") 33 | .help("Specify packed input file") 34 | .required(true) 35 | .short('i') 36 | .long("input") 37 | .value_name("FILE"), 38 | ) 39 | .arg( 40 | Arg::new("OUTPUT") 41 | .help("Specify thinp metadata binary device/file") 42 | .required(true) 43 | .short('o') 44 | .long("output") 45 | .value_name("DEV"), 46 | ); 47 | engine_args(version_args(cmd)) 48 | } 49 | } 50 | 51 | impl<'a> Command<'a> for ThinMetadataUnpackCommand { 52 | fn name(&self) -> &'a str { 53 | "thin_metadata_unpack" 54 | } 55 | 56 | fn run(&self, args: &mut dyn Iterator) -> exitcode::ExitCode { 57 | let matches = self.cli().get_matches_from(args); 58 | display_version(&matches); 59 | 60 | let input_file = Path::new(matches.get_one::("INPUT").unwrap()); 61 | let output_file = Path::new(matches.get_one::("OUTPUT").unwrap()); 62 | 63 | let report = mk_simple_report(); 64 | 65 | if let Err(e) = check_input_file(input_file) { 66 | return to_exit_code::<()>(&report, Err(e)); 67 | } 68 | 69 | if !matches.get_flag("FORCE") { 70 | if let Err(e) = check_overwrite_metadata(&report, output_file) { 71 | return to_exit_code::<()>(&report, Err(e)); 72 | } 73 | } 74 | 75 | let report = std::sync::Arc::new(report); 76 | to_exit_code(&report, unpack(input_file, output_file)) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/commands/thin_rmap.rs: -------------------------------------------------------------------------------- 1 | extern crate clap; 2 | 3 | use clap::{value_parser, Arg}; 4 | use std::ops::Range; 5 | use std::path::Path; 6 | 7 | use crate::commands::engine::*; 8 | use crate::commands::utils::*; 9 | use crate::commands::Command; 10 | use crate::thin::rmap::*; 11 | use crate::version::*; 12 | 13 | //------------------------------------------ 14 | 15 | pub struct ThinRmapCommand; 16 | 17 | impl ThinRmapCommand { 18 | fn cli(&self) -> clap::Command { 19 | let cmd = clap::Command::new(self.name()) 20 | .next_display_order(None) 21 | .version(crate::tools_version!()) 22 | .disable_version_flag(true) 23 | .about("Output reverse map of a thin provisioned region of blocks") 24 | // options 25 | .arg( 26 | // FIXME: clap doesn't support placing the index argument 27 | // after the multiple arguments, e.g., 28 | // thin_rmap --region 0..1 --region 2..3 tmeta.bin 29 | Arg::new("REGION") 30 | .help("Specify range of blocks on the data device") 31 | .long("region") 32 | .action(clap::ArgAction::Append) 33 | .required(true) 34 | .value_name("BLOCK_RANGE") 35 | .value_parser(value_parser!(RangeU64)), 36 | ) 37 | // arguments 38 | .arg( 39 | Arg::new("INPUT") 40 | .help("Specify the input device") 41 | .required(true) 42 | .index(1), 43 | ); 44 | engine_args(version_args(cmd)) 45 | } 46 | } 47 | 48 | impl<'a> Command<'a> for ThinRmapCommand { 49 | fn name(&self) -> &'a str { 50 | "thin_rmap" 51 | } 52 | 53 | fn run(&self, args: &mut dyn Iterator) -> exitcode::ExitCode { 54 | let matches = self.cli().get_matches_from(args); 55 | display_version(&matches); 56 | let input_file = Path::new(matches.get_one::("INPUT").unwrap()); 57 | 58 | let report = mk_report(false); 59 | 60 | if let Err(e) = check_input_file(input_file).and_then(check_file_not_tiny) { 61 | return to_exit_code::<()>(&report, Err(e)); 62 | } 63 | 64 | // FIXME: get rid of the intermediate RangeU64 struct 65 | let regions: Vec> = matches 66 | .get_many::("REGION") 67 | .unwrap() 68 | .map(|v| Range:: { 69 | start: v.start, 70 | end: v.end, 71 | }) 72 | .collect(); 73 | 74 | let engine_opts = parse_engine_opts(ToolType::Thin, &matches); 75 | if engine_opts.is_err() { 76 | return to_exit_code(&report, engine_opts); 77 | } 78 | 79 | let opts = ThinRmapOptions { 80 | input: input_file, 81 | engine_opts: engine_opts.unwrap(), 82 | regions, 83 | report: report.clone(), 84 | }; 85 | 86 | to_exit_code(&report, rmap(opts)) 87 | } 88 | } 89 | 90 | //------------------------------------------ 91 | -------------------------------------------------------------------------------- /src/commands/thin_stat.rs: -------------------------------------------------------------------------------- 1 | use clap::Arg; 2 | use std::path::Path; 3 | 4 | use crate::commands::engine::*; 5 | use crate::commands::utils::*; 6 | use crate::report::mk_simple_report; 7 | use crate::thin::stat::*; 8 | use crate::version::*; 9 | 10 | //------------------------------------------ 11 | use crate::commands::Command; 12 | 13 | pub struct ThinStatCommand; 14 | 15 | impl ThinStatCommand { 16 | fn cli(&self) -> clap::Command { 17 | let cmd = clap::Command::new(self.name()) 18 | .next_display_order(None) 19 | .version(crate::tools_version!()) 20 | .disable_version_flag(true) 21 | .about("Tool to show metadata statistics") 22 | .arg( 23 | Arg::new("OP") 24 | .help("Choose the target to stat") 25 | .long("op") 26 | .default_value("data_blocks"), 27 | ) 28 | .arg( 29 | Arg::new("INPUT") 30 | .help("Specify the input device") 31 | .required(true) 32 | .index(1), 33 | ); 34 | 35 | engine_args(version_args(cmd)) 36 | } 37 | } 38 | 39 | impl<'a> Command<'a> for ThinStatCommand { 40 | fn name(&self) -> &'a str { 41 | "thin_stat" 42 | } 43 | 44 | fn run(&self, args: &mut dyn Iterator) -> exitcode::ExitCode { 45 | let matches = self.cli().get_matches_from(args); 46 | display_version(&matches); 47 | let report = std::sync::Arc::new(mk_simple_report()); 48 | 49 | let engine_opts = parse_engine_opts(ToolType::Thin, &matches); 50 | if engine_opts.is_err() { 51 | return to_exit_code(&report, engine_opts); 52 | } 53 | 54 | let opts = ThinStatOpts { 55 | engine_opts: engine_opts.unwrap(), 56 | input: Path::new(matches.get_one::("INPUT").unwrap()), 57 | op: match matches.get_one::("OP").unwrap().as_ref() { 58 | "data_blocks" => StatOp::DataBlockRefCounts, 59 | "metadata_blocks" => StatOp::MetadataBlockRefCounts, 60 | "data_run_len" => StatOp::DataRunLength, 61 | _ => return exitcode::USAGE, 62 | }, 63 | }; 64 | 65 | to_exit_code(&report, stat(opts)) 66 | } 67 | } 68 | 69 | //------------------------------------------ 70 | -------------------------------------------------------------------------------- /src/commands/utils/range_parsing_tests.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | //------------------------------------------ 4 | 5 | #[test] 6 | fn test_normal() -> anyhow::Result<()> { 7 | let range = "0..18446744073709551615".parse::()?; 8 | assert_eq!(range.start, 0); 9 | assert_eq!(range.end, u64::MAX); 10 | Ok(()) 11 | } 12 | 13 | #[test] 14 | fn test_no_dots() -> anyhow::Result<()> { 15 | assert!("0".parse::().is_err()); 16 | assert!("0.10".parse::().is_err()); 17 | Ok(()) 18 | } 19 | 20 | #[test] 21 | fn test_dots_only() -> anyhow::Result<()> { 22 | assert!("..".parse::().is_err()); 23 | Ok(()) 24 | } 25 | 26 | #[test] 27 | fn test_extra_characters() -> anyhow::Result<()> { 28 | assert!("0..10,".parse::().is_err()); 29 | assert!("0..10..".parse::().is_err()); 30 | Ok(()) 31 | } 32 | 33 | #[test] 34 | fn test_invalid_begin() -> anyhow::Result<()> { 35 | assert!("foo".parse::().is_err()); 36 | assert!("foo..10".parse::().is_err()); 37 | Ok(()) 38 | } 39 | 40 | #[test] 41 | fn test_invalid_end() -> anyhow::Result<()> { 42 | assert!("0..bar".parse::().is_err()); 43 | Ok(()) 44 | } 45 | 46 | #[test] 47 | fn test_negative_begin() -> anyhow::Result<()> { 48 | assert!("-1..0".parse::().is_err()); 49 | Ok(()) 50 | } 51 | 52 | #[test] 53 | fn test_negative_end() -> anyhow::Result<()> { 54 | assert!("0..-1".parse::().is_err()); 55 | Ok(()) 56 | } 57 | 58 | #[test] 59 | fn test_end_not_greater_than_begin() -> anyhow::Result<()> { 60 | assert!("0..0".parse::().is_err()); 61 | assert!("18446744073709551615..0".parse::().is_err()); 62 | Ok(()) 63 | } 64 | 65 | //------------------------------------------ 66 | -------------------------------------------------------------------------------- /src/copier/base.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::sync::Arc; 3 | 4 | //------------------------------------- 5 | 6 | pub type Block = u64; 7 | 8 | #[derive(Copy, Clone, Debug, PartialEq, Eq)] 9 | pub struct CopyOp { 10 | pub src: Block, 11 | pub dst: Block, 12 | } 13 | 14 | #[derive(Debug)] 15 | pub struct CopyStats { 16 | pub nr_blocks: Block, 17 | pub nr_copied: Block, 18 | pub read_errors: Vec, 19 | pub write_errors: Vec, 20 | } 21 | 22 | impl CopyStats { 23 | pub fn new(nr_blocks: u64) -> Self { 24 | Self { 25 | nr_blocks, 26 | nr_copied: 0, 27 | read_errors: Vec::new(), 28 | write_errors: Vec::new(), 29 | } 30 | } 31 | } 32 | 33 | pub trait CopyProgress { 34 | /// This method is invoked during a copy batch for updating the progress bar 35 | /// more frequently, providing a smoother display of progress. 36 | fn update(&self, stats: &CopyStats); 37 | 38 | /// This method is called with the resulting stats of a copy batch while 39 | /// the copy batch is complete. The implementors should include these values 40 | /// into its internal statistics for further accumulative updates. 41 | fn inc_stats(&self, stats: &CopyStats); 42 | } 43 | 44 | // The constructor for the instance should be passed the src and dst 45 | // paths and the block size. 46 | pub trait Copier { 47 | /// This copies the blocks in roughly the order given, so sort ops before 48 | /// submitting. eg, cache writeback would sort by dst since that's 49 | /// likely a spindle device where ordering really matters. 50 | fn copy( 51 | &mut self, 52 | ops: &[CopyOp], 53 | progress: Arc, 54 | ) -> Result; 55 | } 56 | 57 | //------------------------------------- 58 | -------------------------------------------------------------------------------- /src/copier/batcher.rs: -------------------------------------------------------------------------------- 1 | use anyhow::anyhow; 2 | use std::sync::mpsc::SyncSender; 3 | use std::vec::Vec; 4 | 5 | use crate::copier::CopyOp; 6 | 7 | //------------------------------------------ 8 | 9 | // This builds vectors of copy operations and posts them into a channel. 10 | // The larger the batch size the more chance we have of finding and 11 | // aggregating adjacent copies. 12 | pub struct CopyOpBatcher { 13 | batch_size: usize, 14 | ops: Vec, 15 | tx: SyncSender>, 16 | } 17 | 18 | impl CopyOpBatcher { 19 | pub fn new(batch_size: usize, tx: SyncSender>) -> Self { 20 | Self { 21 | batch_size, 22 | ops: Vec::with_capacity(batch_size), 23 | tx, 24 | } 25 | } 26 | 27 | /// Append a CopyOp to the current batch. 28 | pub fn push(&mut self, op: CopyOp) -> anyhow::Result<()> { 29 | self.ops.push(op); 30 | if self.ops.len() >= self.batch_size { 31 | self.send_ops()?; 32 | } 33 | 34 | Ok(()) 35 | } 36 | 37 | /// Send the current batch and return the dirty ablocks bitmap. 38 | pub fn complete(mut self) -> anyhow::Result<()> { 39 | self.send_ops()?; 40 | Ok(()) 41 | } 42 | 43 | /// Send the current batch. 44 | fn send_ops(&mut self) -> anyhow::Result<()> { 45 | let mut ops = std::mem::take(&mut self.ops); 46 | 47 | // We sort by the destination since this is likely to be a spindle 48 | // and keeping the io in sequential order will help performance. 49 | ops.sort_by(|lhs, rhs| lhs.dst.cmp(&rhs.dst)); 50 | self.tx 51 | .send(ops) 52 | .map_err(|_| anyhow!("unable to send copy op array")) 53 | } 54 | } 55 | 56 | //------------------------------------------ 57 | -------------------------------------------------------------------------------- /src/copier/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod base; 2 | pub mod batcher; 3 | pub mod report; 4 | pub mod rescue_copier; 5 | pub mod sync_copier; 6 | pub mod wrapper; 7 | 8 | pub use crate::copier::base::*; 9 | pub use crate::copier::report::*; 10 | pub use crate::copier::rescue_copier::RescueCopier; 11 | pub use crate::copier::sync_copier::SyncCopier; 12 | 13 | #[cfg(any(test, feature = "devtools"))] 14 | pub mod test_utils; 15 | -------------------------------------------------------------------------------- /src/copier/report.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{Arc, Mutex}; 2 | 3 | use crate::copier::{CopyProgress, CopyStats}; 4 | use crate::report::Report; 5 | 6 | //------------------------------------- 7 | 8 | #[derive(Default)] 9 | pub struct IgnoreProgress {} 10 | 11 | impl CopyProgress for IgnoreProgress { 12 | fn update(&self, _: &CopyStats) {} 13 | fn inc_stats(&self, _: &CopyStats) {} 14 | } 15 | 16 | //----------------------------------------- 17 | 18 | struct AccumulatedStats { 19 | /// Number of blocks that need to be copied 20 | nr_blocks: u64, 21 | 22 | /// Number of blocks that were successfully copied 23 | nr_copied: u64, 24 | 25 | /// Number of read errors 26 | nr_read_errors: u64, 27 | 28 | /// Number of write errors 29 | nr_write_errors: u64, 30 | } 31 | 32 | /// Struct that updates the progress bar in the reporter as the stats are 33 | /// updated by the copier threads. 34 | pub struct ProgressReporter { 35 | report: Arc, 36 | inner: Mutex, 37 | } 38 | 39 | impl ProgressReporter { 40 | /// nr_blocks: total number of blocks to be copied 41 | pub fn new(report: Arc, nr_blocks: u64) -> Self { 42 | Self { 43 | report, 44 | inner: Mutex::new(AccumulatedStats { 45 | nr_blocks, 46 | nr_copied: 0, 47 | nr_read_errors: 0, 48 | nr_write_errors: 0, 49 | }), 50 | } 51 | } 52 | 53 | fn update_error_logs(&self, nr_read_errors: u64, nr_write_errors: u64) { 54 | if nr_read_errors > 0 || nr_write_errors > 0 { 55 | self.report.set_sub_title(&format!( 56 | "read errors {}, write errors {}", 57 | nr_read_errors, nr_write_errors 58 | )); 59 | } 60 | } 61 | 62 | fn update_progress(&self, nr_copied: u64, nr_blocks: u64) { 63 | let percent = (nr_copied * 100).checked_div(nr_blocks).unwrap_or(100); 64 | self.report.progress(percent as u8); 65 | } 66 | } 67 | 68 | impl CopyProgress for ProgressReporter { 69 | // This doesn't update the internal stats in struct, that is left until the copy 70 | // batch is complete. 71 | fn update(&self, stats: &CopyStats) { 72 | let inner = self.inner.lock().unwrap(); 73 | 74 | self.update_error_logs( 75 | inner.nr_read_errors + stats.read_errors.len() as u64, 76 | inner.nr_write_errors + stats.write_errors.len() as u64, 77 | ); 78 | self.update_progress(inner.nr_copied + stats.nr_copied, inner.nr_blocks); 79 | } 80 | 81 | // Adds the resulting stats of a copy batch into the internal base, 82 | // then updates the displayed progress. 83 | fn inc_stats(&self, stats: &CopyStats) { 84 | let mut inner = self.inner.lock().unwrap(); 85 | inner.nr_copied += stats.nr_copied; 86 | inner.nr_read_errors += stats.read_errors.len() as u64; 87 | inner.nr_write_errors += stats.write_errors.len() as u64; 88 | 89 | self.update_error_logs(inner.nr_read_errors, inner.nr_write_errors); 90 | self.update_progress(inner.nr_copied, inner.nr_blocks); 91 | } 92 | } 93 | 94 | //----------------------------------------- 95 | -------------------------------------------------------------------------------- /src/copier/test_utils.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::os::unix::fs::FileExt; 3 | 4 | use crate::io_engine::base::PAGE_SIZE; 5 | use crate::io_engine::buffer::Buffer; 6 | use crate::random::Generator; 7 | 8 | //------------------------------------------ 9 | 10 | pub trait BlockVisitor { 11 | fn visit(&mut self, blocknr: u64) -> Result<()>; 12 | } 13 | 14 | pub fn visit_blocks(nr_blocks: u64, v: &mut dyn BlockVisitor) -> Result<()> { 15 | for b in 0..nr_blocks { 16 | v.visit(b)?; 17 | } 18 | Ok(()) 19 | } 20 | 21 | //------------------------------------------ 22 | 23 | pub struct Stamper { 24 | dev: T, 25 | block_size: usize, // bytes 26 | seed: u64, 27 | buf: Buffer, 28 | offset: u64, // bytes 29 | } 30 | 31 | impl Stamper { 32 | pub fn new(dev: T, seed: u64, block_size: usize) -> Self { 33 | let buf = Buffer::new(block_size, PAGE_SIZE); 34 | Self { 35 | dev, 36 | block_size, 37 | seed, 38 | buf, 39 | offset: 0, 40 | } 41 | } 42 | 43 | pub fn offset(mut self, offset: u64) -> Self { 44 | self.offset = offset; 45 | self 46 | } 47 | } 48 | 49 | impl BlockVisitor for Stamper { 50 | fn visit(&mut self, blocknr: u64) -> Result<()> { 51 | let mut gen = Generator::new(); 52 | gen.fill_buffer(self.seed ^ blocknr, self.buf.get_data())?; 53 | 54 | let offset = blocknr * self.block_size as u64 + self.offset; 55 | self.dev.write_all_at(self.buf.get_data(), offset)?; 56 | Ok(()) 57 | } 58 | } 59 | 60 | //------------------------------------------ 61 | -------------------------------------------------------------------------------- /src/copier/wrapper.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{anyhow, Result}; 2 | use std::sync::mpsc; 3 | use std::sync::Arc; 4 | use std::thread::{self, JoinHandle}; 5 | 6 | use crate::copier::*; 7 | 8 | //--------------------------------------- 9 | 10 | /* 11 | * A simple wrapper that runs Copier in a worker thread. 12 | * The copy process terminates on any read/write error. 13 | */ 14 | pub struct ThreadedCopier { 15 | copier: T, 16 | } 17 | 18 | impl ThreadedCopier { 19 | pub fn new(copier: T) -> ThreadedCopier { 20 | ThreadedCopier { copier } 21 | } 22 | 23 | pub fn run( 24 | self, 25 | rx: mpsc::Receiver>, 26 | progress: Arc, 27 | ) -> JoinHandle> { 28 | thread::spawn(move || Self::run_(rx, self.copier, progress)) 29 | } 30 | 31 | fn run_( 32 | rx: mpsc::Receiver>, 33 | mut copier: T, 34 | progress: Arc, 35 | ) -> Result<()> { 36 | while let Ok(ops) = rx.recv() { 37 | let stats = copier 38 | .copy(&ops, progress.clone()) 39 | .map_err(|e| anyhow!("copy failed: {}", e))?; 40 | 41 | if !stats.read_errors.is_empty() { 42 | return Err(anyhow!("read error in block {}", stats.read_errors[0].src)); 43 | } 44 | 45 | if !stats.write_errors.is_empty() { 46 | return Err(anyhow!( 47 | "write error in block {}", 48 | stats.write_errors[0].dst 49 | )); 50 | } 51 | } 52 | 53 | Ok(()) 54 | } 55 | } 56 | 57 | //--------------------------------------- 58 | -------------------------------------------------------------------------------- /src/devtools/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "devtools")] 2 | pub mod damage_generator; 3 | -------------------------------------------------------------------------------- /src/dump_utils.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Context; 2 | use std::sync::Arc; 3 | 4 | use crate::checksum; 5 | use crate::io_engine::IoEngine; 6 | use crate::pdata::array::{self, unpack_array_block, ArrayBlock}; 7 | use crate::pdata::unpack; 8 | 9 | //------------------------------------------ 10 | 11 | // A wrapper for callers to identify the error type 12 | #[derive(Debug)] 13 | pub struct OutputError; 14 | 15 | impl std::fmt::Display for OutputError { 16 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 17 | write!(f, "output error") 18 | } 19 | } 20 | 21 | #[inline] 22 | pub fn output_context(result: anyhow::Result) -> anyhow::Result { 23 | result.context(OutputError) 24 | } 25 | 26 | //------------------------------------------ 27 | 28 | // A slight variation of array_walker::ArrayVisitor returns anyhow::Result 29 | pub trait ArrayVisitor { 30 | fn visit(&self, index: u64, b: ArrayBlock) -> anyhow::Result<()>; 31 | } 32 | 33 | //------------------------------------------ 34 | 35 | pub fn walk_array_blocks( 36 | engine: Arc, 37 | ablocks: I, 38 | visitor: &dyn ArrayVisitor, 39 | ) -> anyhow::Result<()> 40 | where 41 | I: IntoIterator, u64))>, 42 | V: unpack::Unpack, 43 | { 44 | use crate::pdata::array::array_block_err; 45 | 46 | for (index, (mut path, blocknr)) in ablocks.into_iter() { 47 | path.push(blocknr); 48 | 49 | let b = engine 50 | .read(blocknr) 51 | .map_err(|_| array::io_err(&path, blocknr).index_context(index))?; 52 | 53 | let bt = checksum::metadata_block_type(b.get_data()); 54 | if bt != checksum::BT::ARRAY { 55 | let e = array_block_err( 56 | &path, 57 | &format!("checksum failed for array block {}, {:?}", b.loc, bt), 58 | ) 59 | .index_context(index); 60 | return Err(e.into()); 61 | } 62 | 63 | let ablock = unpack_array_block::(&path, b.get_data())?; 64 | visitor.visit(index, ablock)?; 65 | } 66 | 67 | Ok(()) 68 | } 69 | 70 | //------------------------------------------ 71 | -------------------------------------------------------------------------------- /src/era/ir.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | 3 | //------------------------------------------ 4 | 5 | #[derive(Clone)] 6 | pub struct Superblock { 7 | pub uuid: String, 8 | pub block_size: u32, 9 | pub nr_blocks: u32, 10 | pub current_era: u32, 11 | } 12 | 13 | #[derive(Clone)] 14 | pub struct Writeset { 15 | pub era: u32, 16 | pub nr_bits: u32, 17 | } 18 | 19 | #[derive(Clone)] 20 | pub struct MarkedBlocks { 21 | pub begin: u32, 22 | pub len: u32, 23 | } 24 | 25 | #[derive(Clone)] 26 | pub struct Era { 27 | pub block: u32, 28 | pub era: u32, 29 | } 30 | 31 | //------------------------------------------ 32 | 33 | #[derive(Clone)] 34 | pub enum Visit { 35 | Continue, 36 | Stop, 37 | } 38 | 39 | pub trait MetadataVisitor { 40 | fn superblock_b(&mut self, sb: &Superblock) -> Result; 41 | fn superblock_e(&mut self) -> Result; 42 | 43 | fn writeset_b(&mut self, ws: &Writeset) -> Result; 44 | fn writeset_e(&mut self) -> Result; 45 | fn writeset_blocks(&mut self, blocks: &MarkedBlocks) -> Result; 46 | 47 | fn era_b(&mut self) -> Result; 48 | fn era_e(&mut self) -> Result; 49 | fn era(&mut self, era: &Era) -> Result; 50 | 51 | fn eof(&mut self) -> Result; 52 | } 53 | 54 | //------------------------------------------ 55 | -------------------------------------------------------------------------------- /src/era/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod check; 2 | pub mod dump; 3 | pub mod invalidate; 4 | pub mod ir; 5 | pub mod repair; 6 | pub mod restore; 7 | pub mod superblock; 8 | pub mod writeset; 9 | pub mod xml; 10 | 11 | #[cfg(feature = "devtools")] 12 | pub mod metadata_generator; 13 | -------------------------------------------------------------------------------- /src/era/repair.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::path::Path; 3 | use std::sync::Arc; 4 | 5 | use crate::commands::engine::*; 6 | use crate::era::dump::*; 7 | use crate::era::restore::*; 8 | use crate::era::superblock::*; 9 | use crate::io_engine::*; 10 | use crate::pdata::space_map::metadata::*; 11 | use crate::report::*; 12 | use crate::write_batcher::*; 13 | 14 | //------------------------------------------ 15 | 16 | pub struct EraRepairOptions<'a> { 17 | pub input: &'a Path, 18 | pub output: &'a Path, 19 | pub engine_opts: EngineOptions, 20 | pub report: Arc, 21 | } 22 | 23 | struct Context { 24 | _report: Arc, 25 | engine_in: Arc, 26 | engine_out: Arc, 27 | } 28 | 29 | fn new_context(opts: &EraRepairOptions) -> Result { 30 | let engine_in = EngineBuilder::new(opts.input, &opts.engine_opts).build()?; 31 | let engine_out = EngineBuilder::new(opts.output, &opts.engine_opts) 32 | .write(true) 33 | .build()?; 34 | 35 | Ok(Context { 36 | _report: opts.report.clone(), 37 | engine_in, 38 | engine_out, 39 | }) 40 | } 41 | 42 | //------------------------------------------ 43 | 44 | pub fn repair(opts: EraRepairOptions) -> Result<()> { 45 | let ctx = new_context(&opts)?; 46 | 47 | let sb = read_superblock(ctx.engine_in.as_ref(), SUPERBLOCK_LOCATION)?; 48 | 49 | let sm = core_metadata_sm(ctx.engine_out.get_nr_blocks(), u32::MAX); 50 | let batch_size = ctx.engine_out.get_batch_size(); 51 | let mut w = WriteBatcher::new(ctx.engine_out, sm.clone(), batch_size); 52 | let mut restorer = Restorer::new(&mut w); 53 | 54 | dump_metadata(ctx.engine_in, &mut restorer, &sb, true) 55 | } 56 | 57 | //------------------------------------------ 58 | -------------------------------------------------------------------------------- /src/era/writeset.rs: -------------------------------------------------------------------------------- 1 | use byteorder::{LittleEndian, WriteBytesExt}; 2 | use nom::{number::complete::*, IResult}; 3 | use std::io; 4 | 5 | use crate::pdata::unpack::*; 6 | 7 | //------------------------------------------ 8 | 9 | #[derive(Clone, Copy, Debug)] 10 | pub struct Writeset { 11 | pub nr_bits: u32, 12 | pub root: u64, 13 | } 14 | 15 | impl Unpack for Writeset { 16 | fn disk_size() -> u32 { 17 | 12 18 | } 19 | 20 | fn unpack(i: &[u8]) -> IResult<&[u8], Writeset> { 21 | let (i, nr_bits) = le_u32(i)?; 22 | let (i, root) = le_u64(i)?; 23 | Ok((i, Writeset { nr_bits, root })) 24 | } 25 | } 26 | 27 | impl Pack for Writeset { 28 | fn pack(&self, w: &mut W) -> io::Result<()> { 29 | w.write_u32::(self.nr_bits)?; 30 | w.write_u64::(self.root) 31 | } 32 | } 33 | 34 | //------------------------------------------ 35 | -------------------------------------------------------------------------------- /src/file_utils.rs: -------------------------------------------------------------------------------- 1 | use std::fs::{File, OpenOptions}; 2 | use std::io; 3 | use std::io::{Seek, Write}; 4 | use std::os::unix::ffi::OsStrExt; 5 | use std::os::unix::io::AsRawFd; 6 | use std::path::Path; 7 | 8 | use crate::ioctl::{self, *}; 9 | 10 | //--------------------------------------- 11 | 12 | fn test_bit(mode: u32, flag: u32) -> bool { 13 | (mode & libc::S_IFMT) == flag 14 | } 15 | 16 | fn is_file_or_blk_(info: &libc::stat64) -> bool { 17 | test_bit(info.st_mode, libc::S_IFBLK) || test_bit(info.st_mode, libc::S_IFREG) 18 | } 19 | 20 | // wrapper of libc::stat64 21 | fn libc_stat64(path: &Path) -> io::Result { 22 | let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()) 23 | .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e.to_string()))?; 24 | 25 | unsafe { 26 | let mut st: libc::stat64 = std::mem::zeroed(); 27 | let r = libc::stat64(c_path.as_ptr(), &mut st); 28 | if r == 0 { 29 | Ok(st) 30 | } else { 31 | Err(io::Error::last_os_error()) 32 | } 33 | } 34 | } 35 | 36 | pub fn is_file_or_blk(path: &Path) -> io::Result { 37 | libc_stat64(path).map(|info| is_file_or_blk_(&info)) 38 | } 39 | 40 | pub fn is_file(path: &Path) -> io::Result { 41 | libc_stat64(path).map(|info| test_bit(info.st_mode, libc::S_IFREG)) 42 | } 43 | 44 | //--------------------------------------- 45 | 46 | const BLKGETSIZE64: ioctl::RequestType = crate::request_code_read!(0x12, 114, usize); 47 | 48 | pub fn fail(msg: &str) -> io::Result { 49 | let e = io::Error::new(io::ErrorKind::Other, msg); 50 | Err(e) 51 | } 52 | 53 | pub fn device_size(fd: libc::c_int) -> io::Result { 54 | let mut cap = 0u64; 55 | unsafe { 56 | if libc::ioctl(fd, BLKGETSIZE64, &mut cap) == 0 { 57 | Ok(cap) 58 | } else { 59 | Err(io::Error::last_os_error()) 60 | } 61 | } 62 | } 63 | 64 | fn get_device_size>(path: P) -> io::Result { 65 | let file = File::open(path.as_ref())?; 66 | device_size(file.as_raw_fd()) 67 | } 68 | 69 | pub fn file_size>(path: P) -> io::Result { 70 | libc_stat64(path.as_ref()).and_then(|info| { 71 | if test_bit(info.st_mode, libc::S_IFREG) { 72 | Ok(info.st_size as u64) 73 | } else if test_bit(info.st_mode, libc::S_IFBLK) { 74 | get_device_size(path) 75 | } else { 76 | fail("Not a block device or regular file") 77 | } 78 | }) 79 | } 80 | 81 | //--------------------------------------- 82 | 83 | fn set_size(w: &mut W, nr_bytes: u64) -> io::Result<()> { 84 | let zeroes: Vec = vec![0; 1]; 85 | 86 | if nr_bytes > 0 { 87 | w.seek(io::SeekFrom::Start(nr_bytes - 1))?; 88 | w.write_all(&zeroes)?; 89 | } 90 | 91 | Ok(()) 92 | } 93 | 94 | pub fn create_sized_file(path: &Path, nr_bytes: u64) -> io::Result { 95 | let mut file = OpenOptions::new() 96 | .read(false) 97 | .write(true) 98 | .create(true) 99 | .truncate(true) 100 | .open(path)?; 101 | set_size(&mut file, nr_bytes)?; 102 | Ok(file) 103 | } 104 | 105 | //--------------------------------------- 106 | -------------------------------------------------------------------------------- /src/grid_layout.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::io::Write; 3 | use std::string::String; 4 | use std::vec::Vec; 5 | 6 | pub struct GridLayout { 7 | grid: Vec>, 8 | current: Vec, 9 | max_columns: usize, 10 | } 11 | 12 | impl GridLayout { 13 | pub fn new() -> GridLayout { 14 | GridLayout { 15 | grid: Vec::new(), 16 | current: Vec::new(), 17 | max_columns: 0, 18 | } 19 | } 20 | 21 | pub fn new_with_size(rows: usize, columns: usize) -> GridLayout { 22 | GridLayout { 23 | grid: Vec::with_capacity(rows), 24 | current: Vec::with_capacity(columns), 25 | max_columns: columns, 26 | } 27 | } 28 | 29 | pub fn field(&mut self, s: String) { 30 | self.current.push(s) 31 | } 32 | 33 | pub fn new_row(&mut self) { 34 | if self.current.len() > self.max_columns { 35 | self.max_columns = self.current.len(); 36 | } 37 | let mut last = Vec::with_capacity(self.max_columns); 38 | std::mem::swap(&mut self.current, &mut last); 39 | self.grid.push(last); 40 | } 41 | 42 | fn calc_field_widths(&self) -> Result> { 43 | let mut widths = vec![0; self.max_columns]; 44 | for row in self.grid.iter() { 45 | for (col, width) in row.iter().zip(widths.iter_mut()) { 46 | *width = std::cmp::max(*width, col.len()); 47 | } 48 | } 49 | Ok(widths) 50 | } 51 | 52 | // right align the string 53 | fn push_justified(buf: &mut String, s: &str, width: usize) { 54 | for _ in 0..width - s.len() { 55 | buf.push(' '); 56 | } 57 | buf.push_str(s); 58 | buf.push(' '); 59 | } 60 | 61 | pub fn render(&self, w: &mut dyn Write) -> Result<()> { 62 | let widths = self.calc_field_widths()?; 63 | 64 | for row in self.grid.iter() { 65 | let mut line = String::new(); 66 | for (col, width) in row.iter().zip(widths.iter()) { 67 | Self::push_justified(&mut line, col.as_str(), *width); 68 | } 69 | line.push('\n'); 70 | w.write_all(line.as_bytes())?; 71 | } 72 | 73 | Ok(()) 74 | } 75 | } 76 | 77 | impl Default for GridLayout { 78 | fn default() -> Self { 79 | Self::new() 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/io_engine/buffer.rs: -------------------------------------------------------------------------------- 1 | use std::alloc::{alloc, dealloc, Layout}; 2 | 3 | // Because we use O_DIRECT we need to use page aligned blocks. Buffer 4 | // manages allocation of this aligned memory. 5 | pub struct Buffer { 6 | size: usize, 7 | align: usize, 8 | data: *mut u8, 9 | } 10 | 11 | impl Buffer { 12 | pub fn new(size: usize, align: usize) -> Self { 13 | let layout = Layout::from_size_align(size, align).unwrap(); 14 | let ptr = unsafe { alloc(layout) }; 15 | assert!(!ptr.is_null(), "out of memory"); 16 | 17 | Self { 18 | size, 19 | align, 20 | data: ptr, 21 | } 22 | } 23 | 24 | pub fn get_data<'a>(&self) -> &'a mut [u8] { 25 | unsafe { std::slice::from_raw_parts_mut::<'a>(self.data, self.size) } 26 | } 27 | } 28 | 29 | impl Drop for Buffer { 30 | fn drop(&mut self) { 31 | let layout = Layout::from_size_align(self.size, self.align).unwrap(); 32 | unsafe { 33 | dealloc(self.data, layout); 34 | } 35 | } 36 | } 37 | 38 | unsafe impl Send for Buffer {} 39 | unsafe impl Sync for Buffer {} 40 | 41 | //------------------------------------------ 42 | -------------------------------------------------------------------------------- /src/io_engine/core.rs: -------------------------------------------------------------------------------- 1 | use std::alloc::{alloc, dealloc, Layout}; 2 | use std::io; 3 | use std::vec::*; 4 | 5 | use crate::io_engine::{Block, IoEngine, BLOCK_SIZE}; 6 | 7 | //------------------------------------------ 8 | 9 | const ALIGN: usize = 4096; 10 | 11 | pub struct CoreIoEngine { 12 | nr_blocks: u64, 13 | data: *mut u8, 14 | } 15 | 16 | impl CoreIoEngine { 17 | pub fn new(nr_blocks: u64) -> CoreIoEngine { 18 | assert!(nr_blocks <= usize::MAX as u64); 19 | let capacity = BLOCK_SIZE * nr_blocks as usize; 20 | let layout = Layout::from_size_align(capacity, ALIGN).unwrap(); 21 | let ptr = unsafe { alloc(layout) }; 22 | assert!(!ptr.is_null(), "out of memory"); 23 | CoreIoEngine { 24 | nr_blocks, 25 | data: ptr, 26 | } 27 | } 28 | } 29 | 30 | impl Drop for CoreIoEngine { 31 | fn drop(&mut self) { 32 | let capacity = BLOCK_SIZE * self.nr_blocks as usize; 33 | let layout = Layout::from_size_align(capacity, ALIGN).unwrap(); 34 | unsafe { 35 | dealloc(self.data, layout); 36 | } 37 | } 38 | } 39 | 40 | unsafe impl Send for CoreIoEngine {} 41 | unsafe impl Sync for CoreIoEngine {} 42 | 43 | impl IoEngine for CoreIoEngine { 44 | fn get_nr_blocks(&self) -> u64 { 45 | self.nr_blocks 46 | } 47 | 48 | fn get_batch_size(&self) -> usize { 49 | 1 50 | } 51 | 52 | fn suggest_nr_threads(&self) -> usize { 53 | 1 54 | } 55 | 56 | fn read(&self, b: u64) -> io::Result { 57 | if b >= self.nr_blocks { 58 | return Err(io::Error::from(io::ErrorKind::InvalidInput)); 59 | } 60 | let block = Block::new(b); 61 | unsafe { 62 | let off = b as isize * BLOCK_SIZE as isize; 63 | std::ptr::copy( 64 | self.data.offset(off), 65 | block.get_data().as_mut_ptr(), 66 | BLOCK_SIZE, 67 | ); 68 | } 69 | Ok(block) 70 | } 71 | 72 | fn read_many(&self, blocks: &[u64]) -> io::Result>> { 73 | let mut bs = Vec::new(); 74 | for b in blocks { 75 | bs.push(self.read(*b)); 76 | } 77 | Ok(bs) 78 | } 79 | 80 | fn write(&self, block: &Block) -> io::Result<()> { 81 | if block.loc >= self.nr_blocks { 82 | return Err(io::Error::from(io::ErrorKind::InvalidInput)); 83 | } 84 | unsafe { 85 | let off = block.loc as isize * BLOCK_SIZE as isize; 86 | std::ptr::copy(block.get_data().as_ptr(), self.data.offset(off), BLOCK_SIZE); 87 | } 88 | Ok(()) 89 | } 90 | 91 | fn write_many(&self, blocks: &[Block]) -> io::Result>> { 92 | let mut ret = Vec::new(); 93 | for b in blocks { 94 | ret.push(self.write(b)); 95 | } 96 | Ok(ret) 97 | } 98 | } 99 | 100 | //------------------------------------------ 101 | 102 | pub fn trash_block(engine: &dyn IoEngine, b: u64) { 103 | let block = Block::zeroed(b); 104 | assert!(engine.write(&block).is_ok()); 105 | } 106 | 107 | //------------------------------------------ 108 | -------------------------------------------------------------------------------- /src/io_engine/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod base; 2 | pub mod buffer; 3 | pub mod gaps; 4 | pub mod spindle; 5 | pub mod sync; 6 | pub mod utils; 7 | 8 | pub use crate::io_engine::base::*; 9 | pub use crate::io_engine::spindle::SpindleIoEngine; 10 | pub use crate::io_engine::sync::SyncIoEngine; 11 | 12 | #[cfg(feature = "io_uring")] 13 | pub mod async_; 14 | 15 | #[cfg(feature = "io_uring")] 16 | pub use crate::io_engine::async_::AsyncIoEngine; 17 | 18 | #[cfg(test)] 19 | pub mod core; 20 | 21 | #[cfg(test)] 22 | pub mod ramdisk; 23 | -------------------------------------------------------------------------------- /src/ioctl/tests.rs: -------------------------------------------------------------------------------- 1 | use crate::ioctl::*; 2 | 3 | //------------------------------------------ 4 | 5 | #[cfg(any( 6 | target_arch = "mips", 7 | target_arch = "mips64", 8 | target_arch = "powerpc", 9 | target_arch = "powerpc64", 10 | target_arch = "sparc", 11 | target_arch = "sparc64" 12 | ))] 13 | mod expected { 14 | use super::RequestType; 15 | pub const BLKDISCARD: RequestType = 0x20001277; 16 | 17 | #[cfg(target_pointer_width = "32")] 18 | mod sized { 19 | use super::RequestType; 20 | pub const BLKBSZSET: RequestType = 0x80041271; 21 | pub const BLKGETSIZE64: RequestType = 0x40041272; 22 | } 23 | 24 | #[cfg(target_pointer_width = "64")] 25 | mod sized { 26 | use super::RequestType; 27 | pub const BLKBSZSET: RequestType = 0x80081271; 28 | pub const BLKGETSIZE64: RequestType = 0x40081272; 29 | } 30 | 31 | pub use sized::*; 32 | } 33 | 34 | #[cfg(not(any( 35 | target_arch = "mips", 36 | target_arch = "mips64", 37 | target_arch = "powerpc", 38 | target_arch = "powerpc64", 39 | target_arch = "sparc", 40 | target_arch = "sparc64" 41 | )))] 42 | mod expected { 43 | use super::RequestType; 44 | pub const BLKDISCARD: RequestType = 0x1277; 45 | 46 | #[cfg(target_pointer_width = "32")] 47 | mod sized { 48 | use super::RequestType; 49 | pub const BLKBSZSET: RequestType = 0x40041271; 50 | pub const BLKGETSIZE64: RequestType = 0x80041272; 51 | } 52 | 53 | #[cfg(target_pointer_width = "64")] 54 | mod sized { 55 | use super::RequestType; 56 | pub const BLKBSZSET: RequestType = 0x40081271; 57 | pub const BLKGETSIZE64: RequestType = 0x80081272; 58 | } 59 | 60 | pub use sized::*; 61 | } 62 | 63 | #[test] 64 | fn test_ioc_none() { 65 | assert_eq!(crate::request_code_none!(0x12, 119), expected::BLKDISCARD); 66 | } 67 | 68 | #[test] 69 | fn test_ioc_read_usize() { 70 | assert_eq!( 71 | crate::request_code_read!(0x12, 114, usize), 72 | expected::BLKGETSIZE64 73 | ); 74 | } 75 | 76 | #[test] 77 | fn test_ioc_write_usize() { 78 | assert_eq!( 79 | crate::request_code_write!(0x12, 113, usize), 80 | expected::BLKBSZSET 81 | ); 82 | } 83 | 84 | //------------------------------------------ 85 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | extern crate quickcheck; 3 | #[cfg(test)] 4 | #[macro_use(quickcheck)] 5 | extern crate quickcheck_macros; 6 | 7 | pub mod cache; 8 | pub mod checksum; 9 | pub mod commands; 10 | pub mod copier; 11 | pub mod dump_utils; 12 | pub mod era; 13 | pub mod file_utils; 14 | pub mod grid_layout; 15 | pub mod io_engine; 16 | pub mod ioctl; 17 | pub mod math; 18 | pub mod pack; 19 | pub mod pdata; 20 | pub mod report; 21 | pub mod run_iter; 22 | pub mod shrink; 23 | pub mod thin; 24 | pub mod units; 25 | pub mod utils; 26 | pub mod version; 27 | pub mod write_batcher; 28 | pub mod xml; 29 | 30 | #[cfg(any(test, feature = "devtools"))] 31 | pub mod random; 32 | 33 | #[cfg(feature = "devtools")] 34 | pub mod devtools; 35 | 36 | pub use utils::hashvec; 37 | -------------------------------------------------------------------------------- /src/math.rs: -------------------------------------------------------------------------------- 1 | use std::ops::{Add, Div, Sub}; 2 | 3 | //----------------------------------------- 4 | 5 | pub fn div_up(v: T, divisor: T) -> T 6 | where 7 | T: Copy + Add + Div + Sub + From, 8 | { 9 | (v + (divisor - T::from(1u8))) / divisor 10 | } 11 | 12 | #[test] 13 | fn test_div_up() { 14 | assert_eq!(div_up(4usize, 4usize), 1); 15 | assert_eq!(div_up(5usize, 4usize), 2); 16 | assert_eq!(div_up(34usize, 8usize), 5); 17 | } 18 | 19 | //----------------------------------------- 20 | -------------------------------------------------------------------------------- /src/pack/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod node_encode; 2 | pub mod toplevel; 3 | pub mod vm; 4 | 5 | mod delta_list; 6 | -------------------------------------------------------------------------------- /src/pdata/array/tests.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | //------------------------------------------ 4 | 5 | fn mk_random_block(nr_entries: usize) -> ArrayBlock { 6 | use rand::{thread_rng, Rng}; 7 | 8 | let mut values = vec![0u64; nr_entries]; 9 | thread_rng().fill(&mut values[..]); 10 | 11 | ArrayBlock { 12 | header: ArrayBlockHeader { 13 | max_entries: calc_max_entries::() as u32, 14 | nr_entries: nr_entries as u32, 15 | value_size: u64::disk_size(), 16 | blocknr: 0, 17 | }, 18 | values, 19 | } 20 | } 21 | 22 | #[test] 23 | fn pack_unpack_empty_block() { 24 | let origin = mk_random_block(0); 25 | 26 | let mut buffer: Vec = Vec::with_capacity(BLOCK_SIZE); 27 | let mut cursor = std::io::Cursor::new(&mut buffer); 28 | assert!(pack_array_block(&origin, &mut cursor).is_ok()); 29 | 30 | let path = vec![0]; 31 | let unpacked = unpack_array_block::(&path, buffer.as_slice()).unwrap(); 32 | assert_eq!(unpacked.header, origin.header); 33 | assert!(unpacked.values.is_empty()); 34 | } 35 | 36 | #[test] 37 | fn pack_unpack_fully_populated_block() { 38 | let origin = mk_random_block(calc_max_entries::()); 39 | 40 | let mut buffer: Vec = Vec::with_capacity(BLOCK_SIZE); 41 | let mut cursor = std::io::Cursor::new(&mut buffer); 42 | assert!(pack_array_block(&origin, &mut cursor).is_ok()); 43 | 44 | let path = vec![0]; 45 | let unpacked = unpack_array_block::(&path, buffer.as_slice()).unwrap(); 46 | assert_eq!(unpacked.header, origin.header); 47 | assert_eq!(unpacked.values, origin.values); 48 | } 49 | 50 | //------------------------------------------ 51 | -------------------------------------------------------------------------------- /src/pdata/array_builder/test_utils.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | use crate::pdata::btree_builder::test_utils::*; 4 | 5 | //------------------------------------------ 6 | 7 | pub struct ArrayLayout { 8 | ablocks: Vec, 9 | pub block_tree: BTreeLayout, 10 | } 11 | 12 | impl ArrayLayout { 13 | pub fn height(&self) -> usize { 14 | self.block_tree.height() 15 | } 16 | 17 | pub fn nr_nodes(&self) -> u64 { 18 | self.block_tree.nr_nodes() 19 | } 20 | 21 | pub fn nr_leaves(&self) -> u64 { 22 | self.block_tree.nr_leaves() 23 | } 24 | 25 | pub fn root(&self) -> NodeInfo { 26 | self.block_tree.root() 27 | } 28 | 29 | pub fn nodes(&self, height: usize) -> &[NodeInfo] { 30 | self.block_tree.nodes(height) 31 | } 32 | 33 | pub fn leaves(&self) -> &[NodeInfo] { 34 | self.block_tree.leaves() 35 | } 36 | 37 | pub fn array_blocks(&self) -> &[u64] { 38 | &self.ablocks 39 | } 40 | 41 | pub fn nr_array_blocks(&self) -> u64 { 42 | self.ablocks.len() as u64 43 | } 44 | 45 | // Returns the index of the first array block under a specific node 46 | pub fn first_array_block(&self, height: usize, index: u64) -> u64 { 47 | let leaf_idx = self.block_tree.first_leaf(height, index); 48 | let nr_leaves = self.block_tree.nr_leaves(); 49 | assert!(leaf_idx <= nr_leaves); 50 | 51 | if leaf_idx == nr_leaves { 52 | self.nr_array_blocks() 53 | } else { 54 | self.block_tree.leaves()[leaf_idx as usize].entries_begin 55 | } 56 | } 57 | } 58 | 59 | //------------------------------------------ 60 | 61 | pub fn build_array_blocks( 62 | w: &mut WriteBatcher, 63 | values: &[V], 64 | ) -> Vec { 65 | let mut builder = ArrayBlockBuilder::::new(values.len() as u64); 66 | for (i, v) in values.iter().enumerate() { 67 | assert!(builder.push_value(w, i as u64, v.clone()).is_ok()); 68 | } 69 | builder.complete(w).unwrap() 70 | } 71 | 72 | pub fn build_array_from_values( 73 | w: &mut WriteBatcher, 74 | values: &[V], 75 | ) -> ArrayLayout { 76 | assert!(!values.is_empty()); 77 | let ablocks = build_array_blocks(w, values); 78 | 79 | // manually build the block tree so that we could obtain the layout 80 | let mappings: Vec<(u64, u64)> = ablocks 81 | .iter() 82 | .enumerate() 83 | .map(|(i, v)| (i as u64, *v)) 84 | .collect(); 85 | let block_tree = build_btree_from_mappings(w, &mappings); 86 | 87 | ArrayLayout { 88 | ablocks, 89 | block_tree, 90 | } 91 | } 92 | 93 | //------------------------------------------ 94 | -------------------------------------------------------------------------------- /src/pdata/btree/tests.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | //------------------------------------------ 4 | 5 | fn mk_random_leaf(nr_entries: usize) -> Node { 6 | use rand::{thread_rng, Rng}; 7 | 8 | let mut keys = vec![0u64; nr_entries]; 9 | let mut values = vec![0u64; nr_entries]; 10 | thread_rng().fill(&mut keys[..]); 11 | thread_rng().fill(&mut values[..]); 12 | keys.sort_unstable(); 13 | 14 | Node::Leaf { 15 | header: NodeHeader { 16 | block: 0, 17 | is_leaf: true, 18 | nr_entries: nr_entries as u32, 19 | max_entries: calc_max_entries::() as u32, 20 | value_size: u64::disk_size(), 21 | }, 22 | keys, 23 | values, 24 | } 25 | } 26 | 27 | #[test] 28 | fn pack_unpack_empty_node() { 29 | let node = mk_random_leaf(0); 30 | 31 | let mut buffer: Vec = Vec::with_capacity(BLOCK_SIZE); 32 | let mut cursor = std::io::Cursor::new(&mut buffer); 33 | assert!(pack_node(&node, &mut cursor).is_ok()); 34 | 35 | let path = vec![0]; 36 | let unpacked = unpack_node::(&path, buffer.as_slice(), false, true).unwrap(); 37 | assert!( 38 | matches!(unpacked, Node::Leaf {header, keys, values} if header.eq(node.get_header()) && keys.is_empty() && values.is_empty()) 39 | ); 40 | } 41 | 42 | #[test] 43 | fn pack_unpack_fully_populated_node() { 44 | let node = mk_random_leaf(calc_max_entries::()); 45 | let v = if let Node::Leaf { ref values, .. } = node { 46 | values.clone() 47 | } else { 48 | Vec::new() 49 | }; 50 | 51 | let mut buffer: Vec = Vec::with_capacity(BLOCK_SIZE); 52 | let mut cursor = std::io::Cursor::new(&mut buffer); 53 | assert!(pack_node(&node, &mut cursor).is_ok()); 54 | 55 | let path = vec![0]; 56 | let unpacked = unpack_node::(&path, buffer.as_slice(), false, true).unwrap(); 57 | assert!( 58 | matches!(unpacked, Node::Leaf {header, keys, values} if header.eq(node.get_header()) && keys.eq(node.get_keys()) && values.eq(&v)) 59 | ); 60 | } 61 | 62 | //------------------------------------------ 63 | -------------------------------------------------------------------------------- /src/pdata/btree_lookup.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | 3 | use crate::io_engine::*; 4 | use crate::pdata::btree::*; 5 | use crate::pdata::unpack::*; 6 | 7 | //------------------------------------------ 8 | 9 | // Inefficient because we unpack entire nodes, but I think that's 10 | // fine for the current use cases (eg, lookup of dev_details). 11 | pub fn btree_lookup(engine: &dyn IoEngine, root: u64, key: u64) -> Result> 12 | where 13 | V: Unpack + Clone, 14 | { 15 | let mut path = Vec::new(); 16 | let mut loc = root; 17 | let mut is_root = true; 18 | 19 | loop { 20 | let block = engine.read(loc)?; 21 | let node = unpack_node::(&path, block.get_data(), true, is_root)?; 22 | match node { 23 | Node::Internal { keys, values, .. } => { 24 | // Select a child ... 25 | let idx = match keys.binary_search(&key) { 26 | Ok(idx) => idx, 27 | Err(idx) => { 28 | if idx == 0 { 29 | return Ok(None); 30 | } 31 | 32 | idx - 1 33 | } 34 | }; 35 | 36 | // ... and move to it. 37 | loc = values[idx]; 38 | path.push(values[idx]); 39 | } 40 | Node::Leaf { keys, values, .. } => { 41 | let idx = keys.binary_search(&key); 42 | return match idx { 43 | Ok(idx) => Ok(Some(values[idx].clone())), 44 | Err(_) => Ok(None), 45 | }; 46 | } 47 | } 48 | 49 | is_root = false; // After the first iteration, we are no longer at the root. 50 | } 51 | } 52 | 53 | //------------------------------------------ 54 | -------------------------------------------------------------------------------- /src/pdata/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod array; 2 | pub mod array_builder; 3 | pub mod array_walker; 4 | pub mod bitset; 5 | pub mod btree; 6 | pub mod btree_builder; 7 | pub mod btree_error; 8 | pub mod btree_iterator; 9 | pub mod btree_leaf_walker; 10 | pub mod btree_lookup; 11 | pub mod btree_merge; 12 | pub mod btree_walker; 13 | pub mod space_map; 14 | pub mod unpack; 15 | -------------------------------------------------------------------------------- /src/pdata/space_map/allocated_blocks.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use roaring::RoaringBitmap; 3 | 4 | use crate::io_engine::*; 5 | use crate::pdata::space_map::common::*; 6 | use crate::pdata::space_map::metadata::*; 7 | use crate::pdata::unpack::*; 8 | 9 | //---------------------------------- 10 | 11 | struct IndexInfo { 12 | key: u64, 13 | loc: u64, 14 | } 15 | 16 | pub fn allocated_blocks( 17 | engine: &dyn IoEngine, 18 | sm_root: u64, 19 | nr_blocks: u64, 20 | ) -> Result { 21 | // Walk index tree to find where the bitmaps are. 22 | let b = engine.read(sm_root)?; 23 | let indexes = load_metadata_index(&b, nr_blocks)?; 24 | 25 | let mut infos: Vec<_> = indexes 26 | .indexes 27 | .iter() 28 | .enumerate() 29 | .map(|(key, entry)| IndexInfo { 30 | key: key as u64, 31 | loc: entry.blocknr, 32 | }) 33 | .collect(); 34 | 35 | // Read bitmaps in sequence 36 | infos.sort_by(|lhs, rhs| lhs.loc.partial_cmp(&rhs.loc).unwrap()); 37 | 38 | let mut bits = RoaringBitmap::new(); 39 | for info in &infos { 40 | let b = engine.read(info.loc)?; 41 | let base = info.key * ENTRIES_PER_BITMAP as u64; 42 | let (_, bm) = Bitmap::unpack(b.get_data())?; 43 | 44 | for i in 0..bm.entries.len() { 45 | if let BitmapEntry::Small(0) = bm.entries[i] { 46 | // nothing 47 | } else { 48 | bits.insert((base + i as u64) as u32); 49 | } 50 | } 51 | } 52 | 53 | Ok(bits) 54 | } 55 | 56 | //---------------------------------- 57 | -------------------------------------------------------------------------------- /src/pdata/space_map/disk.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | 3 | use crate::pdata::btree_builder::*; 4 | use crate::pdata::space_map::common::*; 5 | use crate::pdata::space_map::*; 6 | use crate::write_batcher::*; 7 | 8 | //------------------------------------------ 9 | 10 | pub fn write_disk_sm(w: &mut WriteBatcher, sm: &dyn SpaceMap) -> Result { 11 | let (index_entries, ref_count_root) = write_common(w, sm)?; 12 | 13 | let mut index_builder: BTreeBuilder = BTreeBuilder::new(Box::new(NoopRC {})); 14 | for (i, ie) in index_entries.iter().enumerate() { 15 | index_builder.push_value(w, i as u64, *ie)?; 16 | } 17 | 18 | let bitmap_root = index_builder.complete(w)?; 19 | w.flush()?; 20 | 21 | Ok(SMRoot { 22 | nr_blocks: sm.get_nr_blocks()?, 23 | nr_allocated: sm.get_nr_allocated()?, 24 | bitmap_root, 25 | ref_count_root, 26 | }) 27 | } 28 | 29 | //------------------------------------------ 30 | -------------------------------------------------------------------------------- /src/pdata/space_map/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod allocated_blocks; 2 | pub mod base; 3 | pub mod checker; 4 | pub mod common; 5 | pub mod disk; 6 | pub mod metadata; 7 | 8 | pub use crate::pdata::space_map::base::*; 9 | 10 | #[cfg(test)] 11 | pub mod tests; 12 | -------------------------------------------------------------------------------- /src/pdata/unpack.rs: -------------------------------------------------------------------------------- 1 | use byteorder::{LittleEndian, WriteBytesExt}; 2 | use nom::{number::complete::*, IResult}; 3 | use std::io::{self, ErrorKind}; 4 | 5 | //------------------------------------------ 6 | 7 | pub trait Unpack { 8 | // The size of the value when on disk. 9 | fn disk_size() -> u32; 10 | fn unpack(data: &[u8]) -> IResult<&[u8], Self> 11 | where 12 | Self: std::marker::Sized; 13 | } 14 | 15 | pub fn unpack(data: &[u8]) -> io::Result { 16 | match U::unpack(data) { 17 | Err(_e) => Err(io::Error::from(ErrorKind::InvalidData)), 18 | Ok((_i, v)) => Ok(v), 19 | } 20 | } 21 | 22 | //------------------------------------------ 23 | 24 | pub trait Pack { 25 | fn pack(&self, data: &mut W) -> io::Result<()>; 26 | } 27 | 28 | //------------------------------------------ 29 | 30 | impl Unpack for u64 { 31 | fn disk_size() -> u32 { 32 | 8 33 | } 34 | 35 | fn unpack(i: &[u8]) -> IResult<&[u8], u64> { 36 | le_u64(i) 37 | } 38 | } 39 | 40 | impl Pack for u64 { 41 | fn pack(&self, out: &mut W) -> io::Result<()> { 42 | out.write_u64::(*self) 43 | } 44 | } 45 | 46 | impl Unpack for u32 { 47 | fn disk_size() -> u32 { 48 | 4 49 | } 50 | 51 | fn unpack(i: &[u8]) -> IResult<&[u8], u32> { 52 | le_u32(i) 53 | } 54 | } 55 | 56 | impl Pack for u32 { 57 | fn pack(&self, out: &mut W) -> io::Result<()> { 58 | out.write_u32::(*self) 59 | } 60 | } 61 | 62 | //------------------------------------------ 63 | -------------------------------------------------------------------------------- /src/random.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; 3 | use std::io::Cursor; 4 | 5 | //------------------------------------ 6 | 7 | // A simple linear congruence generator used to create the data to 8 | // go into the thin/cache blocks. 9 | pub struct Generator { 10 | x: u64, 11 | a: u64, 12 | c: u64, 13 | } 14 | 15 | impl Generator { 16 | pub fn new() -> Generator { 17 | Generator { 18 | x: 0, 19 | a: 6364136223846793005, 20 | c: 1442695040888963407, 21 | } 22 | } 23 | 24 | fn step(&mut self) { 25 | self.x = self.a.wrapping_mul(self.x).wrapping_add(self.c) 26 | } 27 | 28 | pub fn fill_buffer(&mut self, seed: u64, bytes: &mut [u8]) -> Result<()> { 29 | self.x = seed; 30 | 31 | assert!(bytes.len() % 8 == 0); 32 | let nr_words = bytes.len() / 8; 33 | let mut out = Cursor::new(bytes); 34 | 35 | for _ in 0..nr_words { 36 | out.write_u64::(self.x)?; 37 | self.step(); 38 | } 39 | 40 | Ok(()) 41 | } 42 | 43 | pub fn verify_buffer(&mut self, seed: u64, bytes: &[u8]) -> Result { 44 | self.x = seed; 45 | 46 | assert!(bytes.len() % 8 == 0); 47 | let nr_words = bytes.len() / 8; 48 | let mut input = Cursor::new(bytes); 49 | 50 | for _ in 0..nr_words { 51 | let w = input.read_u64::()?; 52 | if w != self.x { 53 | eprintln!("{} != {}", w, self.x); 54 | return Ok(false); 55 | } 56 | self.step(); 57 | } 58 | 59 | Ok(true) 60 | } 61 | } 62 | 63 | impl Default for Generator { 64 | fn default() -> Self { 65 | Self::new() 66 | } 67 | } 68 | 69 | //------------------------------------ 70 | -------------------------------------------------------------------------------- /src/run_iter.rs: -------------------------------------------------------------------------------- 1 | use roaring::*; 2 | use std::ops::Range; 3 | 4 | //----------------------------------------- 5 | 6 | pub struct RunIter { 7 | len: u32, 8 | current: u32, 9 | bits: RoaringBitmap, 10 | } 11 | 12 | impl RunIter { 13 | pub fn new(bits: RoaringBitmap, len: u32) -> Self { 14 | Self { 15 | len, 16 | current: 0, 17 | bits, 18 | } 19 | } 20 | } 21 | 22 | impl Iterator for RunIter { 23 | type Item = (bool, Range); 24 | 25 | fn next(&mut self) -> Option { 26 | if self.current == self.len { 27 | None 28 | } else { 29 | let b = self.bits.contains(self.current); 30 | let start = self.current; 31 | self.current += 1; 32 | while self.current < self.len && self.bits.contains(self.current) == b { 33 | self.current += 1; 34 | } 35 | Some((b, start..self.current)) 36 | } 37 | } 38 | } 39 | 40 | //----------------------------------------- 41 | 42 | #[cfg(test)] 43 | mod run_iter_tests { 44 | use super::*; 45 | 46 | struct Test { 47 | bits: Vec, 48 | expected: Vec<(bool, Range)>, 49 | } 50 | 51 | #[test] 52 | fn test_run_iter() { 53 | let tests = vec![ 54 | Test { 55 | bits: vec![], 56 | expected: vec![], 57 | }, 58 | Test { 59 | bits: vec![false, false, false], 60 | expected: vec![(false, 0..3)], 61 | }, 62 | Test { 63 | bits: vec![false, true, true, false, false, false, true], 64 | expected: vec![(false, 0..1), (true, 1..3), (false, 3..6), (true, 6..7)], 65 | }, 66 | Test { 67 | bits: vec![false, true, true, false, false, false, true, false, false], 68 | expected: vec![ 69 | (false, 0..1), 70 | (true, 1..3), 71 | (false, 3..6), 72 | (true, 6..7), 73 | (false, 7..9), 74 | ], 75 | }, 76 | ]; 77 | 78 | for t in tests { 79 | let mut bits = RoaringBitmap::new(); 80 | for (i, b) in t.bits.iter().enumerate() { 81 | if *b { 82 | bits.insert(i as u32); 83 | } 84 | } 85 | 86 | let it = RunIter::new(bits, t.bits.len() as u32); 87 | let actual: Vec<(bool, Range)> = it.collect(); 88 | assert_eq!(actual, t.expected); 89 | } 90 | } 91 | } 92 | 93 | //----------------------------------------- 94 | -------------------------------------------------------------------------------- /src/shrink/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod toplevel; 2 | -------------------------------------------------------------------------------- /src/thin/block_time.rs: -------------------------------------------------------------------------------- 1 | use byteorder::WriteBytesExt; 2 | use nom::{number::complete::*, IResult}; 3 | use std::fmt; 4 | use std::io; 5 | 6 | use crate::pdata::unpack::*; 7 | 8 | //------------------------------------------ 9 | 10 | #[derive(Clone, Copy, PartialEq, Eq)] 11 | pub struct BlockTime { 12 | pub block: u64, 13 | pub time: u32, 14 | } 15 | 16 | impl Unpack for BlockTime { 17 | fn disk_size() -> u32 { 18 | 8 19 | } 20 | 21 | fn unpack(i: &[u8]) -> IResult<&[u8], BlockTime> { 22 | let (i, n) = le_u64(i)?; 23 | let block = n >> 24; 24 | let time = n & ((1 << 24) - 1); 25 | 26 | Ok(( 27 | i, 28 | BlockTime { 29 | block, 30 | time: time as u32, 31 | }, 32 | )) 33 | } 34 | } 35 | 36 | impl Pack for BlockTime { 37 | fn pack(&self, data: &mut W) -> io::Result<()> { 38 | let bt: u64 = (self.block << 24) | self.time as u64; 39 | bt.pack(data) 40 | } 41 | } 42 | 43 | impl fmt::Display for BlockTime { 44 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 45 | write!(f, "{} @ {}", self.block, self.time) 46 | } 47 | } 48 | 49 | //------------------------------------------ 50 | -------------------------------------------------------------------------------- /src/thin/damage_generator.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | 3 | use std::path::Path; 4 | use std::sync::Arc; 5 | 6 | use crate::commands::engine::*; 7 | use crate::devtools::damage_generator::*; 8 | use crate::io_engine::IoEngine; 9 | 10 | use crate::pdata::space_map::common::*; 11 | 12 | use crate::pdata::unpack::unpack; 13 | use crate::thin::superblock::*; 14 | 15 | //------------------------------------------ 16 | 17 | pub struct SuperblockOverrides { 18 | pub mapping_root: Option, 19 | pub details_root: Option, 20 | pub metadata_snapshot: Option, 21 | } 22 | 23 | pub fn override_superblock( 24 | engine: Arc, 25 | opts: &SuperblockOverrides, 26 | ) -> Result<()> { 27 | let mut sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; 28 | if let Some(v) = opts.mapping_root { 29 | sb.mapping_root = v; 30 | } 31 | if let Some(v) = opts.details_root { 32 | sb.details_root = v; 33 | } 34 | if let Some(v) = opts.metadata_snapshot { 35 | sb.metadata_snap = v; 36 | } 37 | write_superblock(engine.as_ref(), 0, &sb) 38 | } 39 | 40 | //------------------------------------------ 41 | 42 | pub enum DamageOp { 43 | CreateMetadataLeaks { 44 | nr_blocks: usize, 45 | expected_rc: u32, 46 | actual_rc: u32, 47 | }, 48 | OverrideSuperblock(SuperblockOverrides), 49 | } 50 | 51 | pub struct ThinDamageOpts<'a> { 52 | pub engine_opts: EngineOptions, 53 | pub op: DamageOp, 54 | pub output: &'a Path, 55 | } 56 | 57 | pub fn damage_metadata(opts: ThinDamageOpts) -> Result<()> { 58 | let engine = EngineBuilder::new(opts.output, &opts.engine_opts) 59 | .write(true) 60 | .build()?; 61 | let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; 62 | let sm_root = unpack::(&sb.metadata_sm_root)?; 63 | 64 | match opts.op { 65 | DamageOp::CreateMetadataLeaks { 66 | nr_blocks, 67 | expected_rc, 68 | actual_rc, 69 | } => create_metadata_leaks(engine, sm_root, nr_blocks, expected_rc, actual_rc), 70 | DamageOp::OverrideSuperblock(opts) => override_superblock(engine, &opts), 71 | } 72 | } 73 | 74 | //------------------------------------------ 75 | -------------------------------------------------------------------------------- /src/thin/device_detail.rs: -------------------------------------------------------------------------------- 1 | use byteorder::{LittleEndian, WriteBytesExt}; 2 | use nom::{number::complete::*, IResult}; 3 | use std::fmt; 4 | use std::io; 5 | 6 | use crate::pdata::unpack::*; 7 | 8 | //------------------------------------------ 9 | 10 | #[derive(Clone, Copy, Debug)] 11 | pub struct DeviceDetail { 12 | pub mapped_blocks: u64, 13 | pub transaction_id: u64, 14 | pub creation_time: u32, 15 | pub snapshotted_time: u32, 16 | } 17 | 18 | impl fmt::Display for DeviceDetail { 19 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 20 | write!( 21 | f, 22 | "mapped = {}, trans = {}, create = {}, snap = {}", 23 | self.mapped_blocks, self.transaction_id, self.creation_time, self.snapshotted_time 24 | )?; 25 | Ok(()) 26 | } 27 | } 28 | 29 | impl Unpack for DeviceDetail { 30 | fn disk_size() -> u32 { 31 | 24 32 | } 33 | 34 | fn unpack(i: &[u8]) -> IResult<&[u8], DeviceDetail> { 35 | let (i, mapped_blocks) = le_u64(i)?; 36 | let (i, transaction_id) = le_u64(i)?; 37 | let (i, creation_time) = le_u32(i)?; 38 | let (i, snapshotted_time) = le_u32(i)?; 39 | 40 | Ok(( 41 | i, 42 | DeviceDetail { 43 | mapped_blocks, 44 | transaction_id, 45 | creation_time, 46 | snapshotted_time, 47 | }, 48 | )) 49 | } 50 | } 51 | 52 | impl Pack for DeviceDetail { 53 | fn pack(&self, w: &mut W) -> io::Result<()> { 54 | w.write_u64::(self.mapped_blocks)?; 55 | w.write_u64::(self.transaction_id)?; 56 | w.write_u32::(self.creation_time)?; 57 | w.write_u32::(self.snapshotted_time) 58 | } 59 | } 60 | 61 | //------------------------------------------ 62 | -------------------------------------------------------------------------------- /src/thin/human_readable_format.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::io::Write; 3 | 4 | use crate::thin::ir::*; 5 | 6 | //--------------------------------------- 7 | 8 | pub struct HumanReadableWriter { 9 | w: W, 10 | } 11 | 12 | impl HumanReadableWriter { 13 | pub fn new(w: W) -> HumanReadableWriter { 14 | HumanReadableWriter { w } 15 | } 16 | } 17 | 18 | const METADATA_VERSION: u32 = 2; 19 | 20 | impl MetadataVisitor for HumanReadableWriter { 21 | fn superblock_b(&mut self, sb: &Superblock) -> Result { 22 | write!( 23 | self.w, 24 | "begin superblock: \"{}\", {}, {}, {}, {}, {}, {}", 25 | sb.uuid, 26 | sb.time, 27 | sb.transaction, 28 | sb.flags.unwrap_or(0), 29 | sb.version.unwrap_or(METADATA_VERSION), 30 | sb.data_block_size, 31 | sb.nr_data_blocks 32 | )?; 33 | 34 | if let Some(b) = sb.metadata_snap { 35 | writeln!(self.w, ", {}", b)?; 36 | } else { 37 | self.w.write_all(b"\n")?; 38 | } 39 | 40 | Ok(Visit::Continue) 41 | } 42 | 43 | fn superblock_e(&mut self) -> Result { 44 | self.w.write_all(b"end superblock\n")?; 45 | Ok(Visit::Continue) 46 | } 47 | 48 | fn def_shared_b(&mut self, name: &str) -> Result { 49 | writeln!(self.w, "def: {}", name)?; 50 | Ok(Visit::Continue) 51 | } 52 | 53 | fn def_shared_e(&mut self) -> Result { 54 | self.w.write_all(b"\n")?; 55 | Ok(Visit::Continue) 56 | } 57 | 58 | fn device_b(&mut self, d: &Device) -> Result { 59 | // The words "mapped_blocks" is connected by an underscore for backward compatibility. 60 | writeln!( 61 | self.w, 62 | "device: {}\nmapped_blocks: {}\ntransaction: {}\ncreation time: {}\nsnap time: {}", 63 | d.dev_id, d.mapped_blocks, d.transaction, d.creation_time, d.snap_time 64 | )?; 65 | Ok(Visit::Continue) 66 | } 67 | 68 | fn device_e(&mut self) -> Result { 69 | self.w.write_all(b"\n")?; 70 | Ok(Visit::Continue) 71 | } 72 | 73 | fn map(&mut self, m: &Map) -> Result { 74 | writeln!( 75 | self.w, 76 | " ({}..{}) -> ({}..{}), {}", 77 | m.thin_begin, 78 | m.thin_begin + m.len - 1, 79 | m.data_begin, 80 | m.data_begin + m.len - 1, 81 | m.time 82 | )?; 83 | Ok(Visit::Continue) 84 | } 85 | 86 | fn ref_shared(&mut self, name: &str) -> Result { 87 | writeln!(self.w, " ref: {}", name)?; 88 | Ok(Visit::Continue) 89 | } 90 | 91 | fn eof(&mut self) -> Result { 92 | self.w.flush()?; 93 | Ok(Visit::Continue) 94 | } 95 | } 96 | 97 | //--------------------------------------- 98 | -------------------------------------------------------------------------------- /src/thin/ir.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | 3 | //------------------------------------------ 4 | 5 | #[derive(Clone)] 6 | pub struct Superblock { 7 | pub uuid: String, 8 | pub time: u32, 9 | pub transaction: u64, 10 | pub flags: Option, 11 | pub version: Option, 12 | pub data_block_size: u32, 13 | pub nr_data_blocks: u64, 14 | pub metadata_snap: Option, 15 | } 16 | 17 | #[derive(Clone)] 18 | pub struct Device { 19 | pub dev_id: u32, 20 | pub mapped_blocks: u64, 21 | pub transaction: u64, 22 | pub creation_time: u32, 23 | pub snap_time: u32, 24 | } 25 | 26 | #[derive(Clone)] 27 | pub struct Map { 28 | pub thin_begin: u64, 29 | pub data_begin: u64, 30 | pub time: u32, 31 | pub len: u64, 32 | } 33 | 34 | //------------------------------------------ 35 | 36 | #[derive(Clone)] 37 | pub enum Visit { 38 | Continue, 39 | Stop, 40 | } 41 | 42 | pub trait MetadataVisitor { 43 | fn superblock_b(&mut self, sb: &Superblock) -> Result; 44 | fn superblock_e(&mut self) -> Result; 45 | 46 | // Defines a shared sub tree. May only contain a 'map' (no 'ref' allowed). 47 | fn def_shared_b(&mut self, name: &str) -> Result; 48 | fn def_shared_e(&mut self) -> Result; 49 | 50 | // A device contains a number of 'map' or 'ref' items. 51 | fn device_b(&mut self, d: &Device) -> Result; 52 | fn device_e(&mut self) -> Result; 53 | 54 | fn map(&mut self, m: &Map) -> Result; 55 | fn ref_shared(&mut self, name: &str) -> Result; 56 | 57 | fn eof(&mut self) -> Result; 58 | } 59 | 60 | //------------------------------------------ 61 | -------------------------------------------------------------------------------- /src/thin/metadata_generator.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::path::Path; 3 | use std::sync::Arc; 4 | 5 | use crate::commands::engine::*; 6 | use crate::io_engine::*; 7 | use crate::pdata::space_map::metadata::core_metadata_sm; 8 | use crate::report::mk_quiet_report; 9 | use crate::thin::ir::MetadataVisitor; 10 | use crate::thin::restore::Restorer; 11 | use crate::write_batcher::WriteBatcher; 12 | 13 | //------------------------------------------ 14 | 15 | pub trait MetadataGenerator { 16 | fn generate_metadata(&self, v: &mut dyn MetadataVisitor) -> Result<()>; 17 | } 18 | 19 | struct ThinGenerator; 20 | 21 | impl MetadataGenerator for ThinGenerator { 22 | fn generate_metadata(&self, _v: &mut dyn MetadataVisitor) -> Result<()> { 23 | Ok(()) // TODO 24 | } 25 | } 26 | 27 | //------------------------------------------ 28 | 29 | fn format(engine: Arc, gen: ThinGenerator) -> Result<()> { 30 | let sm = core_metadata_sm(engine.get_nr_blocks(), u32::MAX); 31 | let batch_size = engine.get_batch_size(); 32 | let mut w = WriteBatcher::new(engine, sm, batch_size); 33 | let mut restorer = Restorer::new(&mut w, Arc::new(mk_quiet_report())); 34 | 35 | gen.generate_metadata(&mut restorer) 36 | } 37 | 38 | fn set_needs_check(engine: Arc, flag: bool) -> Result<()> { 39 | use crate::thin::superblock::*; 40 | 41 | let mut sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; 42 | sb.flags.needs_check = flag; 43 | write_superblock(engine.as_ref(), SUPERBLOCK_LOCATION, &sb) 44 | } 45 | 46 | //------------------------------------------ 47 | 48 | pub struct ThinFormatOpts { 49 | pub data_block_size: u32, 50 | pub nr_data_blocks: u64, 51 | } 52 | 53 | pub enum MetadataOp { 54 | Format(ThinFormatOpts), 55 | SetNeedsCheck(bool), 56 | } 57 | 58 | pub struct ThinGenerateOpts<'a> { 59 | pub engine_opts: EngineOptions, 60 | pub op: MetadataOp, 61 | pub output: &'a Path, 62 | } 63 | 64 | pub fn generate_metadata(opts: ThinGenerateOpts) -> Result<()> { 65 | let engine = EngineBuilder::new(opts.output, &opts.engine_opts) 66 | .write(true) 67 | .build()?; 68 | match opts.op { 69 | // FIXME: parameterize ThinGenerator 70 | MetadataOp::Format(_op) => format(engine, ThinGenerator), 71 | MetadataOp::SetNeedsCheck(flag) => set_needs_check(engine, flag), 72 | } 73 | } 74 | 75 | //------------------------------------------ 76 | -------------------------------------------------------------------------------- /src/thin/metadata_repair/sorting_roots_tests.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | //------------------------------------------ 4 | 5 | #[test] 6 | fn test_with_empty_lhs() { 7 | let lhs = BTreeMap::new(); 8 | let rhs = BTreeMap::from([(0, 10)]); 9 | let ordering = compare_time_counts(&lhs, &rhs); 10 | assert_eq!(ordering, Ordering::Greater); 11 | } 12 | 13 | #[test] 14 | fn test_with_empty_rhs() { 15 | let lhs = BTreeMap::from([(0, 10)]); 16 | let rhs = BTreeMap::new(); 17 | let ordering = compare_time_counts(&lhs, &rhs); 18 | assert_eq!(ordering, Ordering::Less); 19 | } 20 | 21 | #[test] 22 | fn test_with_two_empty_sets() { 23 | let lhs = BTreeMap::new(); 24 | let rhs = BTreeMap::new(); 25 | let ordering = compare_time_counts(&lhs, &rhs); 26 | assert_eq!(ordering, Ordering::Equal); 27 | } 28 | 29 | #[test] 30 | fn test_with_greater_time_at_lhs() { 31 | let lhs = BTreeMap::from([(1, 1), (0, 10)]); 32 | let rhs = BTreeMap::from([(0, 10)]); 33 | let ordering = compare_time_counts(&lhs, &rhs); 34 | assert_eq!(ordering, Ordering::Less); 35 | } 36 | 37 | #[test] 38 | fn test_with_greater_time_at_rhs() { 39 | let lhs = BTreeMap::from([(0, 10)]); 40 | let rhs = BTreeMap::from([(1, 1), (0, 10)]); 41 | let ordering = compare_time_counts(&lhs, &rhs); 42 | assert_eq!(ordering, Ordering::Greater); 43 | } 44 | 45 | #[test] 46 | fn test_with_greater_counts_at_lhs() { 47 | let lhs = BTreeMap::from([(1, 10), (0, 10)]); 48 | let rhs = BTreeMap::from([(1, 5), (0, 10)]); 49 | let ordering = compare_time_counts(&lhs, &rhs); 50 | assert_eq!(ordering, Ordering::Less); 51 | } 52 | 53 | #[test] 54 | fn test_with_greater_counts_at_rhs() { 55 | let lhs = BTreeMap::from([(1, 5), (0, 10)]); 56 | let rhs = BTreeMap::from([(1, 10), (0, 10)]); 57 | let ordering = compare_time_counts(&lhs, &rhs); 58 | assert_eq!(ordering, Ordering::Greater); 59 | } 60 | 61 | //------------------------------------------ 62 | -------------------------------------------------------------------------------- /src/thin/metadata_size.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{anyhow, Result}; 2 | 3 | use crate::io_engine::BLOCK_SIZE; 4 | use crate::math::div_up; 5 | use crate::pdata::btree::calc_max_entries; 6 | use crate::pdata::space_map::metadata::MAX_METADATA_BLOCKS; 7 | use crate::thin::block_time::BlockTime; 8 | 9 | //------------------------------------------ 10 | 11 | const MIN_DATA_BLOCK_SIZE: u64 = 65536; 12 | const MAX_DATA_BLOCK_SIZE: u64 = 1073741824; 13 | 14 | pub struct ThinMetadataSizeOptions { 15 | pub nr_blocks: u64, 16 | pub max_thins: u64, 17 | } 18 | 19 | pub fn check_data_block_size(block_size: u64) -> Result<()> { 20 | if block_size == 0 || (block_size & (MIN_DATA_BLOCK_SIZE - 1)) != 0 { 21 | return Err(anyhow!("block size must be a multiple of 64 KiB")); 22 | } 23 | 24 | if block_size > MAX_DATA_BLOCK_SIZE { 25 | return Err(anyhow!("maximum block size is 1 GiB")); 26 | } 27 | 28 | Ok(()) 29 | } 30 | 31 | // Returns estimated size in bytes 32 | pub fn metadata_size(opts: &ThinMetadataSizeOptions) -> Result { 33 | // assuming 50% residency on mapping tree leaves 34 | let entries_per_node: u64 = calc_max_entries::() as u64 / 2; 35 | // number of leaves for data mappings 36 | let nr_leaves = div_up(opts.nr_blocks, entries_per_node); 37 | 38 | // one for the superblock, plus additional roots for each device 39 | let mut nr_blocks = 1 + nr_leaves + opts.max_thins; 40 | nr_blocks = std::cmp::min(nr_blocks, MAX_METADATA_BLOCKS as u64); 41 | 42 | Ok(nr_blocks * BLOCK_SIZE as u64) 43 | } 44 | 45 | //------------------------------------------ 46 | -------------------------------------------------------------------------------- /src/thin/migrate/metadata.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{anyhow, Result}; 2 | use std::sync::Arc; 3 | 4 | use crate::io_engine::*; 5 | use crate::pdata::btree_iterator::*; 6 | use crate::pdata::btree_lookup::*; 7 | use crate::pdata::unpack::*; 8 | use crate::thin::block_time::*; 9 | use crate::thin::device_detail::*; 10 | use crate::thin::superblock::*; 11 | 12 | //--------------------------------- 13 | 14 | type ArcEngine = Arc; 15 | 16 | fn read_by_thin_id(engine: &ArcEngine, root: u64, thin_id: u32) -> Result 17 | where 18 | V: Unpack + Clone, 19 | { 20 | let lookup_result = btree_lookup::(engine.as_ref(), root, thin_id as u64)?; 21 | 22 | if let Some(d) = lookup_result { 23 | Ok(d.clone()) 24 | } else { 25 | Err(anyhow!("couldn't find thin device with id {}", thin_id)) 26 | } 27 | } 28 | 29 | fn read_device_detail(engine: &ArcEngine, details_root: u64, thin_id: u32) -> Result { 30 | read_by_thin_id(engine, details_root, thin_id) 31 | } 32 | 33 | fn read_mapping_root( 34 | engine: &ArcEngine, 35 | mappings_top_level_root: u64, 36 | thin_id: u32, 37 | ) -> Result { 38 | read_by_thin_id(engine, mappings_top_level_root, thin_id) 39 | } 40 | 41 | pub struct ThinIterator { 42 | pub thin_id: u32, 43 | pub data_block_size: u64, 44 | pub mappings: BTreeIterator, 45 | pub mapped_blocks: u64, 46 | } 47 | 48 | impl ThinIterator { 49 | pub fn new(engine: &ArcEngine, thin_id: u32) -> Result { 50 | let sb = read_superblock_snap(engine.as_ref())?; 51 | let details = read_device_detail(engine, sb.details_root, thin_id)?; 52 | let mapping_root = read_mapping_root(engine, sb.mapping_root, thin_id)?; 53 | let mappings = BTreeIterator::new(engine.clone(), mapping_root)?; 54 | 55 | Ok(Self { 56 | thin_id, 57 | data_block_size: sb.data_block_size as u64, 58 | mappings, 59 | mapped_blocks: details.mapped_blocks, 60 | }) 61 | } 62 | } 63 | 64 | //--------------------------------- 65 | -------------------------------------------------------------------------------- /src/thin/migrate/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod base; 2 | pub mod devices; 3 | pub mod metadata; 4 | pub mod stream; 5 | 6 | pub use base::*; 7 | -------------------------------------------------------------------------------- /src/thin/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod block_time; 2 | pub mod check; 3 | pub mod delta; 4 | pub mod delta_visitor; 5 | pub mod device_detail; 6 | pub mod dump; 7 | pub mod human_readable_format; 8 | pub mod ir; 9 | pub mod ls; 10 | pub mod metadata; 11 | pub mod metadata_repair; 12 | pub mod metadata_size; 13 | pub mod migrate; 14 | pub mod repair; 15 | pub mod restore; 16 | pub mod rmap; 17 | pub mod runs; 18 | pub mod shrink; 19 | pub mod superblock; 20 | pub mod trim; 21 | pub mod xml; 22 | 23 | #[cfg(feature = "devtools")] 24 | pub mod metadata_generator; 25 | 26 | #[cfg(feature = "devtools")] 27 | pub mod damage_generator; 28 | 29 | #[cfg(feature = "devtools")] 30 | pub mod stat; 31 | -------------------------------------------------------------------------------- /src/thin/repair.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::path::Path; 3 | use std::sync::Arc; 4 | 5 | use crate::commands::engine::*; 6 | use crate::io_engine::*; 7 | use crate::pdata::space_map::metadata::*; 8 | use crate::report::*; 9 | use crate::thin::dump::*; 10 | use crate::thin::metadata::*; 11 | use crate::thin::metadata_repair::*; 12 | use crate::thin::restore::*; 13 | use crate::thin::superblock::*; 14 | use crate::write_batcher::*; 15 | 16 | //------------------------------------------ 17 | 18 | pub struct ThinRepairOptions<'a> { 19 | pub input: &'a Path, 20 | pub output: &'a Path, 21 | pub engine_opts: EngineOptions, 22 | pub report: Arc, 23 | pub overrides: SuperblockOverrides, 24 | } 25 | 26 | struct Context { 27 | report: Arc, 28 | engine_in: Arc, 29 | engine_out: Arc, 30 | } 31 | 32 | fn new_context(opts: &ThinRepairOptions) -> Result { 33 | let engine_in = EngineBuilder::new(opts.input, &opts.engine_opts).build()?; 34 | let engine_out = EngineBuilder::new(opts.output, &opts.engine_opts) 35 | .write(true) 36 | .build()?; 37 | 38 | Ok(Context { 39 | report: opts.report.clone(), 40 | engine_in, 41 | engine_out, 42 | }) 43 | } 44 | 45 | //------------------------------------------ 46 | 47 | pub fn repair(opts: ThinRepairOptions) -> Result<()> { 48 | let ctx = new_context(&opts)?; 49 | 50 | let sb = read_or_rebuild_superblock( 51 | ctx.engine_in.clone(), 52 | ctx.report.clone(), 53 | SUPERBLOCK_LOCATION, 54 | &opts.overrides, 55 | )?; 56 | let md = build_metadata(ctx.engine_in.clone(), &sb)?; 57 | let md = optimise_metadata(md)?; 58 | 59 | let sm = core_metadata_sm(ctx.engine_out.get_nr_blocks(), u32::MAX); 60 | let batch_size = ctx.engine_out.get_batch_size(); 61 | let mut w = WriteBatcher::new(ctx.engine_out, sm.clone(), batch_size); 62 | let mut restorer = Restorer::new(&mut w, ctx.report); 63 | 64 | dump_metadata(ctx.engine_in, &mut restorer, &sb, &md) 65 | } 66 | 67 | //------------------------------------------ 68 | -------------------------------------------------------------------------------- /src/utils/hashvec.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::vec::Vec; 3 | 4 | //------------------------------------------ 5 | 6 | /// A HashVec is an associative array indexed by a HashMap. 7 | #[derive(Debug)] 8 | pub struct HashVec { 9 | map: HashMap, // TODO: parameterize the index type 10 | entries: Vec, 11 | } 12 | 13 | impl Default for HashVec { 14 | fn default() -> HashVec { 15 | HashVec::new() 16 | } 17 | } 18 | 19 | impl HashVec { 20 | pub fn new() -> HashVec { 21 | HashVec:: { 22 | map: HashMap::new(), 23 | entries: Vec::new(), 24 | } 25 | } 26 | 27 | pub fn with_capacity(capacity: u32) -> HashVec { 28 | HashVec:: { 29 | map: HashMap::with_capacity(capacity as usize), 30 | entries: Vec::with_capacity(capacity as usize), 31 | } 32 | } 33 | 34 | pub fn insert(&mut self, index: u32, value: T) { 35 | self.map 36 | .entry(index) 37 | .and_modify(|e| { 38 | self.entries[*e as usize] = value.clone(); 39 | }) 40 | .or_insert_with(|| { 41 | self.entries.push(value); 42 | (self.entries.len() - 1) as u32 43 | }); 44 | } 45 | 46 | pub fn get(&self, index: u32) -> Option<&T> { 47 | self.map.get(&index).map(|i| &self.entries[*i as usize]) 48 | } 49 | 50 | pub fn len(&self) -> usize { 51 | self.entries.len() 52 | } 53 | 54 | pub fn is_empty(&self) -> bool { 55 | self.entries.is_empty() 56 | } 57 | 58 | pub fn reserve(&mut self, additional: u32) { 59 | self.map.reserve(additional as usize); 60 | self.entries.reserve(additional as usize); 61 | } 62 | 63 | pub fn values(&self) -> std::slice::Iter { 64 | self.entries.iter() 65 | } 66 | } 67 | 68 | //------------------------------------------ 69 | -------------------------------------------------------------------------------- /src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod hashvec; 2 | -------------------------------------------------------------------------------- /src/version.rs: -------------------------------------------------------------------------------- 1 | use clap::ArgMatches; 2 | use std::io::Write; 3 | 4 | //------------------------------------------ 5 | 6 | #[macro_export] 7 | macro_rules! tools_version { 8 | () => { 9 | env!("CARGO_PKG_VERSION") 10 | }; 11 | } 12 | 13 | pub fn version_args(cmd: clap::Command) -> clap::Command { 14 | use clap::Arg; 15 | 16 | cmd.arg( 17 | Arg::new("VERSION") 18 | .help("Print version") 19 | .short('V') 20 | .long("version") 21 | .exclusive(true) 22 | .action(clap::ArgAction::SetTrue), 23 | ) 24 | } 25 | 26 | pub fn display_version(matches: &ArgMatches) { 27 | if matches.get_flag("VERSION") { 28 | let mut stdout = std::io::stdout(); 29 | // ignore broken pipe errors 30 | let _ = stdout.write_all(tools_version!().as_bytes()); 31 | let _ = stdout.write_all(b"\n"); 32 | let _ = stdout.flush(); 33 | 34 | std::process::exit(0); 35 | } 36 | } 37 | 38 | //------------------------------------------ 39 | -------------------------------------------------------------------------------- /src/xml.rs: -------------------------------------------------------------------------------- 1 | use anyhow::anyhow; 2 | use quick_xml::events::attributes::Attribute; 3 | use quick_xml::name::QName; 4 | use std::borrow::Cow; 5 | use std::fmt::Display; 6 | 7 | //------------------------------------------ 8 | 9 | // FIXME: nasty unwraps 10 | pub fn string_val(kv: &Attribute) -> anyhow::Result { 11 | kv.unescape_value() 12 | .map_or_else(|e| Err(e.into()), |s| Ok(s.as_ref().to_string())) 13 | } 14 | 15 | fn parse_val(kv: &Attribute) -> anyhow::Result 16 | where 17 | T: std::str::FromStr, 18 | ::Err: std::error::Error + Send + Sync + 'static, 19 | { 20 | std::str::from_utf8(kv.value.as_ref()) 21 | .map_or_else(|e| Err(e.into()), |s| s.parse::().map_err(|e| e.into())) 22 | } 23 | 24 | pub fn u64_val(kv: &Attribute) -> anyhow::Result { 25 | parse_val::(kv) 26 | } 27 | 28 | pub fn u32_val(kv: &Attribute) -> anyhow::Result { 29 | parse_val::(kv) 30 | } 31 | 32 | pub fn bool_val(kv: &Attribute) -> anyhow::Result { 33 | parse_val::(kv) 34 | } 35 | 36 | pub fn bad_attr(tag: &str, attr: &[u8]) -> anyhow::Result { 37 | Err(anyhow!( 38 | "unknown attribute {}in tag '{}'", 39 | std::str::from_utf8(attr) 40 | .map(|s| format!("'{}' ", s)) 41 | .unwrap_or_default(), 42 | tag 43 | )) 44 | } 45 | 46 | pub fn check_attr(tag: &str, name: &str, maybe_v: Option) -> anyhow::Result { 47 | match maybe_v { 48 | None => missing_attr(tag, name), 49 | Some(v) => Ok(v), 50 | } 51 | } 52 | 53 | fn missing_attr(tag: &str, attr: &str) -> anyhow::Result { 54 | let msg = format!("missing attribute '{}' for tag '{}", attr, tag); 55 | Err(anyhow!(msg)) 56 | } 57 | 58 | pub fn mk_attr(key: &[u8], value: T) -> Attribute { 59 | Attribute { 60 | key: QName(key), 61 | value: mk_attr_(value), 62 | } 63 | } 64 | 65 | fn mk_attr_<'a, T: Display>(n: T) -> Cow<'a, [u8]> { 66 | let str = format!("{}", n); 67 | Cow::Owned(str.into_bytes()) 68 | } 69 | 70 | //------------------------------------------ 71 | -------------------------------------------------------------------------------- /tests/cache_dump.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | 3 | mod common; 4 | 5 | use common::cache::*; 6 | use common::common_args::*; 7 | use common::fixture::*; 8 | use common::input_arg::*; 9 | use common::process::*; 10 | use common::program::*; 11 | use common::target::*; 12 | use common::test_dir::*; 13 | 14 | //------------------------------------------ 15 | 16 | const USAGE: &str = "Dump the cache metadata to stdout in XML format 17 | 18 | Usage: cache_dump [OPTIONS] 19 | 20 | Arguments: 21 | Specify the input device to dump 22 | 23 | Options: 24 | -h, --help Print help 25 | -o, --output Specify the output file rather than stdout 26 | -r, --repair Repair the metadata whilst dumping it 27 | -V, --version Print version"; 28 | 29 | //------------------------------------------ 30 | 31 | struct CacheDump; 32 | 33 | impl<'a> Program<'a> for CacheDump { 34 | fn name() -> &'a str { 35 | "cache_dump" 36 | } 37 | 38 | fn cmd(args: I) -> Command 39 | where 40 | I: IntoIterator, 41 | I::Item: Into, 42 | { 43 | cache_dump_cmd(args) 44 | } 45 | 46 | fn usage() -> &'a str { 47 | USAGE 48 | } 49 | 50 | fn arg_type() -> ArgType { 51 | ArgType::InputArg 52 | } 53 | 54 | fn bad_option_hint(option: &str) -> String { 55 | msg::bad_option_hint(option) 56 | } 57 | } 58 | 59 | impl<'a> InputProgram<'a> for CacheDump { 60 | fn mk_valid_input(td: &mut TestDir) -> Result { 61 | mk_valid_md(td) 62 | } 63 | 64 | fn file_not_found() -> &'a str { 65 | msg::FILE_NOT_FOUND 66 | } 67 | 68 | fn missing_input_arg() -> &'a str { 69 | msg::MISSING_INPUT_ARG 70 | } 71 | 72 | fn corrupted_input() -> &'a str { 73 | msg::BAD_SUPERBLOCK 74 | } 75 | } 76 | 77 | //------------------------------------------ 78 | 79 | test_accepts_help!(CacheDump); 80 | test_accepts_version!(CacheDump); 81 | test_rejects_bad_option!(CacheDump); 82 | 83 | test_missing_input_arg!(CacheDump); 84 | test_input_file_not_found!(CacheDump); 85 | test_input_cannot_be_a_directory!(CacheDump); 86 | test_unreadable_input_file!(CacheDump); 87 | 88 | test_readonly_input_file!(CacheDump); 89 | 90 | //------------------------------------------ 91 | 92 | // TODO: share with thin_dump 93 | #[test] 94 | fn dump_restore_cycle() -> Result<()> { 95 | let mut td = TestDir::new()?; 96 | let md = mk_valid_md(&mut td)?; 97 | let output = run_ok_raw(cache_dump_cmd(args![&md]))?; 98 | 99 | let xml = td.mk_path("meta.xml"); 100 | write_file(&xml, &output.stdout)?; 101 | 102 | let md2 = mk_zeroed_md(&mut td)?; 103 | run_ok(cache_restore_cmd(args!["-i", &xml, "-o", &md2]))?; 104 | 105 | let output2 = run_ok_raw(cache_dump_cmd(args![&md2]))?; 106 | assert_eq!(output.stdout, output2.stdout); 107 | 108 | Ok(()) 109 | } 110 | 111 | //------------------------------------------ 112 | // test no stderr on broken pipe errors 113 | 114 | #[test] 115 | fn no_stderr_on_broken_pipe_xml() -> Result<()> { 116 | common::piping::test_no_stderr_on_broken_pipe::(mk_valid_md, &[]) 117 | } 118 | 119 | #[test] 120 | fn no_stderr_on_broken_fifo_xml() -> Result<()> { 121 | common::piping::test_no_stderr_on_broken_fifo::(mk_valid_md, &[]) 122 | } 123 | 124 | //------------------------------------------ 125 | -------------------------------------------------------------------------------- /tests/cache_repair.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | 3 | mod common; 4 | 5 | use common::cache::*; 6 | use common::common_args::*; 7 | use common::fixture::*; 8 | use common::input_arg::*; 9 | use common::output_option::*; 10 | use common::program::*; 11 | use common::target::*; 12 | use common::test_dir::*; 13 | 14 | //------------------------------------------ 15 | 16 | const USAGE: &str = "Repair binary cache metadata, and write it to a different device or file 17 | 18 | Usage: cache_repair [OPTIONS] --input --output 19 | 20 | Options: 21 | -h, --help Print help 22 | -i, --input Specify the input device 23 | -o, --output Specify the output device 24 | -q, --quiet Suppress output messages, return only exit code. 25 | -V, --version Print version"; 26 | 27 | //----------------------------------------- 28 | 29 | struct CacheRepair; 30 | 31 | impl<'a> Program<'a> for CacheRepair { 32 | fn name() -> &'a str { 33 | "cache_repair" 34 | } 35 | 36 | fn cmd(args: I) -> Command 37 | where 38 | I: IntoIterator, 39 | I::Item: Into, 40 | { 41 | cache_repair_cmd(args) 42 | } 43 | 44 | fn usage() -> &'a str { 45 | USAGE 46 | } 47 | 48 | fn arg_type() -> ArgType { 49 | ArgType::IoOptions 50 | } 51 | 52 | fn bad_option_hint(option: &str) -> String { 53 | msg::bad_option_hint(option) 54 | } 55 | } 56 | 57 | impl<'a> InputProgram<'a> for CacheRepair { 58 | fn mk_valid_input(td: &mut TestDir) -> Result { 59 | mk_valid_md(td) 60 | } 61 | 62 | fn file_not_found() -> &'a str { 63 | msg::FILE_NOT_FOUND 64 | } 65 | 66 | fn missing_input_arg() -> &'a str { 67 | msg::MISSING_INPUT_ARG 68 | } 69 | 70 | fn corrupted_input() -> &'a str { 71 | "bad checksum in superblock" 72 | } 73 | } 74 | 75 | impl<'a> OutputProgram<'a> for CacheRepair { 76 | fn missing_output_arg() -> &'a str { 77 | msg::MISSING_OUTPUT_ARG 78 | } 79 | } 80 | 81 | impl<'a> MetadataWriter<'a> for CacheRepair { 82 | fn file_not_found() -> &'a str { 83 | msg::FILE_NOT_FOUND 84 | } 85 | } 86 | 87 | //----------------------------------------- 88 | 89 | test_accepts_help!(CacheRepair); 90 | test_accepts_version!(CacheRepair); 91 | test_rejects_bad_option!(CacheRepair); 92 | 93 | test_input_file_not_found!(CacheRepair); 94 | test_input_cannot_be_a_directory!(CacheRepair); 95 | test_corrupted_input_data!(CacheRepair); 96 | 97 | test_missing_output_option!(CacheRepair); 98 | 99 | test_readonly_input_file!(CacheRepair); 100 | 101 | //----------------------------------------- 102 | // accepts empty argument 103 | 104 | #[test] 105 | fn accepts_empty_argument() -> Result<()> { 106 | let mut td = TestDir::new()?; 107 | let input = mk_valid_md(&mut td)?; 108 | let output = mk_zeroed_md(&mut td)?; 109 | run_ok(cache_repair_cmd(args!["-i", &input, "-o", &output, ""]))?; 110 | Ok(()) 111 | } 112 | 113 | //----------------------------------------- 114 | -------------------------------------------------------------------------------- /tests/common/cache.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::path::{Path, PathBuf}; 3 | use thinp::cache::metadata_generator::CacheGenerator; 4 | use thinp::file_utils; 5 | 6 | use thinp::io_engine::SyncIoEngine; 7 | 8 | use crate::args; 9 | use crate::common::cache_xml_generator::write_xml; 10 | use crate::common::process::*; 11 | use crate::common::target::*; 12 | use crate::common::test_dir::TestDir; 13 | 14 | //----------------------------------------------- 15 | 16 | pub fn mk_valid_xml(td: &mut TestDir) -> Result { 17 | let xml = td.mk_path("meta.xml"); 18 | // bs, cblocks, oblocks, res, dirty, version, hotspot_size 19 | let mut gen = CacheGenerator::new(512, 128, 1024, 80, 50, 2, 16); 20 | write_xml(&xml, &mut gen)?; 21 | Ok(xml) 22 | } 23 | 24 | pub fn mk_valid_md(td: &mut TestDir) -> Result { 25 | let xml = td.mk_path("meta.xml"); 26 | let md = td.mk_path("meta.bin"); 27 | 28 | let mut gen = CacheGenerator::new(512, 4096, 32768, 80, 50, 2, 16); 29 | write_xml(&xml, &mut gen)?; 30 | 31 | let _file = file_utils::create_sized_file(&md, 4096 * 4096); 32 | run_ok(cache_restore_cmd(args!["-i", &xml, "-o", &md]))?; 33 | 34 | Ok(md) 35 | } 36 | 37 | //----------------------------------------------- 38 | 39 | pub fn generate_metadata_leaks( 40 | md: &Path, 41 | nr_blocks: u64, 42 | expected: u32, 43 | actual: u32, 44 | ) -> Result<()> { 45 | let nr_blocks_str = nr_blocks.to_string(); 46 | let expected_str = expected.to_string(); 47 | let actual_str = actual.to_string(); 48 | let args = args![ 49 | "-o", 50 | &md, 51 | "--create-metadata-leaks", 52 | "--nr-blocks", 53 | &nr_blocks_str, 54 | "--expected", 55 | &expected_str, 56 | "--actual", 57 | &actual_str 58 | ]; 59 | run_ok(cache_generate_damage_cmd(args))?; 60 | 61 | Ok(()) 62 | } 63 | 64 | //----------------------------------------------- 65 | 66 | pub fn get_clean_shutdown(md: &Path) -> Result { 67 | use thinp::cache::superblock::*; 68 | 69 | let engine = SyncIoEngine::new(md, false)?; 70 | let sb = read_superblock(&engine, SUPERBLOCK_LOCATION)?; 71 | Ok(sb.flags.clean_shutdown) 72 | } 73 | 74 | pub fn get_needs_check(md: &Path) -> Result { 75 | use thinp::cache::superblock::*; 76 | 77 | let engine = SyncIoEngine::new(md, false)?; 78 | let sb = read_superblock(&engine, SUPERBLOCK_LOCATION)?; 79 | Ok(sb.flags.needs_check) 80 | } 81 | 82 | pub fn unset_clean_shutdown(md: &Path) -> Result<()> { 83 | let args = args!["-o", &md, "--set-clean-shutdown=false"]; 84 | run_ok(cache_generate_metadata_cmd(args))?; 85 | Ok(()) 86 | } 87 | 88 | pub fn set_needs_check(md: &Path) -> Result<()> { 89 | let args = args!["-o", &md, "--set-needs-check"]; 90 | run_ok(cache_generate_metadata_cmd(args))?; 91 | Ok(()) 92 | } 93 | 94 | //----------------------------------------------- 95 | -------------------------------------------------------------------------------- /tests/common/cache_xml_generator.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::fs::OpenOptions; 3 | use std::path::Path; 4 | use thinp::cache::metadata_generator::MetadataGenerator; 5 | use thinp::cache::xml; 6 | 7 | //------------------------------------------ 8 | 9 | pub fn write_xml(path: &Path, g: &mut dyn MetadataGenerator) -> Result<()> { 10 | let xml_out = OpenOptions::new() 11 | .read(false) 12 | .write(true) 13 | .create(true) 14 | .truncate(true) 15 | .open(path)?; 16 | let mut w = xml::XmlWriter::new(xml_out); 17 | 18 | g.generate_metadata(&mut w) 19 | } 20 | 21 | //------------------------------------------ 22 | -------------------------------------------------------------------------------- /tests/common/common_args.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | 3 | use thinp::tools_version; 4 | 5 | use crate::args; 6 | use crate::common::process::*; 7 | use crate::common::program::*; 8 | 9 | //------------------------------------------ 10 | // help 11 | 12 | pub fn test_help_short<'a, P>() -> Result<()> 13 | where 14 | P: Program<'a>, 15 | { 16 | let stdout = run_ok(P::cmd(args!["-h"]))?; 17 | assert_eq!(stdout, P::usage()); 18 | Ok(()) 19 | } 20 | 21 | pub fn test_help_long<'a, P>() -> Result<()> 22 | where 23 | P: Program<'a>, 24 | { 25 | let stdout = run_ok(P::cmd(vec!["--help"]))?; 26 | assert_eq!(stdout, P::usage()); 27 | Ok(()) 28 | } 29 | 30 | #[macro_export] 31 | macro_rules! test_accepts_help { 32 | ($program: ident) => { 33 | #[test] 34 | fn accepts_h() -> Result<()> { 35 | test_help_short::<$program>() 36 | } 37 | 38 | #[test] 39 | fn accepts_help() -> Result<()> { 40 | test_help_long::<$program>() 41 | } 42 | }; 43 | } 44 | 45 | //------------------------------------------ 46 | // version 47 | 48 | pub fn test_version_short<'a, P>() -> Result<()> 49 | where 50 | P: Program<'a>, 51 | { 52 | let stdout = run_ok(P::cmd(args!["-V"]))?; 53 | assert!(stdout.starts_with(tools_version!())); 54 | Ok(()) 55 | } 56 | 57 | pub fn test_version_long<'a, P>() -> Result<()> 58 | where 59 | P: Program<'a>, 60 | { 61 | let stdout = run_ok(P::cmd(args!["--version"]))?; 62 | assert!(stdout.starts_with(tools_version!())); 63 | Ok(()) 64 | } 65 | 66 | #[macro_export] 67 | macro_rules! test_accepts_version { 68 | ($program: ident) => { 69 | #[test] 70 | fn accepts_v() -> Result<()> { 71 | test_version_short::<$program>() 72 | } 73 | 74 | #[test] 75 | fn accepts_version() -> Result<()> { 76 | test_version_long::<$program>() 77 | } 78 | }; 79 | } 80 | 81 | //------------------------------------------ 82 | 83 | pub fn test_rejects_bad_option<'a, P>() -> Result<()> 84 | where 85 | P: Program<'a>, 86 | { 87 | let option = "--hedgehogs-only"; 88 | let stderr = run_fail(P::cmd(args![option]))?; 89 | assert!(stderr.contains(&P::bad_option_hint(option))); 90 | Ok(()) 91 | } 92 | 93 | #[macro_export] 94 | macro_rules! test_rejects_bad_option { 95 | ($program: ident) => { 96 | #[test] 97 | fn rejects_bad_option() -> Result<()> { 98 | test_rejects_bad_option::<$program>() 99 | } 100 | }; 101 | } 102 | 103 | //------------------------------------------ 104 | -------------------------------------------------------------------------------- /tests/common/era.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::path::PathBuf; 3 | 4 | use thinp::era::metadata_generator::CleanShutdownMeta; 5 | use thinp::file_utils; 6 | 7 | use crate::args; 8 | use crate::common::era_xml_generator::write_xml; 9 | use crate::common::process::*; 10 | use crate::common::target::*; 11 | use crate::common::test_dir::TestDir; 12 | 13 | //----------------------------------------------- 14 | 15 | pub fn mk_valid_xml(td: &mut TestDir) -> Result { 16 | let xml = td.mk_path("meta.xml"); 17 | let mut gen = CleanShutdownMeta::new(128, 512, 32, 4); // bs, nr_blocks, era, nr_wsets 18 | write_xml(&xml, &mut gen)?; 19 | Ok(xml) 20 | } 21 | 22 | pub fn mk_valid_md(td: &mut TestDir) -> Result { 23 | let xml = td.mk_path("meta.xml"); 24 | let md = td.mk_path("meta.bin"); 25 | 26 | let mut gen = CleanShutdownMeta::new(128, 512, 32, 4); 27 | write_xml(&xml, &mut gen)?; 28 | 29 | let _file = file_utils::create_sized_file(&md, 4096 * 4096); 30 | run_ok(era_restore_cmd(args!["-i", &xml, "-o", &md]))?; 31 | 32 | Ok(md) 33 | } 34 | 35 | //----------------------------------------------- 36 | -------------------------------------------------------------------------------- /tests/common/era_xml_generator.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::fs::OpenOptions; 3 | use std::path::Path; 4 | use thinp::era::metadata_generator::MetadataGenerator; 5 | use thinp::era::xml; 6 | 7 | //------------------------------------------ 8 | 9 | pub fn write_xml(path: &Path, g: &mut dyn MetadataGenerator) -> Result<()> { 10 | let xml_out = OpenOptions::new() 11 | .read(false) 12 | .write(true) 13 | .create(true) 14 | .truncate(true) 15 | .open(path)?; 16 | let mut w = xml::XmlWriter::new(xml_out, false); 17 | 18 | g.generate_metadata(&mut w) 19 | } 20 | 21 | //------------------------------------------ 22 | -------------------------------------------------------------------------------- /tests/common/fixture.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::fs::OpenOptions; 3 | use std::io::{Read, Write}; 4 | use std::path::{Path, PathBuf}; 5 | 6 | use thinp::file_utils; 7 | 8 | use crate::common::test_dir::TestDir; 9 | 10 | //------------------------------------------ 11 | 12 | pub fn mk_zeroed_md(td: &mut TestDir) -> Result { 13 | let md = td.mk_path("meta.bin"); 14 | eprintln!("path = {:?}", md); 15 | let _file = file_utils::create_sized_file(&md, 1024 * 1024 * 16); 16 | Ok(md) 17 | } 18 | 19 | pub fn mk_zeroed_md_sized(td: &mut TestDir, nr_bytes: u64) -> Result { 20 | let md = td.mk_path("meta.bin"); 21 | eprintln!("path = {:?}", md); 22 | let _file = file_utils::create_sized_file(&md, nr_bytes); 23 | Ok(md) 24 | } 25 | 26 | pub fn damage_superblock(path: &Path) -> Result<()> { 27 | let mut output = OpenOptions::new().read(false).write(true).open(path)?; 28 | let buf = [0u8; 512]; 29 | output.write_all(&buf)?; 30 | Ok(()) 31 | } 32 | 33 | //------------------------------------------ 34 | 35 | pub fn sha256sum(md: &Path) -> Result { 36 | let output = duct::cmd!("sha256sum", "-b", &md).stdout_capture().run()?; 37 | let csum = std::str::from_utf8(&output.stdout[0..])?.to_string(); 38 | let csum = csum.split_ascii_whitespace().next().unwrap().to_string(); 39 | Ok(csum) 40 | } 41 | 42 | // This checksums the file before and after the thunk is run to 43 | // ensure it is unchanged. 44 | pub fn ensure_untouched(p: &Path, thunk: F) -> Result<()> 45 | where 46 | F: Fn() -> Result<()>, 47 | { 48 | let csum = sha256sum(p)?; 49 | thunk()?; 50 | assert_eq!(csum, sha256sum(p)?); 51 | Ok(()) 52 | } 53 | 54 | pub fn superblock_all_zeroes(path: &Path) -> Result { 55 | let mut input = OpenOptions::new().read(true).write(false).open(path)?; 56 | let mut buf = vec![0; 4096]; 57 | input.read_exact(&mut buf[0..])?; 58 | for b in buf { 59 | if b != 0 { 60 | return Ok(false); 61 | } 62 | } 63 | 64 | Ok(true) 65 | } 66 | 67 | pub fn ensure_superblock_zeroed(p: &Path, thunk: F) -> Result<()> 68 | where 69 | F: Fn() -> Result<()>, 70 | { 71 | thunk()?; 72 | assert!(superblock_all_zeroes(p)?); 73 | Ok(()) 74 | } 75 | 76 | //------------------------------------------ 77 | 78 | pub fn write_file(p: &Path, buf: &[u8]) -> Result<()> { 79 | let mut output = std::fs::File::create(p)?; 80 | output.write_all(buf)?; 81 | Ok(()) 82 | } 83 | 84 | //------------------------------------------ 85 | -------------------------------------------------------------------------------- /tests/common/mod.rs: -------------------------------------------------------------------------------- 1 | // suppress all the false alarms by cargo test 2 | // https://github.com/rust-lang/rust/issues/46379 3 | #![allow(dead_code)] 4 | 5 | pub mod cache; 6 | pub mod cache_xml_generator; 7 | pub mod common_args; 8 | pub mod era; 9 | pub mod era_xml_generator; 10 | pub mod fixture; 11 | pub mod input_arg; 12 | pub mod output_option; 13 | pub mod piping; 14 | pub mod process; 15 | pub mod program; 16 | pub mod target; 17 | pub mod test_dir; 18 | pub mod thin; 19 | pub mod thin_xml_generator; 20 | -------------------------------------------------------------------------------- /tests/common/program.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::path::PathBuf; 3 | 4 | pub use crate::common::process::*; 5 | use crate::common::test_dir::TestDir; 6 | 7 | //------------------------------------------ 8 | 9 | pub enum ArgType { 10 | InputArg, 11 | IoOptions, 12 | } 13 | 14 | pub trait Program<'a> { 15 | fn name() -> &'a str; 16 | fn cmd(args: I) -> Command 17 | where 18 | I: IntoIterator, 19 | I::Item: Into; 20 | fn usage() -> &'a str; 21 | fn arg_type() -> ArgType; 22 | fn required_args() -> &'a [&'a str] { 23 | &[] 24 | } 25 | 26 | // error messages 27 | fn bad_option_hint(option: &str) -> String; 28 | } 29 | 30 | pub trait InputProgram<'a>: Program<'a> { 31 | fn mk_valid_input(td: &mut TestDir) -> Result; 32 | 33 | // error messages 34 | fn missing_input_arg() -> &'a str; 35 | fn file_not_found() -> &'a str; 36 | fn corrupted_input() -> &'a str; 37 | } 38 | 39 | pub trait MetadataReader<'a>: InputProgram<'a> {} 40 | 41 | pub trait OutputProgram<'a>: InputProgram<'a> { 42 | // error messages 43 | fn missing_output_arg() -> &'a str; 44 | } 45 | 46 | // programs that write existed files 47 | pub trait MetadataWriter<'a>: OutputProgram<'a> { 48 | // error messages 49 | fn file_not_found() -> &'a str; 50 | } 51 | 52 | // programs that create output files (O_CREAT) 53 | pub trait MetadataCreator<'a>: OutputProgram<'a> {} 54 | 55 | //------------------------------------------ 56 | -------------------------------------------------------------------------------- /tests/common/test_dir.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{anyhow, Result}; 2 | use rand::prelude::*; 3 | use std::fs; 4 | use std::path::PathBuf; 5 | 6 | //--------------------------------------- 7 | 8 | pub struct TestDir { 9 | dir: PathBuf, 10 | files: Vec, 11 | clean_up: bool, 12 | file_count: usize, 13 | } 14 | 15 | fn mk_dir(prefix: &str) -> Result { 16 | for _n in 0..100 { 17 | let mut p = PathBuf::new(); 18 | let nr = rand::thread_rng().gen_range(1000000..9999999); 19 | p.push(format!("./{}_{}", prefix, nr)); 20 | if let Ok(()) = fs::create_dir(&p) { 21 | return Ok(p); 22 | } 23 | } 24 | 25 | Err(anyhow!("Couldn't create test directory")) 26 | } 27 | 28 | impl TestDir { 29 | pub fn new() -> Result { 30 | let dir = mk_dir("test_fixture")?; 31 | Ok(TestDir { 32 | dir, 33 | files: Vec::new(), 34 | clean_up: true, 35 | file_count: 0, 36 | }) 37 | } 38 | 39 | pub fn dont_clean_up(&mut self) { 40 | self.clean_up = false; 41 | } 42 | 43 | pub fn mk_path(&mut self, file: &str) -> PathBuf { 44 | let mut p = PathBuf::new(); 45 | p.push(&self.dir); 46 | p.push(PathBuf::from(format!("{:02}_{}", self.file_count, file))); 47 | self.files.push(p.clone()); 48 | self.file_count += 1; 49 | p 50 | } 51 | } 52 | 53 | impl Drop for TestDir { 54 | fn drop(&mut self) { 55 | if !std::thread::panicking() && self.clean_up { 56 | while let Some(f) = self.files.pop() { 57 | // It's not guaranteed that the path generated was actually created. 58 | let _ignore = fs::remove_file(f); 59 | } 60 | fs::remove_dir(&self.dir).expect("couldn't remove test directory"); 61 | } else { 62 | eprintln!("leaving test directory: {:?}", self.dir); 63 | } 64 | } 65 | } 66 | 67 | //--------------------------------------- 68 | -------------------------------------------------------------------------------- /tests/era_invalidate.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | 3 | mod common; 4 | 5 | use common::common_args::*; 6 | use common::era::*; 7 | use common::input_arg::*; 8 | use common::program::*; 9 | use common::target::*; 10 | use common::test_dir::*; 11 | 12 | //------------------------------------------ 13 | 14 | const USAGE: &str = "List blocks that may have changed since a given era 15 | 16 | Usage: era_invalidate [OPTIONS] --written-since 17 | 18 | Arguments: 19 | Specify the input device 20 | 21 | Options: 22 | -h, --help 23 | Print help 24 | --metadata-snapshot 25 | Use the metadata snapshot rather than the current superblock 26 | -o, --output 27 | Specify the output file rather than stdout 28 | -V, --version 29 | Print version 30 | --written-since 31 | Blocks written since the given era will be listed"; 32 | 33 | //------------------------------------------ 34 | 35 | struct EraInvalidate; 36 | 37 | impl<'a> Program<'a> for EraInvalidate { 38 | fn name() -> &'a str { 39 | "era_invalidate" 40 | } 41 | 42 | fn cmd(args: I) -> Command 43 | where 44 | I: IntoIterator, 45 | I::Item: Into, 46 | { 47 | era_invalidate_cmd(args) 48 | } 49 | 50 | fn usage() -> &'a str { 51 | USAGE 52 | } 53 | 54 | fn arg_type() -> ArgType { 55 | ArgType::InputArg 56 | } 57 | 58 | fn required_args() -> &'a [&'a str] { 59 | &["--written-since", "0"] 60 | } 61 | 62 | fn bad_option_hint(option: &str) -> String { 63 | msg::bad_option_hint(option) 64 | } 65 | } 66 | 67 | impl<'a> InputProgram<'a> for EraInvalidate { 68 | fn mk_valid_input(td: &mut TestDir) -> Result { 69 | mk_valid_md(td) 70 | } 71 | 72 | fn file_not_found() -> &'a str { 73 | msg::FILE_NOT_FOUND 74 | } 75 | 76 | fn missing_input_arg() -> &'a str { 77 | msg::MISSING_INPUT_ARG 78 | } 79 | 80 | fn corrupted_input() -> &'a str { 81 | msg::BAD_SUPERBLOCK 82 | } 83 | } 84 | 85 | impl MetadataReader<'_> for EraInvalidate {} 86 | 87 | //------------------------------------------ 88 | 89 | test_accepts_help!(EraInvalidate); 90 | test_accepts_version!(EraInvalidate); 91 | test_rejects_bad_option!(EraInvalidate); 92 | 93 | test_missing_input_arg!(EraInvalidate); 94 | test_input_file_not_found!(EraInvalidate); 95 | test_input_cannot_be_a_directory!(EraInvalidate); 96 | test_unreadable_input_file!(EraInvalidate); 97 | test_tiny_input_file!(EraInvalidate); 98 | 99 | test_readonly_input_file!(EraInvalidate); 100 | 101 | //------------------------------------------ 102 | 103 | #[test] 104 | fn written_since_unspecified() -> Result<()> { 105 | let mut td = TestDir::new()?; 106 | let md = mk_valid_md(&mut td)?; 107 | let stderr = run_fail(era_invalidate_cmd(args![&md]))?; 108 | assert!(stderr.contains( 109 | "the following required arguments were not provided: 110 | --written-since " 111 | )); 112 | Ok(()) 113 | } 114 | 115 | //------------------------------------------ 116 | -------------------------------------------------------------------------------- /tests/testdata/corrupted_tmeta_with_metadata_snap.pack: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jthornber/thin-provisioning-tools/ee63f66ca4105c644321a9a445102a2921339ca8/tests/testdata/corrupted_tmeta_with_metadata_snap.pack -------------------------------------------------------------------------------- /tests/testdata/readme.md: -------------------------------------------------------------------------------- 1 | List of packed thin-pool metadata for tests: 2 | 3 | * tmeta.pack: A pool with a volume and few snapshots. The origin volume is 4 | randomly updated by a synthetic workload between snapshots, causes exclusive 5 | mappings within each snapshot. The maximum ref count of a mapping tree leaf 6 | is equivalent to the number of devices in this pool (11 in this case). 7 | * tmeta_with_metadata_snap.pack: Identical to `tmeta.pack` with additional 8 | metadata snapshot. 9 | * tmeta_with_corrupted_metadata_snap.pack: A small pool with one device and damaged 10 | metadata snapshot broke. 11 | * corrupted_tmeta_with_metadata_snap.pack: Data structures except that of the 12 | metadata snapshot are all corrupted. 13 | * tmeta_device_id_reuse: Two different subtrees share the same device id 14 | * tmeta_device_id_reuse_with_corrupted_thins: Same as above but the subtree in 15 | metadata snapshot broke. 16 | -------------------------------------------------------------------------------- /tests/testdata/scripts/corrupted_tmeta_with_metadata_snap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_name="corrupted_tmeta_with_metadata_snap" 4 | metadata_dump="${test_name}.bin" 5 | metadata_pack="${test_name}.pack" 6 | 7 | lvcreate vg1 --type thin-pool --thinpool tp1 --size 4g --poolmetadatasize 4m -Zn --poolmetadataspare=n 8 | lvcreate vg1 --type thin --thinpool tp1 --virtualsize 1g --name lv1 9 | dmsetup message vg1-tp1-tpool 0 "reserve_metadata_snap" 10 | lvcreate vg1 --type thin --thinpool tp1 --virtualsize 1g --name lv2 11 | lvremove vg1/lv1 -f 12 | 13 | # dump metadata 14 | lvchange -an vg1 15 | lvchange -ay vg1/tp1_tmeta -f -y 16 | dd if=/dev/mapper/vg1-tp1_tmeta of=${metadata_dump} oflag=direct 17 | lvchange -an vg1/tp1_tmeta 18 | 19 | lvremove vg1/tp1 -f 20 | 21 | # erase the subtree root of lv2 22 | m1_root=$(xxd -e -l 8 -g 8 -s 320 ${metadata_dump} | cut -d ' ' -f 2) 23 | m2_root=$(xxd -e -l 8 -g 8 -s $((0x1000*0x${m1_root}+0x800)) ${metadata_dump} | cut -d ' ' -f 2) 24 | dd if=/dev/zero of="${metadata_dump}" bs=4K count=1 seek=$((0x${m2_root})) oflag=direct conv=notrunc 25 | 26 | # locations of top-level root, details tree root, 27 | # and the bitmap root & ref count tree root for data & metadata space maps in superblock 28 | offsets=(320, 328, 80, 88, 208, 216) 29 | 30 | # erase the above blocks 31 | for off in "${offsets[@]}" 32 | do 33 | blocknr=$(xxd -e -l 8 -g 8 -s ${off} "${metadata_dump}" | cut -d ' ' -f 2) 34 | dd if=/dev/zero of="${metadata_dump}" bs=4K count=1 seek=$((0x${blocknr})) oflag=direct conv=notrunc 35 | done 36 | 37 | ../../../target/release/pdata_tools thin_metadata_pack -i "${metadata_dump}" -o "../${metadata_pack}" 38 | 39 | rm ${metadata_dump} 40 | -------------------------------------------------------------------------------- /tests/testdata/scripts/tmeta.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_name="tmeta" 4 | metadata_dump="${test_name}.bin" 5 | metadata_pack="${test_name}.pack" 6 | 7 | vg=vg1 8 | tp=tp1 9 | pool_size=4g 10 | metadata_size=64m # 64MB is the minimum size of having two bitmap blocks 11 | blocksize=64k 12 | lv_name=lv1 13 | lv_size=1g 14 | 15 | lvcreate ${vg} --type thin-pool --name ${tp} --size ${pool_size} \ 16 | --chunksize ${blocksize} --poolmetadatasize ${metadata_size} \ 17 | -Zn --poolmetadataspare=n 18 | 19 | lvcreate ${vg} --type thin --name ${lv_name} --thinpool ${tp} --virtualsize ${lv_size} 20 | fio --filename "/dev/mapper/${vg}-${lv_name}" --rw=randwrite --percentage_random=20 \ 21 | --bs ${blocksize} --randseed=32767 --randrepeat=0 \ 22 | --name test --direct=1 --output-format terse 23 | 24 | # create snapshots with some exclusive mappings 25 | for i in {1..15} 26 | do 27 | lvcreate "${vg}/${lv_name}" --snapshot --name "snap${i}" 28 | fio --filename "/dev/mapper/${vg}-${lv_name}" --rw=randwrite --percentage_random=20 \ 29 | --bs ${blocksize} --randseed=${i} --randrepeat=0 \ 30 | --name test --direct=1 --io_size 4m --output-format terse 31 | done 32 | 33 | # remove snapshots to produce holes in data space map, for thin_shrink tests 34 | lvremove "${vg}/snap4" -f 35 | lvremove "${vg}/snap5" -f 36 | lvremove "${vg}/snap6" -f 37 | lvremove "${vg}/snap10" -f 38 | lvremove "${vg}/snap13" -f 39 | 40 | lvchange -an "${vg}/${lv_name}" 41 | lvchange -an "${vg}/${tp}" 42 | 43 | # dump metadata 44 | lvchange -an "${vg}" 45 | lvchange -ay "${vg}/${tp}_tmeta" -f -y 46 | dd if="/dev/mapper/${vg}-${tp}_tmeta" of="${metadata_dump}" oflag=direct 47 | lvchange -an "${vg}/${tp}_tmeta" 48 | 49 | ../../../target/release/pdata_tools thin_metadata_pack -i "${metadata_dump}" -o "../${metadata_pack}" 50 | 51 | rm ${metadata_dump} 52 | 53 | lvremove vg1 -f 54 | -------------------------------------------------------------------------------- /tests/testdata/scripts/tmeta_device_id_reuse.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_name="tmeta_device_id_reuse" 4 | metadata_dump="${test_name}.bin" 5 | metadata_pack="${test_name}.pack" 6 | 7 | lvcreate vg1 --type thin-pool --thinpool tp1 --size 4g --poolmetadatasize 4m -Zn --poolmetadataspare=n 8 | lvcreate vg1 --type thin --thinpool tp1 --virtualsize 1g --name lv1 9 | dmsetup message vg1-tp1-tpool 0 "reserve_metadata_snap" 10 | 11 | # create another device that reuses the id of the deleted one 12 | lvremove vg1/lv1 -f 13 | lvcreate vg1 --type thin --thinpool tp1 --virtualsize 1g --name lv1 14 | 15 | # dump metadata 16 | lvchange -an vg1 17 | lvchange -ay vg1/tp1_tmeta -f -y 18 | dd if=/dev/mapper/vg1-tp1_tmeta of="${metadata_dump}" oflag=direct 19 | lvchange -an vg1/tp1_tmeta 20 | 21 | lvremove vg1/tp1 -f 22 | 23 | ../../../target/release/pdata_tools thin_metadata_pack -i "${metadata_dump}" -o "../${metadata_pack}" 24 | 25 | # erase the subtree root of the lv1 in metadata snapshot 26 | metadata_snap=$(xxd -e -l 8 -g 8 -s 56 "${metadata_dump}" | cut -d ' ' -f 2) 27 | m1_root=$(xxd -e -l 8 -g 8 -s $((0x1000*0x${metadata_snap}+320)) "${metadata_dump}" | cut -d ' ' -f 2) 28 | m2_root=$(xxd -e -l 8 -g 8 -s $((0x1000*0x${m1_root}+0x800)) ${metadata_dump} | cut -d ' ' -f 2) 29 | dd if=/dev/zero of="${metadata_dump}" bs=4K count=1 seek=$((0x${m2_root})) oflag=direct conv=notrunc 30 | 31 | ../../../target/release/pdata_tools thin_metadata_pack -i "${metadata_dump}" -o "../${test_name}_with_corrupted_thins.pack" 32 | 33 | rm ${metadata_dump} 34 | -------------------------------------------------------------------------------- /tests/testdata/scripts/tmeta_with_corrupted_metadata_snap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_name="tmeta_with_corrupted_metadata_snap" 4 | metadata_dump="${test_name}.bin" 5 | metadata_pack="${test_name}.pack" 6 | 7 | lvcreate vg1 --type thin-pool --thinpool tp1 --size 4g --poolmetadatasize 4m -Zn --poolmetadataspare=n 8 | lvcreate vg1 --type thin --thinpool tp1 --virtualsize 1g --name lv1 9 | dmsetup message vg1-tp1-tpool 0 "reserve_metadata_snap" 10 | 11 | # dump metadata 12 | lvchange -an vg1 13 | lvchange -ay vg1/tp1_tmeta -f -y 14 | dd if=/dev/mapper/vg1-tp1_tmeta of="${metadata_dump}" oflag=direct 15 | lvchange -an vg1/tp1_tmeta 16 | 17 | lvremove vg1/tp1 -f 18 | 19 | # location of metadata_snap 20 | metadata_snap=$(xxd -e -l 8 -g 8 -s 56 "${metadata_dump}" | cut -d ' ' -f 2) 21 | 22 | # erase the superblock of metadata snapshot 23 | dd if=/dev/zero of="${metadata_dump}" bs=4K count=1 seek=$((0x${metadata_snap})) oflag=direct conv=notrunc 24 | 25 | ../../../target/release/pdata_tools thin_metadata_pack -i "${metadata_dump}" -o "../${metadata_pack}" 26 | 27 | rm ${metadata_dump} 28 | -------------------------------------------------------------------------------- /tests/testdata/scripts/tmeta_with_deleted_snapshot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_name="tmeta_with_deleted_snapshot" 4 | metadata_dump="${test_name}.bin" 5 | metadata_pack="${test_name}.pack" 6 | 7 | lvcreate vg1 --type thin-pool --thinpool tp1 --size 4g --poolmetadatasize 4m -Zn --poolmetadataspare=n 8 | lvcreate vg1 --type thin --thinpool tp1 --virtualsize 1g --name lv1 9 | dmsetup message vg1-tp1-tpool 0 "reserve_metadata_snap" 10 | 11 | # let the metadata snapshot own a deleted device points to a shared root 12 | lvcreate --snapshot vg1/lv1 --name snap1 13 | lvremove vg1/lv1 -f 14 | 15 | # dump metadata 16 | lvchange -an vg1 17 | lvchange -ay vg1/tp1_tmeta -f -y 18 | dd if=/dev/mapper/vg1-tp1_tmeta of="${metadata_dump}" oflag=direct 19 | lvchange -an vg1/tp1_tmeta 20 | 21 | lvremove vg1/tp1 -f 22 | 23 | ../../../target/release/pdata_tools thin_metadata_pack -i "${metadata_dump}" -o "../${metadata_pack}" 24 | 25 | rm ${metadata_dump} 26 | -------------------------------------------------------------------------------- /tests/testdata/scripts/tmeta_with_empty_roots.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_name="tmeta_with_empty_roots" 4 | metadata_dump="${test_name}.bin" 5 | metadata_pack="${test_name}.pack" 6 | 7 | vg=vg1 8 | tp=tp1 9 | pool_size=64m 10 | metadata_size=4m 11 | blocksize=64k 12 | lv_name=lv1 13 | lv_size=16m 14 | snap_name=snap1 15 | 16 | lvcreate ${vg} --type thin-pool --name ${tp} --size ${pool_size} \ 17 | --chunksize ${blocksize} --poolmetadatasize ${metadata_size} \ 18 | -Zn --poolmetadataspare=n 19 | 20 | lvcreate ${vg} --type thin --name ${lv_name} --thinpool ${tp} --virtualsize ${lv_size} 21 | 22 | # commit a metadata transaction 23 | dmsetup status "${vg}/${tp}-tpool" 24 | 25 | # write some data ... 26 | dd if=/dev/zero of="/dev/mapper/${vg}-${lv_name}" bs=1M count=4 27 | 28 | # create a snapshot 29 | lvcreate "${vg}/${lv_name}" --snapshot --name ${snap_name} 30 | 31 | lvchange -an "${vg}/${lv_name}" 32 | lvchange -an "${vg}/${tp}" 33 | 34 | # dump metadata 35 | lvchange -an "${vg}" 36 | lvchange -ay "${vg}/${tp}_tmeta" -f -y 37 | dd if="/dev/mapper/${vg}-${tp}_tmeta" of="${metadata_dump}" oflag=direct 38 | lvchange -an "${vg}/${tp}_tmeta" 39 | 40 | ../../../target/release/pdata_tools thin_metadata_pack -i "${metadata_dump}" -o "../${metadata_pack}" 41 | 42 | rm ${metadata_dump} 43 | 44 | lvremove "${vg}/${snap_name}" 45 | lvremove "${vg}/${lv_name}" 46 | lvremove "${vg}/${tp}" 47 | -------------------------------------------------------------------------------- /tests/testdata/tmeta.pack: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jthornber/thin-provisioning-tools/ee63f66ca4105c644321a9a445102a2921339ca8/tests/testdata/tmeta.pack -------------------------------------------------------------------------------- /tests/testdata/tmeta_device_id_reuse.pack: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jthornber/thin-provisioning-tools/ee63f66ca4105c644321a9a445102a2921339ca8/tests/testdata/tmeta_device_id_reuse.pack -------------------------------------------------------------------------------- /tests/testdata/tmeta_device_id_reuse_with_corrupted_thins.pack: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jthornber/thin-provisioning-tools/ee63f66ca4105c644321a9a445102a2921339ca8/tests/testdata/tmeta_device_id_reuse_with_corrupted_thins.pack -------------------------------------------------------------------------------- /tests/testdata/tmeta_with_corrupted_metadata_snap.pack: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jthornber/thin-provisioning-tools/ee63f66ca4105c644321a9a445102a2921339ca8/tests/testdata/tmeta_with_corrupted_metadata_snap.pack -------------------------------------------------------------------------------- /tests/testdata/tmeta_with_deleted_snapshot.pack: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jthornber/thin-provisioning-tools/ee63f66ca4105c644321a9a445102a2921339ca8/tests/testdata/tmeta_with_deleted_snapshot.pack -------------------------------------------------------------------------------- /tests/testdata/tmeta_with_empty_roots.pack: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jthornber/thin-provisioning-tools/ee63f66ca4105c644321a9a445102a2921339ca8/tests/testdata/tmeta_with_empty_roots.pack -------------------------------------------------------------------------------- /tests/testdata/tmeta_with_metadata_snap.pack: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jthornber/thin-provisioning-tools/ee63f66ca4105c644321a9a445102a2921339ca8/tests/testdata/tmeta_with_metadata_snap.pack -------------------------------------------------------------------------------- /tests/thin_ls.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | 3 | mod common; 4 | 5 | use common::common_args::*; 6 | use common::input_arg::*; 7 | use common::process::*; 8 | use common::program::*; 9 | use common::target::*; 10 | use common::test_dir::*; 11 | use common::thin::*; 12 | 13 | //------------------------------------------ 14 | 15 | const USAGE: &str = "List thin volumes within a pool 16 | 17 | Usage: thin_ls [OPTIONS] 18 | 19 | Arguments: 20 | Specify the input device 21 | 22 | Options: 23 | -h, --help Print help 24 | -m, --metadata-snap Use metadata snapshot 25 | --no-headers Don't output headers 26 | -o, --format Give a comma separated list of fields to be output 27 | -V, --version Print version"; 28 | 29 | //----------------------------------------- 30 | 31 | struct ThinLs; 32 | 33 | impl<'a> Program<'a> for ThinLs { 34 | fn name() -> &'a str { 35 | "thin_ls" 36 | } 37 | 38 | fn cmd(args: I) -> Command 39 | where 40 | I: IntoIterator, 41 | I::Item: Into, 42 | { 43 | thin_ls_cmd(args) 44 | } 45 | 46 | fn usage() -> &'a str { 47 | USAGE 48 | } 49 | 50 | fn arg_type() -> ArgType { 51 | ArgType::InputArg 52 | } 53 | 54 | fn bad_option_hint(option: &str) -> String { 55 | msg::bad_option_hint(option) 56 | } 57 | } 58 | 59 | impl<'a> InputProgram<'a> for ThinLs { 60 | fn mk_valid_input(td: &mut TestDir) -> Result { 61 | mk_valid_md(td) 62 | } 63 | 64 | fn file_not_found() -> &'a str { 65 | msg::FILE_NOT_FOUND 66 | } 67 | 68 | fn missing_input_arg() -> &'a str { 69 | msg::MISSING_INPUT_ARG 70 | } 71 | 72 | fn corrupted_input() -> &'a str { 73 | msg::BAD_SUPERBLOCK 74 | } 75 | } 76 | 77 | //------------------------------------------ 78 | 79 | test_accepts_help!(ThinLs); 80 | test_accepts_version!(ThinLs); 81 | test_rejects_bad_option!(ThinLs); 82 | 83 | test_missing_input_arg!(ThinLs); 84 | test_input_file_not_found!(ThinLs); 85 | test_input_cannot_be_a_directory!(ThinLs); 86 | test_unreadable_input_file!(ThinLs); 87 | 88 | test_readonly_input_file!(ThinLs); 89 | 90 | //------------------------------------------ 91 | // test reading metadata snapshot from a live metadata. 92 | // here we use a corrupted metadata to ensure that "thin_ls -m" reads the 93 | // metadata snapshot only. 94 | 95 | #[test] 96 | fn read_metadata_snapshot() -> Result<()> { 97 | let mut td = TestDir::new()?; 98 | let md = prep_metadata_from_file(&mut td, "corrupted_tmeta_with_metadata_snap.pack")?; 99 | let _ = run_ok(thin_ls_cmd(args![&md, "-m"]))?; 100 | Ok(()) 101 | } 102 | 103 | //------------------------------------------ 104 | -------------------------------------------------------------------------------- /tests/thin_metadata_pack.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | 3 | mod common; 4 | 5 | use common::common_args::*; 6 | use common::input_arg::*; 7 | use common::output_option::*; 8 | use common::program::*; 9 | use common::target::*; 10 | use common::test_dir::*; 11 | use common::thin::*; 12 | 13 | //------------------------------------------ 14 | 15 | const USAGE: &str = "Produces a compressed file of thin metadata. Only packs metadata blocks that are actually used. 16 | 17 | Usage: thin_metadata_pack [OPTIONS] --input --output 18 | 19 | Options: 20 | -f, --force Force overwrite the output file 21 | -h, --help Print help 22 | -i, --input Specify thinp metadata binary device/file 23 | -o, --output Specify packed output file 24 | -V, --version Print version"; 25 | 26 | //------------------------------------------ 27 | 28 | struct ThinMetadataPack; 29 | 30 | impl<'a> Program<'a> for ThinMetadataPack { 31 | fn name() -> &'a str { 32 | "thin_metadata_pack" 33 | } 34 | 35 | fn cmd(args: I) -> Command 36 | where 37 | I: IntoIterator, 38 | I::Item: Into, 39 | { 40 | thin_metadata_pack_cmd(args) 41 | } 42 | 43 | fn usage() -> &'a str { 44 | USAGE 45 | } 46 | 47 | fn arg_type() -> ArgType { 48 | ArgType::IoOptions 49 | } 50 | 51 | fn bad_option_hint(option: &str) -> String { 52 | msg::bad_option_hint(option) 53 | } 54 | } 55 | 56 | impl<'a> InputProgram<'a> for ThinMetadataPack { 57 | fn mk_valid_input(td: &mut TestDir) -> Result { 58 | mk_valid_md(td) 59 | } 60 | 61 | fn file_not_found() -> &'a str { 62 | msg::FILE_NOT_FOUND 63 | } 64 | 65 | fn missing_input_arg() -> &'a str { 66 | msg::MISSING_INPUT_ARG 67 | } 68 | 69 | fn corrupted_input() -> &'a str { 70 | msg::BAD_SUPERBLOCK 71 | } 72 | } 73 | 74 | impl<'a> OutputProgram<'a> for ThinMetadataPack { 75 | fn missing_output_arg() -> &'a str { 76 | msg::MISSING_OUTPUT_ARG 77 | } 78 | } 79 | 80 | //------------------------------------------ 81 | 82 | test_accepts_help!(ThinMetadataPack); 83 | test_accepts_version!(ThinMetadataPack); 84 | test_rejects_bad_option!(ThinMetadataPack); 85 | 86 | test_missing_input_option!(ThinMetadataPack); 87 | test_missing_output_option!(ThinMetadataPack); 88 | test_input_file_not_found!(ThinMetadataPack); 89 | 90 | //----------------------------------------- 91 | -------------------------------------------------------------------------------- /tests/thin_metadata_unpack.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | 3 | mod common; 4 | 5 | use common::common_args::*; 6 | use common::input_arg::*; 7 | use common::output_option::*; 8 | use common::process::*; 9 | use common::program::*; 10 | use common::target::*; 11 | use common::test_dir::*; 12 | use common::thin::*; 13 | 14 | //------------------------------------------ 15 | 16 | const USAGE: &str = "Unpack a compressed file of thin metadata. 17 | 18 | Usage: thin_metadata_unpack [OPTIONS] --input --output 19 | 20 | Options: 21 | -f, --force Force overwrite the output file 22 | -h, --help Print help 23 | -i, --input Specify packed input file 24 | -o, --output Specify thinp metadata binary device/file 25 | -V, --version Print version"; 26 | 27 | //------------------------------------------ 28 | 29 | struct ThinMetadataUnpack; 30 | 31 | impl<'a> Program<'a> for ThinMetadataUnpack { 32 | fn name() -> &'a str { 33 | "thin_metadata_pack" 34 | } 35 | 36 | fn cmd(args: I) -> Command 37 | where 38 | I: IntoIterator, 39 | I::Item: Into, 40 | { 41 | thin_metadata_unpack_cmd(args) 42 | } 43 | 44 | fn usage() -> &'a str { 45 | USAGE 46 | } 47 | 48 | fn arg_type() -> ArgType { 49 | ArgType::IoOptions 50 | } 51 | 52 | fn bad_option_hint(option: &str) -> String { 53 | msg::bad_option_hint(option) 54 | } 55 | } 56 | 57 | impl<'a> InputProgram<'a> for ThinMetadataUnpack { 58 | fn mk_valid_input(_td: &mut TestDir) -> Result { 59 | path_to(TestData::PackedMetadata) 60 | } 61 | 62 | fn file_not_found() -> &'a str { 63 | msg::FILE_NOT_FOUND 64 | } 65 | 66 | fn missing_input_arg() -> &'a str { 67 | msg::MISSING_INPUT_ARG 68 | } 69 | 70 | fn corrupted_input() -> &'a str { 71 | "Not a pack file" 72 | } 73 | } 74 | 75 | impl<'a> OutputProgram<'a> for ThinMetadataUnpack { 76 | fn missing_output_arg() -> &'a str { 77 | msg::MISSING_OUTPUT_ARG 78 | } 79 | } 80 | 81 | //------------------------------------------ 82 | 83 | test_accepts_help!(ThinMetadataUnpack); 84 | test_accepts_version!(ThinMetadataUnpack); 85 | test_rejects_bad_option!(ThinMetadataUnpack); 86 | 87 | test_missing_input_option!(ThinMetadataUnpack); 88 | test_input_file_not_found!(ThinMetadataUnpack); 89 | test_corrupted_input_data!(ThinMetadataUnpack); 90 | 91 | test_missing_output_option!(ThinMetadataUnpack); 92 | 93 | //------------------------------------------ 94 | 95 | // TODO: share with thin_restore/cache_restore/era_restore 96 | 97 | #[test] 98 | fn end_to_end() -> Result<()> { 99 | let mut td = TestDir::new()?; 100 | let md_in = mk_valid_md(&mut td)?; 101 | let md_packed = td.mk_path("meta.pack"); 102 | let md_out = td.mk_path("meta.out"); 103 | run_ok(thin_metadata_pack_cmd(args![ 104 | "-i", &md_in, "-o", &md_packed 105 | ]))?; 106 | run_ok(thin_metadata_unpack_cmd(args![ 107 | "-i", &md_packed, "-o", &md_out 108 | ]))?; 109 | 110 | let dump1 = run_ok(thin_dump_cmd(args![&md_in]))?; 111 | let dump2 = run_ok(thin_dump_cmd(args![&md_out]))?; 112 | assert_eq!(dump1, dump2); 113 | Ok(()) 114 | } 115 | 116 | //------------------------------------------ 117 | -------------------------------------------------------------------------------- /tests/thin_rmap.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | 3 | mod common; 4 | 5 | use common::common_args::*; 6 | use common::process::*; 7 | use common::program::*; 8 | use common::target::*; 9 | use common::test_dir::*; 10 | use common::thin::*; 11 | 12 | //------------------------------------------ 13 | 14 | const USAGE: &str = "Output reverse map of a thin provisioned region of blocks 15 | 16 | Usage: thin_rmap --region 17 | 18 | Arguments: 19 | Specify the input device 20 | 21 | Options: 22 | -h, --help Print help 23 | --region Specify range of blocks on the data device 24 | -V, --version Print version"; 25 | 26 | //------------------------------------------ 27 | 28 | struct ThinRmap; 29 | 30 | impl<'a> Program<'a> for ThinRmap { 31 | fn name() -> &'a str { 32 | "thin_rmap" 33 | } 34 | 35 | fn cmd(args: I) -> Command 36 | where 37 | I: IntoIterator, 38 | I::Item: Into, 39 | { 40 | thin_rmap_cmd(args) 41 | } 42 | 43 | fn usage() -> &'a str { 44 | USAGE 45 | } 46 | 47 | fn arg_type() -> ArgType { 48 | ArgType::InputArg 49 | } 50 | 51 | fn bad_option_hint(option: &str) -> String { 52 | msg::bad_option_hint(option) 53 | } 54 | } 55 | 56 | //------------------------------------------ 57 | 58 | test_accepts_help!(ThinRmap); 59 | test_accepts_version!(ThinRmap); 60 | test_rejects_bad_option!(ThinRmap); 61 | 62 | //------------------------------------------ 63 | 64 | #[test] 65 | fn valid_region_format_should_pass() -> Result<()> { 66 | let mut td = TestDir::new()?; 67 | let md = mk_valid_md(&mut td)?; 68 | run_ok(thin_rmap_cmd(args![&md, "--region", "23..7890"]))?; 69 | Ok(()) 70 | } 71 | 72 | #[test] 73 | fn invalid_regions_should_fail() -> Result<()> { 74 | let invalid_regions = [ 75 | "23,7890", 76 | "23..six", 77 | "found..7890", 78 | "89..88", 79 | "89..89", 80 | "89..", 81 | "", 82 | "89...99", 83 | ]; 84 | for r in &invalid_regions { 85 | let mut td = TestDir::new()?; 86 | let md = mk_valid_md(&mut td)?; 87 | run_fail(thin_rmap_cmd(args![&md, r]))?; 88 | } 89 | Ok(()) 90 | } 91 | 92 | #[test] 93 | fn multiple_regions_should_pass() -> Result<()> { 94 | let mut td = TestDir::new()?; 95 | let md = mk_valid_md(&mut td)?; 96 | run_ok(thin_rmap_cmd(args![ 97 | &md, "--region", "1..23", "--region", "45..78" 98 | ]))?; 99 | Ok(()) 100 | } 101 | 102 | #[test] 103 | fn junk_input() -> Result<()> { 104 | let mut td = TestDir::new()?; 105 | let xml = mk_valid_xml(&mut td)?; 106 | run_fail(thin_rmap_cmd(args![&xml, "--region", "0..-1"]))?; 107 | Ok(()) 108 | } 109 | 110 | //------------------------------------------ 111 | -------------------------------------------------------------------------------- /xml_metadata/basic_tests_dd.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | --------------------------------------------------------------------------------