├── .github └── workflows │ └── ci.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── contrib └── bmap_write.py ├── debian ├── bmaptool.docs ├── bmaptool.install ├── bmaptool.manpages ├── changelog ├── control ├── copyright ├── manpages ├── rules └── source │ └── format ├── docs ├── RELEASE_NOTES ├── TODO.md └── man1 │ └── bmaptool.1 ├── make_a_release.sh ├── packaging ├── bmaptool.changes └── bmaptool.spec ├── pyproject.toml ├── src └── bmaptool │ ├── BmapCopy.py │ ├── BmapCreate.py │ ├── BmapHelpers.py │ ├── CLI.py │ ├── Filemap.py │ ├── TransRead.py │ ├── __init__.py │ └── __main__.py └── tests ├── __init__.py ├── helpers.py ├── oldcodebase ├── BmapCopy1_0.py ├── BmapCopy2_0.py ├── BmapCopy2_1.py ├── BmapCopy2_2.py ├── BmapCopy2_3.py ├── BmapCopy2_4.py ├── BmapCopy2_5.py ├── BmapCopy2_6.py ├── BmapCopy3_0.py └── __init__.py ├── test-data ├── test.image.bmap.v1.2 ├── test.image.bmap.v1.3 ├── test.image.bmap.v1.4 ├── test.image.bmap.v2.0 └── test.image.gz ├── test_CLI.py ├── test_api_base.py ├── test_bmap_helpers.py ├── test_compat.py └── test_filemap.py /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Test bmaptool 3 | 4 | on: 5 | - push 6 | - pull_request 7 | 8 | jobs: 9 | test: 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | python-version: 14 | - "3.8" 15 | - "3.9" 16 | - "3.10" 17 | - "3.11" 18 | - "3.12" 19 | # Testing with native host python is required in order to test the 20 | # GPG code, since it must use the host python3-gpg package 21 | - "native" 22 | steps: 23 | - uses: actions/checkout@v4 24 | 25 | - if: matrix.python-version != 'native' 26 | name: Setup Python ${{ matrix.python-version }} 27 | uses: actions/setup-python@v4 28 | with: 29 | python-version: ${{ matrix.python-version }} 30 | 31 | - if: matrix.python-version == 'native' 32 | name: Setup Native Python 33 | run: | 34 | sudo apt-get install -y python3 python3-pip libgpgme11-dev python3-gpg 35 | 36 | - name: Install dependencies 37 | run: | 38 | sudo apt-get install -y pbzip2 pigz lzop liblz4-tool 39 | python3 -m pip install --upgrade pip 40 | python3 -m pip install build 41 | 42 | - name: Build package 43 | run: | 44 | python3 -m build 45 | 46 | - name: Install package 47 | run: | 48 | python3 -m pip install -e .[dev] 49 | 50 | - name: Run tests 51 | run: | 52 | python3 -m unittest -vb 53 | 54 | lint: 55 | runs-on: ubuntu-latest 56 | steps: 57 | - uses: actions/checkout@v3 58 | - uses: psf/black@stable 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | .hypothesis/ 46 | 47 | # Sphinx documentation 48 | docs/_build/ 49 | 50 | # IPython Notebook 51 | .ipynb_checkpoints 52 | 53 | # pyenv 54 | .python-version 55 | 56 | # dotenv 57 | .env 58 | 59 | # virtualenv 60 | venv/ 61 | .venv/ 62 | ENV/ 63 | 64 | #vscode 65 | .vscode/ 66 | .noseids 67 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | ### Added 10 | ### Changed 11 | 12 | ## [3.9.0] 13 | 14 | - copy: add `--removable-device`, `--keyring` and `--fingerprint` options 15 | - Respect query part of the url when operating on the path 16 | - support FTP authentication 17 | - rework GPG tests 18 | 19 | ## [3.8.0] 20 | 21 | - Move project to yoctoproject 22 | - Maintainers change from Artem Bityutskiy (Thank you!) to Trevor Woerner, Joshua Watt, Tim Orling 23 | - Consolidate name as 'bmaptool' 24 | 25 | ## [3.7.0] 26 | ### Added 27 | - Use GitHub Actions for CI (#109) 28 | - Add `poetry` for dependency management and `black` for code formatting (#104) 29 | - Add functionality for copying from standard input (#99) 30 | ### Changed 31 | - Switch from gpg to gpgme module (#103) 32 | 33 | ## [3.6.0] 34 | 35 | 1. Improve ZFS compatibility. 36 | 2. Added the 'zstd' compression type support. 37 | 3. Add '--psplash-pipe' option for interacting with psplash. 38 | 39 | ## [3.5.0] 40 | 41 | 1. Fixed copying of compressed files from URLs, it was a regression introduced 42 | in bmap-tools 3.4. 43 | 2. Python 3.x support fixes and improvements. 44 | 3. RPM packaging fixes. 45 | 4. Improved help and error messages. 46 | 47 | ## [3.4.0] 48 | 49 | 1. bmap-tools has now new home: https://github.com/01org/bmap-tools 50 | 51 | 2. Python 3.x support: bmap-tools now compatible with Python 3.3+ 52 | 53 | 3. bmaptool now can be shipped as standalone application. 54 | See PEP441 (zipapp) for implementation details. 55 | 56 | 4. ZIP archives now supported. Similar to tar.* archives, image must be 57 | first file in archive. 58 | 59 | 5. LZ4 compression now supported. Files with the following extensions are 60 | recognized as LZ4-compressed: ".lz4", ".tar.lz4" and ".tlz4". 61 | 62 | 6. Fixed copying images on XFS file system where predictive caching lead 63 | to more blocks to be mapped than needed. 64 | 65 | 7. Fixed detection of block size on file systems that do not report it 66 | correctly via ioctl FIGETBSZ. 67 | 68 | ## [3.2.0] 69 | 70 | 1. Multi-stream bzip2 archives are now supported. These are usually created 71 | with the 'pbzip2' compressor. 72 | 73 | 2. LZO archives are now supported too. Files with the following extensions are 74 | recognized as LZO-compressed: ".lzo", ".tar.lzo", ".tzo". 75 | 76 | 3. Make 'bmaptool create' (and hence, the BmapCreate module) work with the 77 | "tmpfs" file-system. Tmpfs does not, unfortunately, support the "FIEMAP" 78 | ioctl, but it supports the "SEEK_HOLE" option of the "lseek" system call, 79 | which is now used for finding where the holes are. However, this works only 80 | with Linux kernels of version 3.8 or higher. 81 | 82 | Generally, "FIEMAP" is faster than "SEEK_HOLE" for large files, so we always 83 | try to start with using FIEMAP, and if it is not supported, we fall-back to 84 | using "SEEK_HOLE". Therefore, the "Fiemap" module was re-named to "Filemap", 85 | since it is now supports more than just the FIEMAP ioctl. 86 | 87 | Unfortunately, our "SEEK_HOLE" method requires the directory where the image 88 | resides to be accessible for writing, because in current implementation we 89 | need to create a temporary file there for a short time. The temporary file 90 | is used to detect whether tmpfs really supports SEEK_HOLE, or the system 91 | just fakes it by always returning EOF (this is what happens in pre-3.8 92 | kernels). 93 | 94 | 4. Decompression should now require less memory, which should fix 95 | out-of-memory problems reported by some users recently. Namely, users 96 | reported that decompressing large bz2-compressed sparse files caused 97 | out-of-memory situation on machines with 2GB RAM. This should be fixed now. 98 | 99 | 5. Reading and decompressing is now faster because we now use more parallelism: 100 | reading the data form the source URL is done in separate thread, 101 | decompressing happens in a separate process too. My measurement with Tizen 102 | IVI images from 'tizen.org' showed 10% read speed improvement, but this 103 | depends a lot on where the bottle-neck is: the USB stick, the network, or 104 | the CPU load. 105 | 106 | ## [3.1.0] 107 | 108 | This bug-fix release is about fixing a small screw-up in version 3.0, where we 109 | introduced incompatible bmap format changes, but did not properly increase the 110 | bmap format version number. Instead of making it to be version 2.0, we made it 111 | to be version 1.4. The result is that bmap-tools v2.x crash with those 112 | 1.4-formatted bmap files. 113 | 114 | This release changes the bmap format version from 1.4 to 2.0 in order to 115 | lessen the versioning screw-up. Increased major bmap format version number will 116 | make sure that older bmap-tools fail with a readable error message, instead of 117 | crashing. 118 | 119 | Thus, the situation as follows: 120 | * bmap-tools v2.x: handle bmap format versions 1.0-1.3, crash with 1.4, and 121 | nicely exit with 2.0 122 | * bmap-tools v3.0: handles all 1.x bmap format versions, exits nicely with 2.0 123 | * bmap-tools v3.1: handles all bmap format versions 124 | 125 | ## [3.0.0] 126 | 127 | 1. Switch from using SHA1 checksums in the bmap file to SHA256. This required 128 | bmap format change. The new format version is 1.4. BmapCopy (and thus, 129 | bmaptool supports all the older versions too). Now it is possible to use any 130 | hash functions for checksumming, not only SHA256, but SHA256 is the default 131 | for BmapCreate. 132 | 133 | 2. Support OpenPGP (AKA gpg) signatures for the bmap file. From now on the bmap 134 | file can be signed with gpg, in which case bmaptool verifies the bmap file 135 | signature. If the signature is bad, bmaptool exits with an error message. 136 | The verification can be disabled with the --no-sig-verify option. 137 | 138 | Both detached and "in-band" clearsign signatures are supported. Bmaptool 139 | automatically discovers detached signatures by checking ".sig" and ".asc" 140 | files. 141 | 142 | 3. The Fiemap module (and thus, bmaptool) now always synchronizes the image 143 | before scanning it for mapped areas. This is done by using the 144 | "FIEMAP_FLAG_SYNC" flag of the FIEMAP ioctl. 145 | 146 | The reason for synchronizing the file is bugs in early implementations of 147 | FIEMAP in the kernel and file-systems, and synchronizing the image is a 148 | known way to work around the bugs. 149 | 150 | ## [2.6.0] 151 | 152 | ### Added 153 | 154 | - On-the-fly decompression support for '.xz' and '.tar.xz' files. 155 | 156 | ## [2.5.0] 157 | 158 | 1. bmaptool (or more precisely, the BmapCopy class) has an optimization where 159 | we switch to the "noop" I/O scheduler when writing directly to block 160 | devices. We also lessen the allowed amount of dirty data for this block 161 | device in order to create less memory pressure on the system. These tweaks 162 | are done by touching the corresponding sysfs files of the block device. The 163 | old bmaptool behavior was that it failed when it could not modify these 164 | files. However, there are systems where users can write to some block 165 | devices (USB sticks, for example), but they do not have permissions to 166 | change the sysfs files, and bmaptool did not work for normal users on such 167 | systems. In version 2.5 we change the behavior and do not fail anymore if we 168 | do not have enough permissions for changing sysfs files, simply because this 169 | is an optimization, although a quite important one. However, we do print a 170 | warning message. 171 | 2. Many improvements and fixes in the Debian packaging, which should make it 172 | simpler for distributions to package bmap-tools. 173 | 174 | ## 2.4.0 175 | 176 | 1. Add SSH URLs support. These URLs start with "ssh://" and have the following 177 | format: ssh://user:password@host:path, where 178 | * user - user name (optional) 179 | * password - the password (optional) 180 | * host - hostname 181 | * path - path to the image file on the remote host 182 | 183 | If the password was given in the URL, bmaptool will use password-based SSH 184 | authentication, otherwise key-based SSH authentication will be used. 185 | 186 | ## 2.3.0 187 | 188 | 1. Add bmap file SHA1 checksum into the bmap file itself in order to improve 189 | robustness of bmaptool. Now we verify bmap file integrity before using it, 190 | and if it is corrupted or incomplete, we should be able to detect this. 191 | 192 | The reason for this change was a bug report from a user who somehow ended 193 | up with a corrupted bmap file and experienced weird issues. 194 | 195 | This also means that manual changes the bmap file will end up with a SHA1 196 | mismatch failure. In order to prevent the failure, one has to update the bmap 197 | file's SHA1 by putting all ASCII "0" symbols (should be 40 zeroes) to the 198 | "BmapFileSHA1" tag, then generating SHA1 of the resulting file, and then 199 | put the calculated real SHA1 back to the "BmapFileSHA1" tag. 200 | 201 | In the future, if needed, we can create a "bmaptool checksum" command which 202 | could update SHA1s in the bmap file. 203 | 204 | 2. Re-structure the bmap file layout and put information about mapped blocks 205 | count at the beginning of the bmap XML file, not after the block map table. 206 | This will make it possible to optimize bmap file parsing in the future. This 207 | also makes the bmap file a little bit more human-readable. 208 | 209 | 2. Make the test-suite work on btrfs. 210 | 211 | ## 2.2.0 212 | 213 | 1. Made bmaptool understand URLs which include user name and password 214 | (the format is: https://user:password@server.com) 215 | 216 | ## 2.1.0 217 | 218 | 1. Fixed the out of memory problems when copying .bz2 files. 219 | 2. Added CentOS 6 support in packaging. 220 | 221 | ## 2.0.0 222 | 223 | There are several user-visible changes in 'bmaptool copy': 224 | 225 | 1. In order to copy an image without bmap, the user now has to explicitly 226 | specify the "--nobmap" option. In v1.0 this was not necessary. The reason 227 | for this change is that users forget to use --bmap and do not realize that 228 | they are copying entire the image. IOW, this is a usability improvement. 229 | 230 | 2. The bmap file auto-discovery feature has been added. Now when the user does 231 | not specify the bmap file using the --bmap option, 'bmaptool copy' will try 232 | to find it at the same place where the image resides. It will look for files 233 | with a similar base name and ".bmap" extension. This should make it easier 234 | to use bmaptool. 235 | 236 | 3. 'bmaptool copy' now can open remote files, so it is not necessary to 237 | download the images anymore, and you can specify the URL to bmaptool. For 238 | example: 239 | 240 | bmaptool copy download.tizen.org/snapshots/ivi/.../ivi-2.0.raw.bz2 241 | 242 | The tool will automatically discover the bmap file, read from the image from 243 | the 'download.tizen.org' server, decompress it on-the-fly, and copy to the 244 | target file/device. The proxy is supported via the standard environment 245 | variables like 'http_proxy', 'https_proxy', 'no_proxy', etc. 246 | 247 | 4. Now 'bmaptool' prints the progress while copying. This improves usability 248 | as well: copying may take minutes, and it is nice to let the user know how 249 | much has already been copied. 250 | 251 | 5. Warnings and errors are high-lighted using yellow and red labels now. 252 | 253 | 6. Added bmaptool man page. 254 | 255 | 'bmaptool create' has no changes comparing to release v1.0. 256 | 257 | ## 1.0.0 258 | 259 | The first bmap-tools release. All the planned features are implemented, 260 | automated tests are implemented. We provide nice API modules for bmap creation 261 | ('BmapCreate.py') and copying with bmap ('BmapCopy.py'). The 'Fiemap.py' API 262 | module provides python API to the FIEMAP Linux ioctl. 263 | 264 | The 'bmaptool' command-line tool is a basically a small wrapper over the 265 | API modules. It implements the 'create' and 'copy' sub-commands, which 266 | allow creating bmap for a given file and copying a file to another file 267 | or to a block device using bmap. 268 | 269 | The 'bmaptools copy' command (and thus, 'BmapCopy.py' module) support 270 | accept compressed files and transparently de-compress them. The following 271 | compression types are supported: .bz2, .gz, .tar.bz2, .tar.gz. 272 | 273 | The original user of this project is Tizen IVI where the OS images are 274 | sparse 2.6GiB files which are distributed as .bz2 file. Since the images 275 | are only 40% full, the .bz2 file weights about 300MiB. Tizen IVI uses the 276 | 'BmapCreate.py' API module to generate the bmap file for the 2.6GiB images 277 | (before the image was compressed, because once it is compressed with bzip2, 278 | the information about holes gets lost). Then the bmap file is distributed 279 | together with the .bz2 image. And Tizen IVI users are able to flash the 280 | images to USB stick using the following command: 281 | 282 | $ bmaptool copy --bmap image.bmap image.bz2 /dev/usb_stick 283 | 284 | This command decompresses the image (image.bz2) on-the-fly, and copies all 285 | the mapped blocks (listed in 'image.bmap') to the USB stick (the 286 | '/dev/usb_stick' block device). 287 | 288 | This is a lot faster than the old method: 289 | 290 | $ bzcat image.bz2 | dd of=/dev/usb_stick 291 | 292 | Additionally, 'bmaptool copy' verifies the image - the bmap stores SHA1 293 | checksums for all mapped regions. 294 | 295 | However, bmap-tools may be useful for other projects as well - it is generic 296 | and just implements the idea of fast block-based flashing (as opposed to 297 | file-based flashing). Block-based flashing has a lot of benefits. 298 | 299 | The 'BmapCopy.py' module implements a couple of important optimization when 300 | copying to block device: 301 | 1. Switch the block device I/O scheduler to 'Noop', which is a lot faster 302 | than 'CFQ' for sequential writes. 303 | 2. Limits the amount of memory which the kernel uses for buffering, in 304 | order to have less impact on the overall system performance. 305 | 3. Reads in a separate thread, which is a lot faster when copying compressed 306 | images, because we read/uncompress/verify SHA1 in parallel to writing 307 | to a potentially slow block device. 308 | 309 | We support bmap format versioning. The current format is 1.2. The minor version 310 | number must not break backward compatibility, while the major numbers indicates 311 | some incompatibility. 312 | 313 | [Unreleased]: https://github.com/intel/bmap-tools/compare/v3.6..HEAD 314 | [3.6.0]: https://github.com/intel/bmap-tools/releases/tag/v3.6 315 | [3.5.0]: https://github.com/intel/bmap-tools/releases/tag/v3.5 316 | [3.4.0]: https://github.com/intel/bmap-tools/releases/tag/v3.4 317 | [3.2.0]: https://github.com/intel/bmap-tools/releases/tag/v3.2 318 | [3.1.0]: https://github.com/intel/bmap-tools/releases/tag/v3.1 319 | [3.0.0]: https://github.com/intel/bmap-tools/releases/tag/v3.0 320 | [2.6.0]: https://github.com/intel/bmap-tools/releases/tag/v2.6 321 | [2.5.0]: https://github.com/intel/bmap-tools/releases/tag/v2.5 322 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `bmaptool` 2 | 3 | > The better `dd` for embedded projects, based on block maps. 4 | 5 | ## Introduction 6 | 7 | `bmaptool` is a generic tool for creating the block map (bmap) for a file and 8 | copying files using the block map. The idea is that large files, like raw 9 | system image files, can be copied or flashed a lot faster and more reliably 10 | with `bmaptool` than with traditional tools, like `dd` or `cp`. 11 | 12 | `bmaptool` was originally created for the "Tizen IVI" project and it was used for 13 | flashing system images to USB sticks and other block devices. `bmaptool` can also 14 | be used for general image flashing purposes, for example, flashing Fedora Linux 15 | OS distribution images to USB sticks. 16 | 17 | Originally Tizen IVI images had been flashed using the `dd` tool, but bmaptool 18 | brought a number of advantages. 19 | 20 | * Faster. Depending on various factors, like write speed, image size, how full 21 | is the image, and so on, `bmaptool` was 5-7 times faster than `dd` in the Tizen 22 | IVI project. 23 | * Integrity. `bmaptool` verifies data integrity while flashing, which means that 24 | possible data corruptions will be noticed immediately. 25 | * Usability. `bmaptool` can read images directly from the remote server, so users 26 | do not have to download images and save them locally. 27 | * Protects user's data. Unlike `dd`, if you make a mistake and specify a wrong 28 | block device name, `bmaptool` will less likely destroy your data because it has 29 | protection mechanisms which, for example, prevent `bmaptool` from writing to a 30 | mounted block device. 31 | 32 | ## Usage 33 | 34 | `bmaptool` supports 2 subcommands: 35 | * `copy` - copy a file to another file using bmap or flash an image to a block 36 | device 37 | * `create` - create a bmap for a file 38 | 39 | You can get usage reference for `bmaptool` and all the supported command using 40 | the `-h` or `--help` options: 41 | 42 | ```bash 43 | $ bmaptool -h # General bmaptool help 44 | $ bmaptool -h # Help on the sub-command 45 | ``` 46 | 47 | You can also refer to the `bmaptool` manual page: 48 | ```bash 49 | $ man bmaptool 50 | ``` 51 | 52 | ## Concept 53 | 54 | This section provides general information about the block map (bmap) necessary 55 | for understanding how `bmaptool` works. The structure of the section is: 56 | 57 | * "Sparse files" - the bmap ideas are based on sparse files, so it is important 58 | to understand what sparse files are. 59 | * "The block map" - explains what bmap is. 60 | * "Raw images" - the main usage scenario for `bmaptool` is flashing raw images, 61 | which this section discusses. 62 | * "Usage scenarios" - describes various possible bmap and `bmaptool` usage 63 | scenarios. 64 | 65 | ### Sparse files 66 | 67 | One of the main roles of a filesystem, generally speaking, is to map blocks of 68 | file data to disk sectors. Different file-systems do this mapping differently, 69 | and filesystem performance largely depends on how well the filesystem can do 70 | the mapping. The filesystem block size is usually 4KiB, but may also be 8KiB or 71 | larger. 72 | 73 | Obviously, to implement the mapping, the file-system has to maintain some kind 74 | of on-disk index. For any file on the file-system, and any offset within the 75 | file, the index allows you to find the corresponding disk sector, which stores 76 | the file's data. Whenever we write to a file, the filesystem looks up the index 77 | and writes to the corresponding disk sectors. Sometimes the filesystem has to 78 | allocate new disk sectors and update the index (such as when appending data to 79 | the file). The filesystem index is sometimes referred to as the "filesystem 80 | metadata". 81 | 82 | What happens if a file area is not mapped to any disk sectors? Is this 83 | possible? The answer is yes. It is possible and these unmapped areas are often 84 | called "holes". And those files which have holes are often called "sparse 85 | files". 86 | 87 | All reasonable file-systems like Linux ext[234], btrfs, XFS, or Solaris XFS, 88 | and even Windows' NTFS, support sparse files. Old and less reasonable 89 | filesystems, like FAT, do not support holes. 90 | 91 | Reading holes returns zeroes. Writing to a hole causes the filesystem to 92 | allocate disk sectors for the corresponding blocks. Here is how you can create 93 | a 4GiB file with all blocks unmapped, which means that the file consists of a 94 | huge 4GiB hole: 95 | 96 | ```bash 97 | $ truncate -s 4G image.raw 98 | $ stat image.raw 99 | File: image.raw 100 | Size: 4294967296 Blocks: 0 IO Block: 4096 regular file 101 | ``` 102 | 103 | Notice that `image.raw` is a 4GiB file, which occupies 0 blocks on the disk! 104 | So, the entire file's contents are not mapped anywhere. Reading this file would 105 | result in reading 4GiB of zeroes. If you write to the middle of the image.raw 106 | file, you'll end up with 2 holes and a mapped area in the middle. 107 | 108 | Therefore: 109 | * Sparse files are files with holes. 110 | * Sparse files help save disk space, because, roughly speaking, holes do not 111 | occupy disk space. 112 | * A hole is an unmapped area of a file, meaning that it is not mapped anywhere 113 | on the disk. 114 | * Reading data from a hole returns zeroes. 115 | * Writing data to a hole destroys it by forcing the filesystem to map 116 | corresponding file areas to disk sectors. 117 | * Filesystems usually operate with blocks, so sizes and offsets of holes are 118 | aligned to the block boundary. 119 | 120 | It is also useful to know that you should work with sparse files carefully. It 121 | is easy to accidentally expand a sparse file, that is, to map all holes to 122 | zero-filled disk areas. For example, `scp` always expands sparse files, the 123 | `tar` and `rsync` tools do the same, by default, unless you use the `--sparse` 124 | option. Compressing and then decompressing a sparse file usually expands it. 125 | 126 | There are 2 ioctl's in Linux which allow you to find mapped and unmapped areas: 127 | `FIBMAP` and `FIEMAP`. The former is very old and is probably supported by all 128 | Linux systems, but it is rather limited and requires root privileges. The 129 | latter is a lot more advanced and does not require root privileges, but it is 130 | relatively new (added in Linux kernel, version 2.6.28). 131 | 132 | Recent versions of the Linux kernel (starting from 3.1) also support the 133 | `SEEK_HOLE` and `SEEK_DATA` values for the `whence` argument of the standard 134 | `lseek()` system call. They allow positioning to the next hole and the next 135 | mapped area of the file. 136 | 137 | Advanced Linux filesystems, in modern kernels, also allow "punching holes", 138 | meaning that it is possible to unmap any aligned area and turn it into a hole. 139 | This is implemented using the `FALLOC_FL_PUNCH_HOLE` `mode` of the 140 | `fallocate()` system call. 141 | 142 | ### The bmap 143 | 144 | The bmap is an XML file, which contains a list of mapped areas, plus some 145 | additional information about the file it was created for, for example: 146 | * SHA256 checksum of the bmap file itself 147 | * SHA256 checksum of the mapped areas 148 | * the original file size 149 | * amount of mapped data 150 | 151 | The bmap file is designed to be both easily machine-readable and 152 | human-readable. All the machine-readable information is provided by XML tags. 153 | The human-oriented information is in XML comments, which explain the meaning of 154 | XML tags and provide useful information like amount of mapped data in percent 155 | and in MiB or GiB. 156 | 157 | So, the best way to understand bmap is to just to read it. Here is an 158 | [example of a bmap file](tests/test-data/test.image.bmap.v2.0). 159 | 160 | ### Raw images 161 | 162 | Raw images are the simplest type of system images which may be flashed to the 163 | target block device, block-by-block, without any further processing. Raw images 164 | just "mirror" the target block device: they usually start with the MBR sector. 165 | There is a partition table at the beginning of the image and one or more 166 | partitions containing filesystems, like ext4. Usually, no special tools are 167 | required to flash a raw image to the target block device. The standard `dd` 168 | command can do the job: 169 | 170 | ```bash 171 | $ dd if=tizen-ivi-image.raw of=/dev/usb_stick 172 | ``` 173 | 174 | At first glance, raw images do not look very appealing because they are large 175 | and it takes a lot of time to flash them. However, with bmap, raw images become 176 | a much more attractive type of image. We will demonstrate this, using Tizen IVI 177 | as an example. 178 | 179 | The Tizen IVI project uses raw images which take 3.7GiB in Tizen IVI 2.0 alpha. 180 | The images are created by the MIC tool. Here is a brief description of how MIC 181 | creates them: 182 | 183 | * create a 3.7GiB sparse file, which will become the Tizen IVI image in the end 184 | * partition the file using the `parted` tool 185 | * format the partitions using the `mkfs.ext4` tool 186 | * loop-back mount all the partitions 187 | * install all the required packages to the partitions: copy all the needed 188 | files and do all the tweaks 189 | * unmount all loop-back-mounted image partitions, the image is ready 190 | * generate the block map file for the image 191 | * compress the image using `bzip2`, turning them into a small file, around 192 | 300MiB 193 | 194 | The Tizen IVI raw images are initially sparse files. All the mapped blocks 195 | represent useful data and all the holes represent unused regions, which 196 | "contain" zeroes and do not have to be copied when flashing the image. Although 197 | information about holes is lost once the image gets compressed, the bmap file 198 | still has it and it can be used to reconstruct the uncompressed image or to 199 | flash the image quickly, by copying only the mapped regions. 200 | 201 | Raw images compress extremely well because the holes are essentially zeroes, 202 | which compress perfectly. This is why 3.7GiB Tizen IVI raw images, which 203 | contain about 1.1GiB of mapped blocks, take only 300MiB in a compressed form. 204 | And the important point is that you need to decompress them only while 205 | flashing. The `bmaptool` does this "on-the-fly". 206 | 207 | Therefore: 208 | * raw images are distributed in a compressed form, and they are almost as small 209 | as a tarball (that includes all the data the image would take) 210 | * the bmap file and the `bmaptool` make it possible to quickly flash the 211 | compressed raw image to the target block device 212 | * optionally, the `bmaptool` can reconstruct the original uncompressed sparse raw 213 | image file 214 | 215 | And, what is even more important, is that flashing raw images is extremely fast 216 | because you write directly to the block device, and write sequentially. 217 | 218 | Another great thing about raw images is that they may be 100% ready-to-go and 219 | all you need to do is to put the image on your device "as-is". You do not have 220 | to know the image format, which partitions and filesystems it contains, etc. 221 | This is simple and robust. 222 | 223 | ### Usage scenarios 224 | 225 | Flashing or copying large images is the main `bmaptool` use case. The idea is 226 | that if you have a raw image file and its bmap, you can flash it to a device by 227 | writing only the mapped blocks and skipping the unmapped blocks. 228 | 229 | What this basically means is that with bmap it is not necessary to try to 230 | minimize the raw image size by making the partitions small, which would require 231 | resizing them. The image can contain huge multi-gigabyte partitions, just like 232 | the target device requires. The image will then be a huge sparse file, with 233 | little mapped data. And because unmapped areas "contain" zeroes, the huge image 234 | will compress extremely well, so the huge image will be very small in 235 | compressed form. It can then be distributed in compressed form, and flashed 236 | very quickly with `bmaptool` and the bmap file, because `bmaptool` will decompress 237 | the image on-the-fly and write only mapped areas. 238 | 239 | The additional benefit of using bmap for flashing is the checksum verification. 240 | Indeed, the `bmaptool create` command generates SHA256 checksums for all mapped 241 | block ranges, and the `bmaptool copy` command verifies the checksums while 242 | writing. Integrity of the bmap file itself is also protected by a SHA256 243 | checksum and `bmaptool` verifies it before starting flashing. 244 | 245 | On top of this, the bmap file can be signed using OpenPGP (gpg) and bmaptool 246 | automatically verifies the signature if it is present. This allows for 247 | verifying the bmap file integrity and authoring. And since the bmap file 248 | contains SHA256 checksums for all the mapped image data, the bmap file 249 | signature verification should be enough to guarantee integrity and authoring of 250 | the image file. 251 | 252 | The second usage scenario is reconstructing sparse files Generally speaking, if 253 | you had a sparse file but then expanded it, there is no way to reconstruct it. 254 | In some cases, something like: 255 | 256 | ```bash 257 | $ cp --sparse=always expanded.file reconstructed.file 258 | ``` 259 | 260 | would be enough. However, a file reconstructed this way will not necessarily be 261 | the same as the original sparse file. The original sparse file could have 262 | contained mapped blocks filled with all zeroes (not holes), and, in the 263 | reconstructed file, these blocks will become holes. In some cases, this does 264 | not matter. For example, if you just want to save disk space. However, for raw 265 | images, flashing it does matter, because it is essential to write zero-filled 266 | blocks and not skip them. Indeed, if you do not write the zero-filled block to 267 | corresponding disk sectors which, presumably, contain garbage, you end up with 268 | garbage in those blocks. In other words, when we are talking about flashing raw 269 | images, the difference between zero-filled blocks and holes in the original 270 | image is essential because zero-filled blocks are the required blocks which are 271 | expected to contain zeroes, while holes are just unneeded blocks with no 272 | expectations regarding the contents. 273 | 274 | `bmaptool` may be helpful for reconstructing sparse files properly. Before the 275 | sparse file is expanded, you should generate its bmap (for example, by using 276 | the `bmaptool create` command). Then you may compress your file or, otherwise, 277 | expand it. Later on, you may reconstruct it using the `bmaptool copy` command. 278 | 279 | ## Known Issues 280 | 281 | ### ZFS File System 282 | 283 | If running on the ZFS file system, the Linux ZFS kernel driver parameters 284 | configuration can cause the finding of mapped and unmapped areas to fail. 285 | This can be fixed temporarily by doing the following: 286 | 287 | ```bash 288 | $ echo 1 | sudo tee -a /sys/module/zfs/parameters/zfs_dmu_offset_next_sync 289 | ``` 290 | 291 | However, if a permanent solution is required then perform the following: 292 | 293 | ```bash 294 | $ echo "options zfs zfs_dmu_offset_next_sync=1" | sudo tee -a /etc/modprobe.d/zfs.conf 295 | ``` 296 | 297 | Depending upon your Linux distro, you may also need to do the following to 298 | ensure that the permanent change is updated in all your initramfs images: 299 | 300 | ```bash 301 | $ sudo update-initramfs -u -k all 302 | ``` 303 | 304 | To verify the temporary or permanent change has worked you can use the following 305 | which should return `1`: 306 | 307 | ```bash 308 | $ cat /sys/module/zfs/parameters/zfs_dmu_offset_next_sync 309 | ``` 310 | 311 | More details can be found [in the OpenZFS documentation](https://openzfs.github.io/openzfs-docs/Performance%20and%20Tuning/Module%20Parameters.html). 312 | 313 | ## Hacking 314 | 315 | `bmaptool` uses `hatch` to build python packages. If you would like to make 316 | changes to the project, first create a new virtual environment and activate it: 317 | 318 | ```bash 319 | python3 -m venv .venv 320 | . .venv/bin/activate 321 | ``` 322 | 323 | Next install the project in editable mode with development dependencies: 324 | 325 | ```bash 326 | pip install -e '.[dev]' 327 | ``` 328 | 329 | Note: You may need to install the development package for `libgpgme` on your 330 | system. Depending on your OS this may be called `libgpgme-dev`. 331 | 332 | Finally, to run tests use `unittest`: 333 | 334 | ```bash 335 | python3 -m unittest -bv 336 | ``` 337 | 338 | ## Project and maintainer 339 | 340 | The bmaptool project implements bmap-related tools and API modules. The entire 341 | project is written in python. 342 | 343 | The project author is Artem Bityutskiy (dedekind1@gmail.com). The project 344 | is currently maintained by: 345 | * Trevor Woerner (twoerner@gmail.com) 346 | * Joshua Watt (JPEWhacker@gmail.com) 347 | * Tim Orling (ticotimo@gmail.com) 348 | 349 | Project git repository is here: 350 | https://github.com/yoctoproject/bmaptool 351 | 352 | ## Artem's Credits 353 | 354 | * Ed Bartosh (eduard.bartosh@intel.com) for helping me with learning python 355 | (this is my first python project) and working with the Tizen IVI 356 | infrastructure. Ed also implemented the packaging. 357 | * Alexander Kanevskiy (alexander.kanevskiy@intel.com) and 358 | Kevin Wang (kevin.a.wang@intel.com) for helping with integrating this stuff 359 | to the Tizen IVI infrastructure. 360 | * Simon McVittie (simon.mcvittie@collabora.co.uk) for improving Debian 361 | packaging and fixing bmaptool. 362 | -------------------------------------------------------------------------------- /contrib/bmap_write.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # A super-simple standalone script (works with both Python2 / Python3 and has 4 | # no external dependencies) to show how easily .bmap files can be parsed. 5 | # (Also demonstrates how little code it takes - which might be a useful starting 6 | # point for other languages) 7 | # 8 | # This is effectively a minimal version of 'bmaptool copy'. It only supports 9 | # uncompressed images, it does no verification, and if the image is named 10 | # mydata.img it assumes the corresponding bmap is named mydata.bmap 11 | 12 | # Copyright (C) 2018 Andrew Scheller 13 | # 14 | # This program is free software; you can redistribute it and/or modify 15 | # it under the terms of the GNU General Public License as published by 16 | # the Free Software Foundation; either version 2 of the License, or 17 | # (at your option) any later version. 18 | # 19 | # This program is distributed in the hope that it will be useful, 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 | # GNU General Public License for more details. 23 | # 24 | # You should have received a copy of the GNU General Public License along 25 | # with this program; if not, write to the Free Software Foundation, Inc., 26 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 27 | 28 | import sys 29 | import xml.etree.ElementTree as ET 30 | import re 31 | import os 32 | 33 | if len(sys.argv) != 3: 34 | print("Usage: %s " % os.path.basename(sys.argv[0])) 35 | sys.exit(1) 36 | raw_file = sys.argv[1] 37 | output_file = sys.argv[2] 38 | if not os.path.isfile(raw_file): 39 | print("raw-file '%s' doesn't exist" % raw_file) 40 | sys.exit(1) 41 | file_root, file_ext = os.path.splitext(raw_file) 42 | bmap_file = file_root + ".bmap" 43 | if not os.path.isfile(bmap_file): 44 | print("bmap-file '%s' doesn't exist" % bmap_file) 45 | sys.exit(1) 46 | 47 | bmap_root = ET.parse(bmap_file).getroot() 48 | blocksize = int(bmap_root.find("BlockSize").text) 49 | with open(raw_file, "rb") as filedata: 50 | with open(output_file, "wb") as outdata: 51 | try: 52 | outdata.truncate(int(bmap_root.find("ImageSize").text)) # optional 53 | except: 54 | pass 55 | for bmap_range in bmap_root.find("BlockMap").findall("Range"): 56 | blockrange = bmap_range.text 57 | m = re.match("^\s*(\d+)\s*-\s*(\d+)\s*$", blockrange) 58 | if m: 59 | start = int(m.group(1)) 60 | end = int(m.group(2)) 61 | else: 62 | start = int(blockrange) 63 | end = start 64 | start_offset = start * blocksize 65 | filedata.seek(start_offset, 0) 66 | outdata.seek(start_offset, 0) 67 | for i in range(end - start + 1): 68 | outdata.write(filedata.read(blocksize)) 69 | outdata.flush() 70 | os.fsync(outdata.fileno()) 71 | -------------------------------------------------------------------------------- /debian/bmaptool.docs: -------------------------------------------------------------------------------- 1 | CHANGELOG.md 2 | README.md 3 | -------------------------------------------------------------------------------- /debian/bmaptool.install: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /debian/bmaptool.manpages: -------------------------------------------------------------------------------- 1 | docs/man1/bmaptool.1 2 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | bmaptool (3.9.0) unstable; urgency=low 2 | 3 | * copy: add `--removable-device`, `--keyring` and `--fingerprint` options 4 | * Respect query part of the url when operating on the path 5 | * support FTP authentication 6 | * rework GPG tests 7 | 8 | -- Trevor Woerner Fri, 14 Mar 2025 13:46:05 -0600 9 | 10 | bmaptool (3.8.0) unstable; urgency=low 11 | 12 | * use 'df -P' for POSIX portable output 13 | * bmaptool has new maintainers 14 | * bmaptool has a new home 15 | * bmaptool is now only called 'bmaptool' and not one of a dozen such 16 | variations 17 | * switch to use an X.Y.Z versioning number scheme 18 | 19 | -- Trevor Woerner Mon, 18 Mar 2024 23:44:10 -0400 20 | 21 | bmap-tools (3.7) unstable; urgency=low 22 | 23 | * Use GitHub Actions for CI (#109) 24 | * Add `poetry` for dependency management and `black` for code formatting 25 | (#104) 26 | * Add functionality for copying from standard input (#99) 27 | * Switch from gpg to gpgme module (#103) 28 | 29 | -- Artem Bityutskiy Wed, 02 Aug 2023 15:11:26 +0300 30 | 31 | bmap-tools (3.6) unstable; urgency=low 32 | 33 | * Improve ZFS compatibility. 34 | * Added the 'zstd' compression type support. 35 | * Add '--psplash-pipe' option for interacting with psplash. 36 | 37 | -- Artem Bityutskiy Tue, 02 Feb 2021 14:08:41 +0200 38 | 39 | bmap-tools (3.5) unstable; urgency=low 40 | 41 | * Fixed copying of compressed files from URLs 42 | * Python 3.x support fixes and improvements. 43 | 44 | -- Artem Bityutskiy Thu, 23 Aug 2018 10:34:31 +0300 45 | 46 | bmap-tools (3.4) unstable; urgency=low 47 | 48 | * New homepage: https://github.com/01org/bmap-tools 49 | * Python 3.x support. 50 | * bmaptool can now be shipped as standalone application. 51 | * Added support for ZIP archives. 52 | * Added support for LZ4 archives. 53 | * Fixed bugs related to specific filesystems. 54 | 55 | -- Alexander Kanevskiy Thu, 31 Aug 2017 15:40:12 +0300 56 | 57 | bmap-tools (3.2) unstable; urgency=low 58 | 59 | * Add support for LZO and archives ('.lzo' and '.tar.lzo'). 60 | * Add support for multi-stream bzip2 archives (creted with "pbzip2"). 61 | * Support tmpfs by using the SEEK_HOLE method instead of FIEMAP. 62 | * Use external tools like 'gzip' and 'bzip2' for decompressing, instead of 63 | using internal python libraries. 64 | 65 | -- Artem Bityutskiy Wed, 19 Feb 2014 16:50:12 +0200 66 | 67 | bmap-tools (3.2~rc2) unstable; urgency=low 68 | 69 | * Bump the version number to 3.2~rc2. 70 | 71 | -- Artem Bityutskiy Fri, 31 Jan 2014 12:54:42 +0200 72 | 73 | bmap-tools (3.1) unstable; urgency=low 74 | 75 | * Change bmap format version from 1.4 to 2.0, because there are incompatible 76 | changes in 1.4 comparing to 1.3, so the right version number is 2.0 77 | * Add backward and forward bmap format compatibility unit-tests 78 | 79 | -- Artem Bityutskiy Thu, 07 Nov 2013 17:26:57 +0200 80 | 81 | bmap-tools (3.0) unstable; urgency=low 82 | 83 | * Switch from using SHA1 for checksumming to SHA256. 84 | * Start supporting OpenPGP signatures. Both detached and clearsign signatures 85 | are supported. 86 | * Always sync the image file before creating the bmap for it, to work-around 87 | kernel bugs in early FIEMAP implementations. 88 | 89 | -- Artem Bityutskiy Wed, 02 Oct 2013 09:30:22 +0300 90 | 91 | bmap-tools (2.6) unstable; urgency=low 92 | 93 | * Add support for on-the-fly decompression of '.xz' and '.tar.xz' files. 94 | 95 | -- Artem Bityutskiy Tue, 13 Aug 2013 14:53:49 +0300 96 | 97 | bmap-tools (2.5) unstable; urgency=low 98 | 99 | * Do not fail when lacking permisssions for accessing block device's sysfs 100 | files. 101 | * Improve debian packaging. 102 | 103 | -- Artem Bityutskiy Mon, 05 Aug 2013 10:05:09 +0300 104 | 105 | bmap-tools (2.4) unstable; urgency=low 106 | 107 | * Add support for ssh:// URLs. 108 | 109 | -- Artem Bityutskiy Wed, 05 Jun 2013 18:15:41 +0300 110 | 111 | bmap-tools (2.3) unstable; urgency=low 112 | 113 | * Add bmap file SHA1 verification, make tests work on btrfs. 114 | 115 | -- Artem Bityutskiy Mon, 06 May 2013 10:58:32 +0300 116 | 117 | bmap-tools (2.2) unstable; urgency=low 118 | 119 | * Support username and password in URLs. 120 | 121 | -- Artem Bityutskiy Mon, 11 Mar 2013 14:40:17 +0200 122 | 123 | bmap-tools (2.1) unstable; urgency=low 124 | 125 | * Fix out of memory issues when copying .bz2 files. 126 | 127 | -- Artem Bityutskiy Mon, 18 Feb 2013 16:38:32 +0200 128 | 129 | bmap-tools (2.0) unstable; urgency=low 130 | 131 | * Fix the an issue with running out of memory in TransRead.py. 132 | 133 | -- Artem Bityutskiy Thu, 17 Jan 2013 11:33:15 +0200 134 | 135 | bmap-tools (2.0~rc5) unstable; urgency=low 136 | 137 | * When block device optimzations fail - raise an exception except of muting 138 | the error, because we really want to know about these failures and possibly 139 | fix them. 140 | 141 | -- Artem Bityutskiy Tue, 15 Jan 2013 14:51:27 +0200 142 | 143 | bmap-tools (2.0~rc4) unstable; urgency=low 144 | 145 | * Fix bmap autodiscovery. 146 | 147 | -- Artem Bityutskiy Thu, 10 Jan 2013 13:58:07 +0200 148 | 149 | bmap-tools (2.0~rc3) unstable; urgency=low 150 | 151 | * Fix uncaught urllib2 exception bug introduced in rc1. 152 | 153 | -- Artem Bityutskiy Mon, 07 Jan 2013 10:19:49 +0200 154 | 155 | bmap-tools (2.0~rc2) unstable; urgency=low 156 | 157 | * Fix writing to block devices, which was broken in rc1. 158 | * Make the informational messages a bit nicer. 159 | 160 | -- Artem Bityutskiy Fri, 04 Jan 2013 09:52:41 +0200 161 | 162 | bmap-tools (2.0~rc1) unstable; urgency=low 163 | 164 | * Allow copying without bmap only if --nobmap was specified. 165 | * Auto-discover the bmap file. 166 | * Support reading from URLs. 167 | * Implement progress bar. 168 | * Highlight error and warning messages with red and yellow labels. 169 | 170 | -- Artem Bityutskiy Thu, 20 Dec 2012 10:47:00 +0200 171 | 172 | bmap-tools (1.0) unstable; urgency=low 173 | 174 | * Release version 1.0 of the tools - almost identical to 1.0~rc7 except of few 175 | minor differences like spelling fixes. 176 | 177 | -- Artem Bityutskiy Mon, 03 Dec 2012 10:00:33 +0200 178 | 179 | bmap-tools (1.0~rc7) unstable; urgency=low 180 | 181 | * Add a Fiemap.py module which implements python API to the linux FIEMAP ioct. 182 | * Use the FIEMAP ioctl properly and optimally. 183 | * Add unit-tests, current test coverage is 66%. 184 | * A lot of core rerafactoring. 185 | * Several bug fixes in 'BmapCopy' (e.g., .tar.gz format support was broken). 186 | * Add README and RELEASE_NOTES files. 187 | 188 | -- Artem Bityutskiy Thu, 29 Nov 2012 12:29:39 +0200 189 | 190 | bmap-tools (0.6) unstable; urgency=low 191 | 192 | * Improve the base API test to cover the case when there is no bmap. 193 | * Fix a bug when copying without bmap. 194 | 195 | -- Artem Bityutskiy Wed, 21 Nov 2012 16:43:49 +0200 196 | 197 | bmap-tools (0.5) unstable; urgency=low 198 | 199 | * Fix handling of bmap files which contain ranges with only one block. 200 | * Restore the block device settings which we change on exit. 201 | * Change block device settings correctly for partitions. 202 | * Rework API modules to accept file-like objects, not only paths. 203 | * Fix and silence pylint warnings. 204 | * Implement the base API test-case. 205 | 206 | -- Artem Bityutskiy Tue, 20 Nov 2012 15:40:30 +0200 207 | 208 | bmap-tools (0.4) unstable; urgency=low 209 | 210 | * Improved compressed images flashing speed by exploiting multiple threads: 211 | now we read/decompress the image in one thread and write it in a different 212 | thread. 213 | 214 | -- Artem Bityutskiy Wed, 14 Nov 2012 12:35:06 +0200 215 | 216 | bmap-tools (0.3) unstable; urgency=low 217 | 218 | * Fix flashing speed calculations 219 | * Fix the Ctrl-C freeze issue - now we synchronize the block device 220 | periodically so if a Ctrl-C interruption happens, we terminate withen few 221 | seconds. 222 | 223 | -- Artem Bityutskiy Tue, 13 Nov 2012 10:56:11 +0200 224 | 225 | bmap-tools (0.2) unstable; urgency=low 226 | 227 | * Release 0.2 - mostly internal code re-structuring and renamings, 228 | not much functional changes. 229 | * The 'bmap-flasher' and 'bmap-creator' tools do not exist anymore. Now 230 | we have 'bmaptool' which supports 'copy' and 'create' sub-commands instead. 231 | * The BmapFlasher module was also re-named to BmapCopy. 232 | 233 | -- Artem Bityutskiy Fri, 09 Nov 2012 12:20:37 +0200 234 | 235 | bmap-tools (0.1.1) unstable; urgency=low 236 | 237 | * Release 0.1.1 - a lot of fixes and speed improvements. 238 | 239 | -- Artem Bityutskiy Wed, 07 Nov 2012 11:36:29 +0200 240 | 241 | bmap-tools (0.1.0) unstable; urgency=low 242 | 243 | * Initial release. 244 | 245 | -- Ed Bartosh Sun, 27 Oct 2012 22:31:28 +0300 246 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: bmaptool 2 | Maintainer: Trevor Woerner 3 | Section: utils 4 | Priority: optional 5 | Build-Depends: 6 | debhelper-compat (= 13), 7 | dh-sequence-python3, 8 | pybuild-plugin-pyproject, 9 | python3 (>= 3.8), 10 | python3-gpg, 11 | python3-hatchling, 12 | python3-pytest, 13 | Standards-Version: 4.7.0 14 | Homepage: https://github.com/yoctoproject/bmaptool 15 | 16 | Package: bmaptool 17 | Architecture: all 18 | Depends: 19 | python3, 20 | python3-six, 21 | ${misc:Depends}, 22 | ${python3:Depends}, 23 | Recommends: 24 | bzip2, 25 | lzop, 26 | xz-utils, 27 | Suggests: 28 | lz4, 29 | pbzip2, 30 | pigz, 31 | python3-gpg, 32 | unzip, 33 | Description: Tools to generate block map (AKA bmap) and flash images using 34 | bmap. bmaptool is a generic tool for creating the block map (bmap) for a file, 35 | and copying files using the block map. The idea is that large file containing 36 | unused blocks, like raw system image files, can be copied or flashed a lot 37 | faster with bmaptool than with traditional tools like "dd" or "cp". See 38 | source.tizen.org/documentation/reference/bmaptool for more information. 39 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-name: bmaptool 3 | Upstream-Contact: Trevor Woerner 4 | Source: https://github.com/yoctoproject/bmaptool 5 | . 6 | The initial package was put together by Ed Bartosh 7 | on Sun Oct 27 22:32:19 EEST 2012. 8 | 9 | Files: * 10 | Copyright: © 2012-2013 Intel, Inc. 11 | License: GPL-2 12 | 13 | Files: debian/* 14 | Copyright: © 2012-2013 Intel, Inc. 15 | License: GPL-2 16 | 17 | License: GPL-2 18 | This program is free software; you can redistribute it and/or modify 19 | it under the terms of the GNU General Public License, version 2, 20 | as published by the Free Software Foundation. 21 | . 22 | This program is distributed in the hope that it will be useful, but 23 | WITHOUT ANY WARRANTY; without even the implied warranty of 24 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 25 | General Public License for more details. 26 | Comment: 27 | On Debian systems, the full text of the GPL v2 can be found 28 | in /usr/share/common-licenses/GPL-2. 29 | -------------------------------------------------------------------------------- /debian/manpages: -------------------------------------------------------------------------------- 1 | docs/man1/bmaptool.1 2 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | %: 4 | dh $@ --buildsystem=pybuild 5 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (native) 2 | -------------------------------------------------------------------------------- /docs/RELEASE_NOTES: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoctoproject/bmaptool/618a7316102f6f81faa60537503012a419eafa06/docs/RELEASE_NOTES -------------------------------------------------------------------------------- /docs/TODO.md: -------------------------------------------------------------------------------- 1 | Current TODO list, any help with these is appreciated. 2 | 3 | 1. Teach bmaptool to update the alternate GPT partition 4 | 2. Add a test for bmap with invalid checksums 5 | 3. When writing to a file, and the file did not exist, so we create it, 6 | and then fail, we do not remove the half-written file. 7 | 4. Teach make_a_release.sh to modify the version in the 'doc/man1/bmaptool.1' 8 | file too. 9 | 5. Use __author__ and __version__ in bmaptool, and import them from 10 | 'setup.py' 11 | 6. Move all the documentation from tizen.org to 01.org 12 | 7. Make sure the web documentation describes all features of releases starting 13 | from 3.1. 14 | -------------------------------------------------------------------------------- /docs/man1/bmaptool.1: -------------------------------------------------------------------------------- 1 | .TH BMAPTOOL "1" "March 2025" "bmaptool 3.9.0" "User Commands" 2 | 3 | .SH NAME 4 | 5 | .PP 6 | bmaptool - create block map (bmap) for a file or copy a file using bmap 7 | 8 | .SH SYNOPSIS 9 | 10 | .PP 11 | .B bmaptool 12 | [\-\-help] [\-\-version] [\-\-quiet] [\-\-debug] [] 13 | 14 | .SH DESCRIPTION 15 | 16 | .PP 17 | \fIbmaptool\fR is a generic tool for creating the block map (bmap) for a file and 18 | copying files using the block map. The idea is that large files, like raw 19 | system image files, can be copied or flashed a lot faster with \fIbmaptool\fR than 20 | with traditional tools, like "dd" or "cp". 21 | 22 | .PP 23 | \fIbmaptool\fR supports 2 commands: 24 | .RS 2 25 | 1. \fBcopy\fR - copy a file to another file using bmap or flash an image to a block device 26 | .RE 27 | .RS 2 28 | 2. \fBcreate\fR - create a bmap for a file 29 | .RE 30 | 31 | .PP 32 | Please, find full documentation for the project online. 33 | 34 | .\" =========================================================================== 35 | .\" Global options 36 | .\" =========================================================================== 37 | .SH OPTIONS 38 | 39 | .PP 40 | \-\-version 41 | .RS 2 42 | Print \fIbmaptool\fR version and exit. 43 | .RE 44 | 45 | .PP 46 | \-h, \-\-help 47 | .RS 2 48 | Print short help text and exit. 49 | .RE 50 | 51 | .PP 52 | \-q, \-\-quiet 53 | .RS 2 54 | Be quiet, do not print extra information. 55 | .RE 56 | 57 | .PP 58 | \-d, \-\-debug 59 | .RS 2 60 | Print debugging messages. 61 | .RE 62 | 63 | .\" =========================================================================== 64 | .\" Commands descriptions 65 | .\" =========================================================================== 66 | .SH COMMANDS 67 | 68 | .\" 69 | .\" The "copy" command description 70 | .\" 71 | .SS \fBcopy\fR [options] IMAGE DEST 72 | 73 | .RS 2 74 | Copy file IMAGE to the destination regular file or block device DEST 75 | using bmap. IMAGE may either be a local path or an URL. DEST may either 76 | be a regular file or a block device (only local). 77 | 78 | .PP 79 | Unless the bmap file is explicitly specified with the "--bmap" option, \fIbmaptool\fR 80 | automatically discovers it by looking for a file with the same name as IMAGE 81 | but with the ".bmap" extension. If it was unable to find it that way, it will 82 | try filenames with each extension of IMAGE removed and ".bmap" added to it. So 83 | if your IMAGE is named \fIdisk.img.gz\fR, it will first try 84 | \fIdisk.img.gz.bmap\fR, then \fIdisk.img.bmap\fR and finally \fIdisk.bmap\fR. 85 | The bmap file is only looked for in 86 | IMAGE's directory (or base URL, in case IMAGE was specified as an URL). If the 87 | bmap file is not found, \fIbmaptool\fR fails. To copy without bmap, use 88 | the "--nobmap" option. 89 | 90 | .PP 91 | Both IMAGE and the bmap file may be specified as an URL (http://, ftp://, 92 | https://, file://, ssh://). In order to make \fIbmaptool\fR use a proxy server, 93 | please, specify the proxy using the standard "$http_proxy", "$https_proxy", 94 | "$ftp_proxy" or "$no_proxy" environment variables. 95 | 96 | .PP 97 | If the server requires authentication, user name and password may be specified 98 | in the URL, for example "https://user:password@my.server.org/image.raw.bz2", or 99 | "ssh://user:password@host:path/to/image.raw". 100 | 101 | .PP 102 | IMAGE may be compressed, in which case \fIbmaptool\fR decompresses it on-the-fly. 103 | The compression type is detected by the file extension and the following 104 | extensions are supported: 105 | 106 | .RS 4 107 | 1. ".gz", ".gzip", ".tar.gz" and ".tgz" for files and tar archives compressed with "\fIgzip\fR" program 108 | .RE 109 | .RS 4 110 | 2. ".bz2", "tar.bz2", ".tbz2", ".tbz", and ".tb2" for files and tar archives compressed with "\fIbzip2\fR" program 111 | .RE 112 | .RS 4 113 | 3. ".xz", ".tar.xz", ".txz" for files and tar archives compressed with "\fIxz\fR" program 114 | .RE 115 | .RS 4 116 | 4. ".lzo", "tar.lzo", ".tzo" for files and tar archives compressed with "\fIlzo\fR" program 117 | .RE 118 | .RS 4 119 | 5. ".lz4", "tar.lz4", ".tlz4" for files and tar archives compressed with "\fIlz4\fR" program 120 | .RE 121 | .RS 4 122 | 6. ".zst", "tar.zst", ".tzst" for files and tar archives compressed with "\fIzstd\fR" program 123 | .RE 124 | 125 | .PP 126 | IMAGE files with other extensions are assumed to be uncompressed. Note, 127 | \fIbmaptool\fR uses "\fIpbzip2\fR" and "\fIpigz\fR" programs for decompressing 128 | bzip2 and gzip archives faster, unless they are not available, in which case if 129 | falls-back to using "\fIbzip2\fR" and "\fIgzip\fR". Furthermore, uncompressed 130 | IMAGE files can be piped to the standard input using "-". 131 | 132 | .PP 133 | If DEST is a block device node (e.g., "/dev/sdg"), \fIbmaptool\fR opens it in 134 | exclusive mode. This means that it will fail if any other process has IMAGE 135 | block device node opened. This also means that no other processes will be able 136 | to open IMAGE until \fIbmaptool\fR finishes the copying. Please, see semantics 137 | of the "O_EXCL" flag of the "open()" syscall. 138 | 139 | .PP 140 | The bmap file typically contains SHA-256 checksum for itself as well as SHA-256 141 | checksum for all the mapped data regions, which makes it possible to guarantee 142 | data integrity. \fIbmaptool\fR verifies the checksums and exits with an error 143 | in case of a mismatch. Checksum verification can be disabled using the 144 | "--no-verify" option. \fIbmaptool\fR does not verify that unampped areas 145 | contain only zeroes, because these areas are anyway dropped and are not used for 146 | anything. 147 | 148 | .PP 149 | The bmap file may be signed with OpenPGP (gpg). The signature may be either 150 | detached (a separate file) or "built into" the bmap file (so called "clearsign" 151 | signature). 152 | 153 | .PP 154 | The detached signature can be specified with the "--bmap-sig" option, otherwise 155 | \fIbmaptool\fR tries to automatically discover it by looking for a file with 156 | the same name as the bmap file but with the ".asc" or ".sig" extension. 157 | If it was unable to find it that way, it will try filenames with each extension 158 | of IMAGE removed and ".asc" or ".sig" added to it. 159 | This is very similar to the bmap file auto-discovery. So if a ".asc" or ".sig" 160 | file exists, \fIbmaptool\fR will verify the signature. 161 | 162 | .PP 163 | The clearsign signature is part of the bmap file and \fIbmaptool\fR 164 | automatically detected and verifies it. 165 | 166 | .PP 167 | If the signature is bad, \fIbmaptool\fR exits with an error. Bmap file 168 | signature verification can be disabled using the "--no-sig-verify" option. 169 | .RE 170 | 171 | .\" 172 | .\" The "copy" command's options 173 | .\" 174 | .RS 2 175 | \fBOPTIONS\fR 176 | .RS 2 177 | \-h, \-\-help 178 | .RS 2 179 | Print short help text about the "copy" command and exit. 180 | .RE 181 | 182 | .PP 183 | \-\-bmap BMAP 184 | .RS 2 185 | Use bmap file "BMAP" for copying. If this option is not specified, \fIbmaptool\fR 186 | tries to automatically discover the bmap file. 187 | .RE 188 | 189 | .PP 190 | \-\-bmap-sig SIG 191 | .RS 2 192 | Use a detached OpenPGP signature file "SIG" for verifying the bmap file 193 | integrity and publisher. If this option is not specified, \fIbmaptool\fR 194 | tries to automatically discover the signature file. 195 | .RE 196 | 197 | .PP 198 | \-\-fingerprint FINGERPRINT 199 | .RS 2 200 | The GPG fingerprint which you expect to have signed the bmap file. 201 | .RE 202 | 203 | .PP 204 | \-\-keyring KEYRING 205 | .RS 2 206 | Path to the GPG keyring that will be used when verifying GPG signatures. 207 | .RE 208 | 209 | .PP 210 | \-\-nobmap 211 | .RS 2 212 | Disable automatic bmap file discovery and force flashing entire IMAGE without bmap. 213 | .RE 214 | 215 | .PP 216 | \-\-no-sig-verify 217 | .RS 2 218 | Do not verify the OpenPGP bmap file signature (not recommended). 219 | .RE 220 | 221 | .PP 222 | \-\-no-verify 223 | .RS 2 224 | Do not verify data checksums when copying (not recommended). The checksums are 225 | stored in the bmap file, and normally \fIbmaptool\fR verifies that the data in 226 | IMAGE matches the checksums. 227 | .RE 228 | 229 | .PP 230 | \-\-psplash\-pipe PATH 231 | .RS 2 232 | Write periodic machine-readable progress reports to a fifo in the format 233 | used by \fBpsplash\fR. Each progress report consists of "PROGRESS" followed 234 | by a space, an integer percentage and a newline. 235 | .RE 236 | 237 | .PP 238 | \-\-removable\-device 239 | .RS 2 240 | Copy to destination only if it is a removable block device. This option is 241 | recommended when writing on SD Card or USB key to avoid involuntary 242 | destructive operations on non-removable disks. The copy command fails when the 243 | destination file does not exist, is not a block device or is not removable. 244 | .RE 245 | 246 | .RE 247 | .RE 248 | 249 | .\" 250 | .\" The "copy" command's examples 251 | .\" 252 | .RS 2 253 | \fBEXAMPLES\fR 254 | .RS 2 255 | \fIbmaptool\fR copy image.raw.bz2 /dev/sdg 256 | .RS 2 257 | Copy bz2-compressed local file "image.raw.bz2" to block device "/dev/sdg". The 258 | image file is uncompressed on-the-fly. The bmap file is discovered 259 | automatically. The OpenPGP signature is detected/discovered automatically 260 | too. 261 | .RE 262 | .RE 263 | 264 | .RS 2 265 | \fIbmaptool\fR copy http://my-server.com/files/image.raw.bz2 $HOME/tmp/file 266 | .RS 2 267 | Copy bz2-compressed remote "image.raw.bz2" to regular file "$HOME/tmp/file". 268 | The image file is uncompressed on-the-fly. The bmap file is discovered 269 | automatically. The OpenPGP signature is detected/discovered automatically 270 | too. 271 | .RE 272 | .RE 273 | 274 | .RS 2 275 | \fIbmaptool\fR copy --bmap image.bmap --bmap-sig image.bmap.asc image.raw /dev/sdg 276 | .RS 2 277 | Copy non-compressed local file "image.raw" to block device "/dev/sdg" using bmap file 278 | "image.bmap". Verify the bmap file signature using a detached OpenPGP signature 279 | from "imag.bmap.asc". 280 | .RE 281 | .RE 282 | 283 | .RS 2 284 | cat image.raw | \fIbmaptool\fR copy --bmap image.bmap - /dev/sdg 285 | .RS 2 286 | Copy non-compressed image from standard input to block device "/dev/sdg" using bmap file 287 | "image.bmap". 288 | .RE 289 | .RE 290 | 291 | .\" 292 | .\" The "create" command description 293 | .\" 294 | .SS \fBcreate\fR [options] IMAGE 295 | 296 | .PP 297 | Generate bmap for a regular file IMAGE. Internally, this command uses the 298 | Linux "FIEMAP" ioctl to find out which IMAGE blocks are mapped. However, if 299 | "FIEMAP" is not supported, the "SEEK_HOLE" feature of the "lseek" system call 300 | is used instead. By default, the resulting bmap file is printed to stdout, 301 | unless the "--output" option is used. 302 | 303 | .PP 304 | The IMAGE file is always synchronized before the block map is generated. And it 305 | is important to make sure that the IMAGE file is not modified when the bmap 306 | file is being generated, and after the bmap file has been generated. Otherwise 307 | the bmap file becomes invalid and checksum verification will fail. 308 | 309 | .PP 310 | The image file can further be signed using OpenPGP. 311 | 312 | .\" 313 | .\" The "create" command's options 314 | .\" 315 | .RS 2 316 | \fBOPTIONS\fR 317 | .RS 2 318 | \-h, \-\-help 319 | .RS 2 320 | Print short help text about the "create" command and exit. 321 | .RE 322 | 323 | .PP 324 | \-o, \-\-output OUTPUT 325 | .RS 2 326 | Save the generated bmap in the OUTPUT file (by default the bmap is printed to 327 | stdout). 328 | .RE 329 | 330 | .PP 331 | \-\-no-checksum 332 | .RS 2 333 | Generate a bmap file without SHA1 checksums (not recommended). 334 | .RE 335 | .RE 336 | .RE 337 | 338 | .\" 339 | .\" The "create" command's examples 340 | .\" 341 | .RS 2 342 | \fBEXAMPLES\fR 343 | .RS 2 344 | \fIbmaptool\fR create image.raw 345 | .RS 2 346 | Generate bmap for the "image.raw" file and print it to stdout. 347 | .RE 348 | .RE 349 | 350 | .RS 2 351 | \fIbmaptool\fR create -o image.bmap image.raw 352 | .RS 2 353 | Generate bmap for the "image.raw" file and save it in "image.bmap". 354 | .RE 355 | .RE 356 | 357 | .SH AUTHOR 358 | 359 | Artem Bityutskiy . 360 | 361 | .SH Maintainers 362 | 363 | Trevor Woerner 364 | Joshua Watt 365 | Tim Orling 366 | 367 | .SH REPORTING BUGS 368 | 369 | This project is hosted on github; please use it to report any issues or 370 | post any patches: https://github.com/yoctoproject/bmaptool 371 | -------------------------------------------------------------------------------- /make_a_release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -euf 2 | # 3 | # Copyright (c) 2012-2013 Intel, Inc. 4 | # License: GPLv2 5 | # Author: Artem Bityutskiy 6 | # 7 | # This program is free software; you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License, version 2, 9 | # as published by the Free Software Foundation. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | 16 | # This script automates the process of releasing the bmaptool project. The 17 | # idea is that it should be enough to run this script with few parameters and 18 | # the release is ready. 19 | 20 | # 21 | # This script is supposed to be executed in the root of the bmaptool 22 | # project's source code tree. 23 | 24 | PROG="make_a_release.sh" 25 | 26 | fatal() { 27 | printf "Error: %s\n" "$1" >&2 28 | exit 1 29 | } 30 | 31 | usage() { 32 | cat < 34 | 35 | - new bmaptool version to make in X.Y.Z format 36 | EOF 37 | exit 0 38 | } 39 | 40 | ask_question() { 41 | local question=$1 42 | 43 | while true; do 44 | printf "%s\n" "$question (yes/no)?" 45 | IFS= read answer 46 | if [ "$answer" = "yes" ]; then 47 | printf "%s\n" "Very good!" 48 | return 49 | elif [ "$answer" = "no" ]; then 50 | printf "%s\n" "Please, do that!" 51 | exit 1 52 | else 53 | printf "%s\n" "Please, answer \"yes\" or \"no\"" 54 | fi 55 | done 56 | } 57 | 58 | format_changelog() { 59 | local logfile="$1"; shift 60 | local pfx1="$1"; shift 61 | local pfx2="$1"; shift 62 | local pfx_len="$(printf "%s" "$pfx1" | wc -c)" 63 | local width="$((80-$pfx_len))" 64 | 65 | while IFS= read -r line; do 66 | printf "%s\n" "$line" | fold -s -w "$width" | \ 67 | sed -e "1 s/^/$pfx1/" | sed -e "1! s/^/$pfx2/" | \ 68 | sed -e "s/[\t ]\+$//" 69 | done < "$logfile" 70 | } 71 | 72 | [ $# -eq 0 ] && usage 73 | [ $# -eq 1 ] || fatal "insufficient or too many arguments" 74 | 75 | new_ver="$1"; shift 76 | 77 | # Validate the new version 78 | printf "%s" "$new_ver" | egrep -q -x '[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+' || 79 | fatal "please, provide new version in X.Y.Z format" 80 | 81 | # Make sure the git index is up-to-date 82 | [ -z "$(git status --porcelain)" ] || fatal "git index is not up-to-date" 83 | 84 | # Remind the maintainer about various important things 85 | ask_question "Did you update the man page" 86 | ask_question "Did you update tests: test-data and oldcodebase" 87 | 88 | # Change the version in the 'bmaptool/CLI.py' file 89 | sed -i -e "s/^VERSION = \"[0-9]\+\.[0-9]\+\.[0-9]\+\"$/VERSION = \"$new_ver\"/" src/bmaptool/CLI.py 90 | # Sed the version in the RPM spec file 91 | sed -i -e "s/^Version: [0-9]\+\.[0-9]\+\.[0-9]\+$/Version: $new_ver/" packaging/bmaptool.spec 92 | # Remove the "rc_num" macro from the RPM spec file to make sure we do not have 93 | # the "-rcX" part in the release version 94 | sed -i -e '/^%define[[:blank:]]\+rc_num[[:blank:]]\+[[:digit:]]\+[[:blank:]]*$/d' packaging/bmaptool.spec 95 | # update man page title line (.TH) 96 | export MANVERSTR="\"bmaptool $new_ver\"" 97 | export MANDATE="\"$(date +"%B %Y")\"" 98 | sed -i -e "s/\.TH.*$/\.TH BMAPTOOL \"1\" $MANDATE $MANVERSTR \"User Commands\"/g" docs/man1/bmaptool.1 99 | 100 | # Ask the maintainer for changelog lines 101 | logfile="$(mktemp -t "$PROG.XXXX")" 102 | cat > "$logfile" < "$deblogfile" 120 | format_changelog "$logfile" " * " " " >> "$deblogfile" 121 | printf "\n%s\n\n" " -- Trevor Woerner $(date -R)" >> "$deblogfile" 122 | cat debian/changelog >> "$deblogfile" 123 | mv "$deblogfile" debian/changelog 124 | 125 | # Prepare RPM changelog 126 | rpmlogfile="$(mktemp -t "$PROG.XXXX")" 127 | printf "%s\n" "$(date --utc) - Trevor Woerner ${new_ver}-1" > "$rpmlogfile" 128 | format_changelog "$logfile" "- " " " >> "$rpmlogfile" 129 | printf "\n" >> "$rpmlogfile" 130 | cat packaging/bmaptool.changes >> "$rpmlogfile" 131 | mv "$rpmlogfile" packaging/bmaptool.changes 132 | 133 | rm "$logfile" 134 | 135 | # Commit the changes 136 | git commit -a -s -m "Release version $new_ver" 137 | 138 | outdir="." 139 | tag_name="v$new_ver" 140 | release_name="bmaptool-$new_ver" 141 | 142 | # Get the name of the release branch corresponding to this version 143 | release_branch="release-$(printf "%s" "$new_ver" | sed -e 's/\(.*\)\..*/\1.0/')" 144 | 145 | cat < 3.9.0-1 2 | - copy: add `--removable-device`, `--keyring` and `--fingerprint` options 3 | - Respect query part of the url when operating on the path 4 | - support FTP authentication 5 | - rework GPG tests 6 | 7 | Tue Mar 19 03:44:10 UTC 2024 - Trevor Woerner 3.8.0-1 8 | - use 'df -P' for POSIX portable output 9 | - bmaptool has new maintainers 10 | - bmaptool has a new home 11 | - bmaptool is now only called 'bmaptool' and not one of a dozen such variations 12 | - switch to use an X.Y.Z versioning number scheme 13 | 14 | Wed Aug 2 12:11:26 PM UTC 2023 - Artem Bityutskiy 3.7-1 15 | - Use GitHub Actions for CI (#109) 16 | - Add `poetry` for dependency management and `black` for code formatting (#104) 17 | - Add functionality for copying from standard input (#99) 18 | - Switch from gpg to gpgme module (#103) 19 | 20 | Tue 02 Feb 2021 12:08:41 PM UTC - Artem Bityutskiy 3.6-1 21 | - Improve ZFS compatibility. 22 | - Added the 'zstd' compression type support. 23 | - Add '--psplash-pipe' option for interacting with psplash. 24 | 25 | Thu Aug 23 07:34:31 UTC 2018 - Artem Bityutskiy 3.5-1 26 | - Fixed copying of compressed files from URLs 27 | - Python 3.x support fixes and improvements. 28 | 29 | Thu Aug 31 12:40:00 UTC 2017 Alexander Kanevskiy 3.4-1 30 | - New homepage: https://github.com/01org/bmap-tools 31 | - Python 3.x support. 32 | - bmaptool can now be shipped as standalone application. 33 | - Added support for ZIP archives. 34 | - Added support for LZ4 archives. 35 | - Fixed bugs related to specific filesystems. 36 | 37 | Wed Feb 19 14:50:12 UTC 2014 - Artem Bityutskiy 3.2-1 38 | - Add support for LZO and archives ('.lzo' and '.tar.lzo'). 39 | - Add support for multi-stream bzip2 archives (creted with "pbzip2"). 40 | - Support tmpfs by using the SEEK_HOLE method instead of FIEMAP. 41 | - Use external tools like 'gzip' and 'bzip2' for decompressing, instead of 42 | using internal python libraries. 43 | 44 | Thu Nov 7 15:26:57 UTC 2013 - Artem Bityutskiy 3.1-1 45 | - Change bmap format version from 1.4 to 2.0, because there are incompatible 46 | changes in 1.4 comparing to 1.3, so the right version number is 2.0 47 | - Add backward and forward bmap format compatibility unit-tests 48 | 49 | Wed Oct 2 06:30:22 UTC 2013 - Artem Bityutskiy 3.0-1 50 | - Switch from using SHA1 for checksumming to SHA256. 51 | - Start supporting OpenPGP signatures. Both detached and clearsign signatures 52 | are supported. 53 | - Always sync the image file before creating the bmap for it, to work-around 54 | kernel bugs in early FIEMAP implementations. 55 | 56 | Tue Aug 13 11:54:31 UTC 2013 - Artem Bityutskiy 2.6-1 57 | - Add support for on-the-fly decompression of '.xz' and '.tar.xz' files. 58 | 59 | Mon Aug 5 07:05:59 UTC 2013 - Artem Bityutskiy 2.5-1 60 | - Do not fail when lacking permisssions for accessing block device's sysfs 61 | files. 62 | - Improve debian packaging. 63 | 64 | Wed Jun 5 15:16:42 UTC 2013 - Artem Bityutskiy 2.4-1 65 | - Add ssh:// URLs support. 66 | 67 | Mon May 6 07:59:26 UTC 2013 - Artem Bityutskiy 2.3-1 68 | -Add bmap file SHA1 verification, make tests work on btrfs. 69 | 70 | Mon Mar 11 12:42:03 UTC 2013 - Artem Bityutskiy 2.2-1 71 | - Support username and password in URLs. 72 | 73 | Mon Feb 18 14:39:11 UTC 2013 - Artem Bityutskiy 2.1-1 74 | - Fix out of memory issues when copying .bz2 files. 75 | 76 | Thu Jan 17 09:34:00 UTC 2013 - Artem Bityutskiy 2.0-1 77 | - Fix the an issue with running out of memory in TransRead.py. 78 | 79 | Tue Jan 15 12:52:25 UTC 2013 - Artem Bityutskiy 2.0-0.rc5 80 | - When block device optimzations fail - raise an exception except of muting 81 | the error, because we really want to know about these failures and possibly 82 | fix them. 83 | 84 | Thu Jan 10 11:58:57 UTC 2013 - Artem Bityutskiy 2.0-0.rc4 85 | - Fix bmap autodiscovery. 86 | 87 | Mon Jan 7 08:20:37 UTC 2013 - Artem Bityutskiy 2.0-0.rc3 88 | - Fix uncaught urllib2 exception bug introduced in rc1. 89 | 90 | Fri Jan 4 07:55:05 UTC 2013 - Artem Bityutskiy 2.0-0.rc2 91 | - Fix writing to block devices, which was broken in rc1. 92 | - Make the informational messages a bit nicer. 93 | 94 | Thu Dec 20 08:48:26 UTC 2012 - Artem Bityutskiy 2.0-0.rc1 95 | - Allow copying without bmap only if --nobmap was specified. 96 | - Auto-discover the bmap file. 97 | - Support reading from URLs. 98 | - Implement progress bar. 99 | - Highlight error and warning messages with red and yellow labels. 100 | 101 | Mon Dec 3 08:02:03 UTC 2012 - Artem Bityutskiy 1.0-1 102 | - Release version 1.0 of the tools - almost identical to 1.0-rc7 except of few 103 | minor differences like spelling fixes. 104 | 105 | Thu Nov 29 10:30:20 UTC 2012 - Artem Bityutskiy 1.0-0.rc7 106 | - Add a Fiemap.py module which implements python API to the linux FIEMAP ioct. 107 | - Use the FIEMAP ioctl properly and optimally. 108 | - Add unit-tests, current test coverage is 66%. 109 | - A lot of core rerafactoring. 110 | - Several bug fixes in 'BmapCopy' (e.g., .tar.gz format support was broken). 111 | - Add README and RELEASE_NOTES files. 112 | - Change the versioning scheme. 113 | 114 | Wed Nov 21 14:45:48 UTC 2012 - Artem Bityutskiy 0.6 115 | - Improve the base API test to cover the case when there is no bmap. 116 | - Fix a bug when copying without bmap. 117 | 118 | Tue Nov 20 15:40:30 UTC 2012 - Artem Bityutskiy 0.5 119 | - Fix handling of bmap files which contain ranges with only one block. 120 | - Restore the block device settings which we change on exit. 121 | - Change block device settings correctly for partitions. 122 | - Rework API modules to accept file-like objects, not only paths. 123 | - Fix and silence pylint warnings. 124 | - Implement the base API test-case. 125 | 126 | Wed Nov 14 10:36:10 UTC 2012 - Artem Bityutskiy 0.4 127 | - Improved compressed images flashing speed by exploiting multiple threads: 128 | now we read/decompress the image in one thread and write it in a different 129 | thread. 130 | 131 | Tue Nov 13 08:56:49 UTC 2012 - Artem Bityutskiy 0.3 132 | - Fix flashing speed calculations 133 | - Fix the Ctrl-C freeze issue - now we synchronize the block device 134 | periodically so if a Ctrl-C interruption happens, we terminate withen few 135 | seconds. 136 | 137 | Fri Nov 9 10:21:31 UTC 2012 - Artem Bityutskiy 0.2 138 | - Release 0.2 - mostly internal code re-structuring and renamings, 139 | not much functional changes. 140 | - The 'bmap-flasher' and 'bmap-creator' tools do not exist anymore. Now 141 | we have 'bmaptool' which supports 'copy' and 'create' sub-commands instead. 142 | - The BmapFlasher module was also re-named to BmapCopy. 143 | 144 | Wed Nov 7 09:37:59 UTC 2012 - Artem Bityutskiy 0.1.0 145 | - Release 0.1.1 - a lot of fixes and speed improvements. 146 | 147 | Sat Oct 27 19:13:31 UTC 2012 - Eduard Bartoch 0.0.1 148 | - Initial packaging. 149 | -------------------------------------------------------------------------------- /packaging/bmaptool.spec: -------------------------------------------------------------------------------- 1 | # We follow the Fedora guide for versioning. Fedora recommends to use something 2 | # like '1.0-0.rc7' for release candidate rc7 and '1.0-1' for the '1.0' release. 3 | %define rc_str %{?rc_num:0.rc%{rc_num}}%{!?rc_num:1} 4 | 5 | Name: bmaptool 6 | Summary: Tools to generate block map (AKA bmap) and flash images using bmap 7 | Version: 3.9.0 8 | %if 0%{?opensuse_bs} 9 | Release: %{rc_str}.. 10 | %else 11 | Release: %{rc_str}.0.0 12 | %endif 13 | 14 | Group: Development/Tools/Other 15 | License: GPL-2.0 16 | BuildArch: noarch 17 | URL: https://github.com/yoctoproject/bmaptool 18 | Source0: %{name}-%{version}.tar.gz 19 | 20 | Requires: bzip2 21 | Requires: pbzip2 22 | Requires: gzip 23 | Requires: xz 24 | Requires: tar 25 | Requires: unzip 26 | Requires: lzop 27 | %if ! 0%{?tizen_version:1} 28 | # pigz is not present in Tizen 29 | Requires: pigz 30 | %endif 31 | 32 | %if 0%{?suse_version} 33 | BuildRequires: python-distribute 34 | %endif 35 | %if 0%{?fedora_version} 36 | BuildRequires: python-setuptools 37 | %endif 38 | BuildRequires: python2-rpm-macros 39 | 40 | %if 0%{?suse_version} 41 | # In OpenSuse the xml.etree module is provided by the python-xml package 42 | Requires: python-xml 43 | # The gpgme python module is in python-gpgme 44 | Requires: python-gpgme 45 | %endif 46 | 47 | %if 0%{?fedora_version} 48 | # In Fedora the xml.etree module is provided by the python-libs package 49 | Requires: python-libs 50 | # Tha gpgme python module is in pygpgme package 51 | Requires: pygpgme 52 | %endif 53 | 54 | # Centos6 uses python 2.6, which does not have the argparse module. However, 55 | # argparse is available as a separate package there. 56 | %if 0%{?centos_version} == 600 57 | Requires: python-argparse 58 | %endif 59 | 60 | %description 61 | Tools to generate block map (AKA bmap) and flash images using bmap. bmaptool is 62 | a generic tool for creating the block map (bmap) for a file, and copying files 63 | using the block map. The idea is that large file containing unused blocks, like 64 | raw system image files, can be copied or flashed a lot faster with bmaptool 65 | than with traditional tools like "dd" or "cp". See 66 | source.tizen.org/documentation/reference/bmaptool for more information. 67 | 68 | %prep 69 | %setup -q -n %{name}-%{version} 70 | 71 | %build 72 | 73 | %install 74 | rm -rf %{buildroot} 75 | %{__python2} setup.py install --prefix=%{_prefix} --root=%{buildroot} 76 | 77 | mkdir -p %{buildroot}/%{_mandir}/man1 78 | install -m644 docs/man1/bmaptool.1 %{buildroot}/%{_mandir}/man1 79 | 80 | %files 81 | %defattr(-,root,root,-) 82 | %license COPYING 83 | %dir /usr/lib/python*/site-packages/bmaptool 84 | /usr/lib/python*/site-packages/bmap_tools* 85 | /usr/lib/python*/site-packages/bmaptool/* 86 | %{_bindir}/* 87 | 88 | %doc docs/RELEASE_NOTES 89 | %{_mandir}/man1/* 90 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "bmaptool" 3 | description = "BMAP tools" 4 | dynamic = ["version"] 5 | dependencies = [ 6 | # NOTE: gpg is not installed because it must come from the system GPG package 7 | # (e.g. python3-gpg on Ubuntu) and not from PyPi. The PyPi version is very old 8 | # and no longer functions correctly 9 | #"gpg >= 1.10.0", 10 | ] 11 | required-python = ">= 3.8" 12 | authors = [ 13 | {name = "Joshua Watt", email = "JPEWhacker@gmail.com"}, 14 | {name = "Trevor Woerner", email = "twoerner@gmail.com"}, 15 | {name = "Tim Orling", email = "ticotimo@gmail.com"}, 16 | 17 | ] 18 | readme = "README.md" 19 | classifiers = [ 20 | "Development Status :: 5 - Production/Stable", 21 | "Intended Audience :: Developers", 22 | "Topic :: Software Development :: Build Tools", 23 | "Topic :: Software Development :: Embedded Systems", 24 | "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", 25 | "Programming Language :: Python :: 3", 26 | "Programming Language :: Python :: 3.8", 27 | "Programming Language :: Python :: 3.9", 28 | "Programming Language :: Python :: 3.10", 29 | "Programming Language :: Python :: 3.11", 30 | "Programming Language :: Python :: 3.12", 31 | ] 32 | 33 | [project.optional-dependencies] 34 | dev = [ 35 | "black >= 22.3.0", 36 | "six >= 1.16.0", 37 | ] 38 | 39 | [project.urls] 40 | Homepage = "https://github.com/yoctoproject/bmaptool" 41 | Repository = "https://github.com/yoctoproject/bmaptool.git" 42 | Issues = "https://github.com/yoctoproject/bmaptool/issues" 43 | 44 | [project.scripts] 45 | bmaptool = "bmaptool.CLI:main" 46 | 47 | [build-system] 48 | requires = ["hatchling"] 49 | build-backend = "hatchling.build" 50 | 51 | [tool.hatch.version] 52 | path = "src/bmaptool/CLI.py" 53 | -------------------------------------------------------------------------------- /src/bmaptool/BmapCreate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ts=4 sw=4 tw=88 et ai si 3 | # 4 | # Copyright (c) 2012-2014 Intel, Inc. 5 | # License: GPLv2 6 | # Author: Artem Bityutskiy 7 | # 8 | # This program is free software; you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License, version 2, 10 | # as published by the Free Software Foundation. 11 | # 12 | # This program is distributed in the hope that it will be useful, but 13 | # WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | # General Public License for more details. 16 | 17 | """ 18 | This module implements the block map (bmap) creation functionality and provides 19 | the corresponding API in form of the 'BmapCreate' class. 20 | 21 | The idea is that while image files may generally be very large (e.g., 4GiB), 22 | they may nevertheless contain only little real data, e.g., 512MiB. This data 23 | are files, directories, file-system meta-data, partition table, etc. When 24 | copying the image to the target device, you do not have to copy all the 4GiB of 25 | data, you can copy only 512MiB of it, which is 4 times less, so copying should 26 | presumably be 4 times faster. 27 | 28 | The block map file is an XML file which contains a list of blocks which have to 29 | be copied to the target device. The other blocks are not used and there is no 30 | need to copy them. The XML file also contains some additional information like 31 | block size, image size, count of mapped blocks, etc. There are also many 32 | commentaries, so it is human-readable. 33 | 34 | The image has to be a sparse file. Generally, this means that when you generate 35 | this image file, you should start with a huge sparse file which contains a 36 | single hole spanning the entire file. Then you should partition it, write all 37 | the data (probably by means of loop-back mounting the image or parts of it), 38 | etc. The end result should be a sparse file where mapped areas represent useful 39 | parts of the image and holes represent useless parts of the image, which do not 40 | have to be copied when copying the image to the target device. 41 | 42 | This module uses the FIEMAP ioctl to detect holes. 43 | """ 44 | 45 | # Disable the following pylint recommendations: 46 | # * Too many instance attributes - R0902 47 | # * Too few public methods - R0903 48 | # pylint: disable=R0902,R0903 49 | 50 | import hashlib 51 | from .BmapHelpers import human_size 52 | from . import Filemap 53 | 54 | # The bmap format version we generate. 55 | # 56 | # Changelog: 57 | # o 1.3 -> 2.0: 58 | # Support SHA256 and SHA512 checksums, in 1.3 only SHA1 was supported. 59 | # "BmapFileChecksum" is used instead of "BmapFileSHA1", and "chksum=" 60 | # attribute is used instead "sha1=". Introduced "ChecksumType" tag. This is 61 | # an incompatible change. 62 | # Note, bmap format 1.4 is identical to 2.0. Version 1.4 was a mistake, 63 | # instead of incrementing the major version number, we incremented minor 64 | # version number. Unfortunately, the mistake slipped into bmaptool version 65 | # 3.0, and was only fixed in bmaptool v3.1. 66 | SUPPORTED_BMAP_VERSION = "2.0" 67 | 68 | _BMAP_START_TEMPLATE = """ 69 | 90 | 91 | 92 | 93 | %u 94 | 95 | 96 | %u 97 | 98 | 99 | %u 100 | 101 | """ 102 | 103 | 104 | class Error(Exception): 105 | """ 106 | A class for exceptions generated by this module. We currently support only 107 | one type of exceptions, and we basically throw human-readable problem 108 | description in case of errors. 109 | """ 110 | 111 | pass 112 | 113 | 114 | class BmapCreate(object): 115 | """ 116 | This class implements the bmap creation functionality. To generate a bmap 117 | for an image (which is supposedly a sparse file), you should first create 118 | an instance of 'BmapCreate' and provide: 119 | 120 | * full path or a file-like object of the image to create bmap for 121 | * full path or a file object to use for writing the results to 122 | 123 | Then you should invoke the 'generate()' method of this class. It will use 124 | the FIEMAP ioctl to generate the bmap. 125 | """ 126 | 127 | def __init__(self, image, bmap, chksum_type="sha256"): 128 | """ 129 | Initialize a class instance: 130 | * image - full path or a file-like object of the image to create bmap 131 | for 132 | * bmap - full path or a file object to use for writing the resulting 133 | bmap to 134 | * chksum - type of the check sum to use in the bmap file (all checksum 135 | types which python's "hashlib" module supports are allowed). 136 | """ 137 | 138 | self.image_size = None 139 | self.image_size_human = None 140 | self.block_size = None 141 | self.blocks_cnt = None 142 | self.mapped_cnt = None 143 | self.mapped_size = None 144 | self.mapped_size_human = None 145 | self.mapped_percent = None 146 | 147 | self._mapped_count_pos1 = None 148 | self._mapped_count_pos2 = None 149 | self._chksum_pos = None 150 | 151 | self._f_image_needs_close = False 152 | self._f_bmap_needs_close = False 153 | 154 | self._cs_type = chksum_type.lower() 155 | try: 156 | self._cs_len = len(hashlib.new(self._cs_type).hexdigest()) 157 | except ValueError as err: 158 | raise Error( 159 | 'cannot initialize hash function "%s": %s' % (self._cs_type, err) 160 | ) 161 | 162 | if hasattr(image, "read"): 163 | self._f_image = image 164 | self._image_path = image.name 165 | else: 166 | self._image_path = image 167 | self._open_image_file() 168 | 169 | if hasattr(bmap, "read"): 170 | self._f_bmap = bmap 171 | self._bmap_path = bmap.name 172 | else: 173 | self._bmap_path = bmap 174 | self._open_bmap_file() 175 | 176 | try: 177 | self.filemap = Filemap.filemap(self._f_image) 178 | except (Filemap.Error, Filemap.ErrorNotSupp) as err: 179 | raise Error( 180 | "cannot generate bmap for file '%s': %s" % (self._image_path, err) 181 | ) 182 | 183 | self.image_size = self.filemap.image_size 184 | self.image_size_human = human_size(self.image_size) 185 | if self.image_size == 0: 186 | raise Error( 187 | "cannot generate bmap for zero-sized image file '%s'" % self._image_path 188 | ) 189 | 190 | self.block_size = self.filemap.block_size 191 | self.blocks_cnt = self.filemap.blocks_cnt 192 | 193 | def __del__(self): 194 | """The class destructor which closes the opened files.""" 195 | if self._f_image_needs_close: 196 | self._f_image.close() 197 | if self._f_bmap_needs_close: 198 | self._f_bmap.close() 199 | 200 | def _open_image_file(self): 201 | """Open the image file.""" 202 | try: 203 | self._f_image = open(self._image_path, "rb") 204 | except IOError as err: 205 | raise Error("cannot open image file '%s': %s" % (self._image_path, err)) 206 | 207 | self._f_image_needs_close = True 208 | 209 | def _open_bmap_file(self): 210 | """Open the bmap file.""" 211 | try: 212 | self._f_bmap = open(self._bmap_path, "w+") 213 | except IOError as err: 214 | raise Error("cannot open bmap file '%s': %s" % (self._bmap_path, err)) 215 | 216 | self._f_bmap_needs_close = True 217 | 218 | def _bmap_file_start(self): 219 | """ 220 | A helper function which generates the starting contents of the block 221 | map file: the header comment, image size, block size, etc. 222 | """ 223 | 224 | # We do not know the amount of mapped blocks at the moment, so just put 225 | # whitespaces instead of real numbers. Assume the longest possible 226 | # numbers. 227 | 228 | xml = _BMAP_START_TEMPLATE % ( 229 | SUPPORTED_BMAP_VERSION, 230 | self.image_size_human, 231 | self.image_size, 232 | self.block_size, 233 | self.blocks_cnt, 234 | ) 235 | xml += " \n" % ( 241 | " " * len(self.image_size_human), 242 | " " * len("100.0%"), 243 | ) 244 | xml += " " 245 | 246 | self._f_bmap.write(xml) 247 | self._mapped_count_pos2 = self._f_bmap.tell() 248 | 249 | xml = "%s \n\n" % (" " * len(str(self.blocks_cnt))) 250 | 251 | # pylint: disable=C0301 252 | xml += " \n" 253 | xml += " %s \n\n" % self._cs_type 254 | 255 | xml += " \n' 257 | xml += " " 258 | 259 | self._f_bmap.write(xml) 260 | self._chksum_pos = self._f_bmap.tell() 261 | 262 | xml = "0" * self._cs_len + " \n\n" 263 | xml += ( 264 | " \n" 268 | xml += " \n" 269 | # pylint: enable=C0301 270 | 271 | self._f_bmap.write(xml) 272 | 273 | def _bmap_file_end(self): 274 | """ 275 | A helper function which generates the final parts of the block map 276 | file: the ending tags and the information about the amount of mapped 277 | blocks. 278 | """ 279 | 280 | xml = " \n" 281 | xml += "\n" 282 | 283 | self._f_bmap.write(xml) 284 | 285 | self._f_bmap.seek(self._mapped_count_pos1) 286 | self._f_bmap.write( 287 | "%s or %.1f%%" % (self.mapped_size_human, self.mapped_percent) 288 | ) 289 | 290 | self._f_bmap.seek(self._mapped_count_pos2) 291 | self._f_bmap.write("%u" % self.mapped_cnt) 292 | 293 | self._f_bmap.seek(0) 294 | hash_obj = hashlib.new(self._cs_type) 295 | hash_obj.update(self._f_bmap.read().encode()) 296 | chksum = hash_obj.hexdigest() 297 | self._f_bmap.seek(self._chksum_pos) 298 | self._f_bmap.write("%s" % chksum) 299 | 300 | def _calculate_chksum(self, first, last): 301 | """ 302 | A helper function which calculates checksum for the range of blocks of 303 | the image file: from block 'first' to block 'last'. 304 | """ 305 | 306 | start = first * self.block_size 307 | end = (last + 1) * self.block_size 308 | 309 | self._f_image.seek(start) 310 | hash_obj = hashlib.new(self._cs_type) 311 | 312 | chunk_size = 1024 * 1024 313 | to_read = end - start 314 | read = 0 315 | 316 | while read < to_read: 317 | if read + chunk_size > to_read: 318 | chunk_size = to_read - read 319 | chunk = self._f_image.read(chunk_size) 320 | hash_obj.update(chunk) 321 | read += chunk_size 322 | 323 | return hash_obj.hexdigest() 324 | 325 | def generate(self, include_checksums=True): 326 | """ 327 | Generate bmap for the image file. If 'include_checksums' is 'True', 328 | also generate checksums for block ranges. 329 | """ 330 | 331 | # Save image file position in order to restore it at the end 332 | image_pos = self._f_image.tell() 333 | 334 | self._bmap_file_start() 335 | 336 | # Generate the block map and write it to the XML block map 337 | # file as we go. 338 | self.mapped_cnt = 0 339 | for first, last in self.filemap.get_mapped_ranges(0, self.blocks_cnt): 340 | self.mapped_cnt += last - first + 1 341 | if include_checksums: 342 | chksum = self._calculate_chksum(first, last) 343 | chksum = ' chksum="%s"' % chksum 344 | else: 345 | chksum = "" 346 | 347 | if first != last: 348 | self._f_bmap.write( 349 | " %s-%s \n" % (chksum, first, last) 350 | ) 351 | else: 352 | self._f_bmap.write(" %s \n" % (chksum, first)) 353 | 354 | self.mapped_size = self.mapped_cnt * self.block_size 355 | self.mapped_size_human = human_size(self.mapped_size) 356 | self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt 357 | 358 | self._bmap_file_end() 359 | 360 | try: 361 | self._f_bmap.flush() 362 | except IOError as err: 363 | raise Error("cannot flush the bmap file '%s': %s" % (self._bmap_path, err)) 364 | 365 | self._f_image.seek(image_pos) 366 | -------------------------------------------------------------------------------- /src/bmaptool/BmapHelpers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ts=4 sw=4 tw=88 et ai si 3 | # 4 | # Copyright (c) 2012-2014 Intel, Inc. 5 | # License: GPLv2 6 | # Author: Artem Bityutskiy 7 | # 8 | # This program is free software; you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License, version 2, 10 | # as published by the Free Software Foundation. 11 | # 12 | # This program is distributed in the hope that it will be useful, but 13 | # WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | # General Public License for more details. 16 | 17 | """ 18 | This module contains various shared helper functions. 19 | """ 20 | 21 | import os 22 | import struct 23 | import subprocess 24 | from fcntl import ioctl 25 | from subprocess import PIPE 26 | 27 | # Path to check for zfs compatibility. 28 | ZFS_COMPAT_PARAM_PATH = "/sys/module/zfs/parameters/zfs_dmu_offset_next_sync" 29 | 30 | 31 | class Error(Exception): 32 | """A class for all the other exceptions raised by this module.""" 33 | 34 | pass 35 | 36 | 37 | def human_size(size): 38 | """Transform size in bytes into a human-readable form.""" 39 | if size == 1: 40 | return "1 byte" 41 | 42 | if size < 512: 43 | return "%d bytes" % size 44 | 45 | for modifier in ["KiB", "MiB", "GiB", "TiB"]: 46 | size /= 1024.0 47 | if size < 1024: 48 | return "%.1f %s" % (size, modifier) 49 | 50 | return "%.1f %s" % (size, "EiB") 51 | 52 | 53 | def human_time(seconds): 54 | """Transform time in seconds to the HH:MM:SS format.""" 55 | (minutes, seconds) = divmod(seconds, 60) 56 | (hours, minutes) = divmod(minutes, 60) 57 | 58 | result = "" 59 | if hours: 60 | result = "%dh " % hours 61 | if minutes: 62 | result += "%dm " % minutes 63 | 64 | return result + "%.1fs" % seconds 65 | 66 | 67 | def get_block_size(file_obj): 68 | """ 69 | Return block size for file object 'file_obj'. Errors are indicated by the 70 | 'IOError' exception. 71 | """ 72 | 73 | # Get the block size of the host file-system for the image file by calling 74 | # the FIGETBSZ ioctl (number 2). 75 | try: 76 | binary_data = ioctl(file_obj, 2, struct.pack("I", 0)) 77 | bsize = struct.unpack("I", binary_data)[0] 78 | if not bsize: 79 | raise IOError("get 0 bsize by FIGETBSZ ioctl") 80 | except IOError as err: 81 | stat = os.fstat(file_obj.fileno()) 82 | if hasattr(stat, "st_blksize"): 83 | bsize = stat.st_blksize 84 | else: 85 | raise IOError("Unable to determine block size") 86 | return bsize 87 | 88 | 89 | def program_is_available(name): 90 | """ 91 | This is a helper function which checks if the external program 'name' is 92 | available in the system. 93 | """ 94 | 95 | for path in os.environ["PATH"].split(os.pathsep): 96 | program = os.path.join(path.strip('"'), name) 97 | if os.path.isfile(program) and os.access(program, os.X_OK): 98 | return True 99 | 100 | return False 101 | 102 | 103 | def get_file_system_type(path): 104 | """Return the file system type for 'path'.""" 105 | 106 | abspath = os.path.realpath(path) 107 | proc = subprocess.Popen(["df", "-PT", "--", abspath], stdout=PIPE, stderr=PIPE) 108 | stdout, stderr = proc.communicate() 109 | 110 | # Parse the output of subprocess, for example: 111 | # Filesystem Type 1K-blocks Used Available Use% Mounted on 112 | # rpool/USERDATA/foo_5ucog2 zfs 456499712 86956288 369543424 20% /home/foo 113 | ftype = None 114 | if stdout: 115 | lines = stdout.splitlines() 116 | if len(lines) >= 2: 117 | fields = lines[1].split(None, 2) 118 | if len(fields) >= 2: 119 | ftype = fields[1].lower() 120 | 121 | if not ftype: 122 | raise Error( 123 | "failed to find file system type for path at '%s'\n" 124 | "Here is the 'df -PT' output\nstdout:\n%s\nstderr:\n%s" 125 | % (path, stdout, stderr) 126 | ) 127 | return ftype 128 | 129 | 130 | def is_zfs_configuration_compatible(): 131 | """Return if hosts zfs configuration is compatible.""" 132 | 133 | path = ZFS_COMPAT_PARAM_PATH 134 | if not os.path.isfile(path): 135 | return False 136 | 137 | try: 138 | with open(path, "r") as fobj: 139 | return int(fobj.readline()) == 1 140 | except IOError as err: 141 | raise Error("cannot open zfs param path '%s': %s" % (path, err)) 142 | except ValueError as err: 143 | raise Error("invalid value read from param path '%s': %s" % (path, err)) 144 | 145 | 146 | def is_compatible_file_system(path): 147 | """Return if paths file system is compatible.""" 148 | 149 | fstype = get_file_system_type(path) 150 | if fstype == "zfs": 151 | return is_zfs_configuration_compatible() 152 | return True 153 | -------------------------------------------------------------------------------- /src/bmaptool/Filemap.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ts=4 sw=4 tw=88 et ai si 3 | # 4 | # Copyright (c) 2012-2014 Intel, Inc. 5 | # License: GPLv2 6 | # Author: Artem Bityutskiy 7 | # 8 | # This program is free software; you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License, version 2, 10 | # as published by the Free Software Foundation. 11 | # 12 | # This program is distributed in the hope that it will be useful, but 13 | # WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | # General Public License for more details. 16 | 17 | """ 18 | This module implements python implements a way to get file block. Two methods 19 | are supported - the FIEMAP ioctl and the 'SEEK_HOLE / SEEK_DATA' features of 20 | the file seek syscall. The former is implemented by the 'FilemapFiemap' class, 21 | the latter is implemented by the 'FilemapSeek' class. Both classes provide the 22 | same API. The 'filemap' function automatically selects which class can be used 23 | and returns an instance of the class. 24 | """ 25 | 26 | # Disable the following pylint recommendations: 27 | # * Too many instance attributes (R0902) 28 | # pylint: disable=R0902 29 | 30 | import os 31 | import errno 32 | import struct 33 | import array 34 | import fcntl 35 | import tempfile 36 | import logging 37 | from . import BmapHelpers 38 | 39 | _log = logging.getLogger(__name__) # pylint: disable=C0103 40 | 41 | 42 | class ErrorNotSupp(Exception): 43 | """ 44 | An exception of this type is raised when the 'FIEMAP' or 'SEEK_HOLE' feature 45 | is not supported either by the kernel or the file-system. 46 | """ 47 | 48 | pass 49 | 50 | 51 | class Error(Exception): 52 | """A class for all the other exceptions raised by this module.""" 53 | 54 | pass 55 | 56 | 57 | class _FilemapBase(object): 58 | """ 59 | This is a base class for a couple of other classes in this module. This 60 | class simply performs the common parts of the initialization process: opens 61 | the image file, gets its size, etc. 62 | """ 63 | 64 | def __init__(self, image): 65 | """ 66 | Initialize a class instance. The 'image' argument is full path to the 67 | file or file object to operate on. 68 | """ 69 | 70 | self._f_image_needs_close = False 71 | 72 | if hasattr(image, "fileno"): 73 | self._f_image = image 74 | self._image_path = image.name 75 | else: 76 | self._image_path = image 77 | self._open_image_file() 78 | 79 | try: 80 | self.image_size = os.fstat(self._f_image.fileno()).st_size 81 | except IOError as err: 82 | raise Error( 83 | "cannot get information about file '%s': %s" % (self._f_image.name, err) 84 | ) 85 | 86 | try: 87 | self.block_size = BmapHelpers.get_block_size(self._f_image) 88 | except IOError as err: 89 | raise Error("cannot get block size for '%s': %s" % (self._image_path, err)) 90 | 91 | self.blocks_cnt = (self.image_size + self.block_size - 1) // self.block_size 92 | 93 | try: 94 | self._f_image.flush() 95 | except IOError as err: 96 | raise Error("cannot flush image file '%s': %s" % (self._image_path, err)) 97 | 98 | try: 99 | os.fsync(self._f_image.fileno()), 100 | except OSError as err: 101 | raise Error( 102 | "cannot synchronize image file '%s': %s " 103 | % (self._image_path, err.strerror) 104 | ) 105 | 106 | if not BmapHelpers.is_compatible_file_system(self._image_path): 107 | fstype = BmapHelpers.get_file_system_type(self._image_path) 108 | raise Error( 109 | "image file on incompatible file system '%s': '%s': see docs for fix" 110 | % (self._image_path, fstype) 111 | ) 112 | 113 | _log.debug('opened image "%s"' % self._image_path) 114 | _log.debug( 115 | "block size %d, blocks count %d, image size %d" 116 | % (self.block_size, self.blocks_cnt, self.image_size) 117 | ) 118 | 119 | def __del__(self): 120 | """The class destructor which just closes the image file.""" 121 | if self._f_image_needs_close: 122 | self._f_image.close() 123 | 124 | def _open_image_file(self): 125 | """Open the image file.""" 126 | try: 127 | self._f_image = open(self._image_path, "rb") 128 | except IOError as err: 129 | raise Error("cannot open image file '%s': %s" % (self._image_path, err)) 130 | 131 | self._f_image_needs_close = True 132 | 133 | def block_is_mapped(self, block): # pylint: disable=W0613,R0201 134 | """ 135 | This method has to be implemented by child classes. It returns 136 | 'True' if block number 'block' of the image file is mapped and 'False' 137 | otherwise. 138 | """ 139 | 140 | raise Error("the method is not implemented") 141 | 142 | def block_is_unmapped(self, block): # pylint: disable=W0613,R0201 143 | """ 144 | This method has to be implemented by child classes. It returns 145 | 'True' if block number 'block' of the image file is not mapped (hole) 146 | and 'False' otherwise. 147 | """ 148 | 149 | raise Error("the method is not implemented") 150 | 151 | def get_mapped_ranges(self, start, count): # pylint: disable=W0613,R0201 152 | """ 153 | This method has to be implemented by child classes. This is a 154 | generator which yields ranges of mapped blocks in the file. The ranges 155 | are tuples of 2 elements: [first, last], where 'first' is the first 156 | mapped block and 'last' is the last mapped block. 157 | 158 | The ranges are yielded for the area of the file of size 'count' blocks, 159 | starting from block 'start'. 160 | """ 161 | 162 | raise Error("the method is not implemented") 163 | 164 | def get_unmapped_ranges(self, start, count): # pylint: disable=W0613,R0201 165 | """ 166 | This method has to be implemented by child classes. Just like 167 | 'get_mapped_ranges()', but yields unmapped block ranges instead 168 | (holes). 169 | """ 170 | 171 | raise Error("the method is not implemented") 172 | 173 | 174 | # The 'SEEK_HOLE' and 'SEEK_DATA' options of the file seek system call 175 | _SEEK_DATA = 3 176 | _SEEK_HOLE = 4 177 | 178 | 179 | def _lseek(file_obj, offset, whence): 180 | """This is a helper function which invokes 'os.lseek' for file object 181 | 'file_obj' and with specified 'offset' and 'whence'. The 'whence' 182 | argument is supposed to be either '_SEEK_DATA' or '_SEEK_HOLE'. When 183 | there is no more data or hole starting from 'offset', this function 184 | returns '-1'. Otherwise, the data or hole position is returned.""" 185 | 186 | try: 187 | return os.lseek(file_obj.fileno(), offset, whence) 188 | except OSError as err: 189 | # The 'lseek' system call returns the ENXIO if there is no data or 190 | # hole starting from the specified offset. 191 | if err.errno == errno.ENXIO: 192 | return -1 193 | elif err.errno == errno.EINVAL: 194 | raise ErrorNotSupp( 195 | "the kernel or file-system does not support " 196 | '"SEEK_HOLE" and "SEEK_DATA"' 197 | ) 198 | else: 199 | raise 200 | 201 | 202 | class FilemapSeek(_FilemapBase): 203 | """ 204 | This class uses the 'SEEK_HOLE' and 'SEEK_DATA' to find file block mapping. 205 | Unfortunately, the current implementation requires the caller to have write 206 | access to the image file. 207 | """ 208 | 209 | def __init__(self, image): 210 | """Refer to the '_FilemapBase' class for the documentation.""" 211 | 212 | # Call the base class constructor first 213 | _FilemapBase.__init__(self, image) 214 | _log.debug("FilemapSeek: initializing") 215 | 216 | self._probe_seek_hole() 217 | 218 | def _probe_seek_hole(self): 219 | """ 220 | Check whether the system implements 'SEEK_HOLE' and 'SEEK_DATA'. 221 | Unfortunately, there seems to be no clean way for detecting this, 222 | because often the system just fakes them by just assuming that all 223 | files are fully mapped, so 'SEEK_HOLE' always returns EOF and 224 | 'SEEK_DATA' always returns the requested offset. 225 | 226 | I could not invent a better way of detecting the fake 'SEEK_HOLE' 227 | implementation than just to create a temporary file in the same 228 | directory where the image file resides. It would be nice to change this 229 | to something better. 230 | """ 231 | 232 | directory = os.path.dirname(self._image_path) 233 | 234 | try: 235 | tmp_obj = tempfile.TemporaryFile("w+", dir=directory) 236 | except OSError as err: 237 | raise ErrorNotSupp( 238 | 'cannot create a temporary in "%s": %s' % (directory, err) 239 | ) 240 | 241 | try: 242 | os.ftruncate(tmp_obj.fileno(), self.block_size) 243 | except OSError as err: 244 | raise ErrorNotSupp( 245 | 'cannot truncate temporary file in "%s": %s' % (directory, err) 246 | ) 247 | 248 | offs = _lseek(tmp_obj, 0, _SEEK_HOLE) 249 | if offs != 0: 250 | # We are dealing with the stub 'SEEK_HOLE' implementation which 251 | # always returns EOF. 252 | _log.debug("lseek(0, SEEK_HOLE) returned %d" % offs) 253 | raise ErrorNotSupp( 254 | "the file-system does not support " 255 | '"SEEK_HOLE" and "SEEK_DATA" but only ' 256 | "provides a stub implementation" 257 | ) 258 | 259 | tmp_obj.close() 260 | 261 | def block_is_mapped(self, block): 262 | """Refer to the '_FilemapBase' class for the documentation.""" 263 | offs = _lseek(self._f_image, block * self.block_size, _SEEK_DATA) 264 | if offs == -1: 265 | result = False 266 | else: 267 | result = offs // self.block_size == block 268 | 269 | _log.debug("FilemapSeek: block_is_mapped(%d) returns %s" % (block, result)) 270 | return result 271 | 272 | def block_is_unmapped(self, block): 273 | """Refer to the '_FilemapBase' class for the documentation.""" 274 | return not self.block_is_mapped(block) 275 | 276 | def _get_ranges(self, start, count, whence1, whence2): 277 | """ 278 | This function implements 'get_mapped_ranges()' and 279 | 'get_unmapped_ranges()' depending on what is passed in the 'whence1' 280 | and 'whence2' arguments. 281 | """ 282 | 283 | assert whence1 != whence2 284 | end = start * self.block_size 285 | limit = end + count * self.block_size 286 | 287 | while True: 288 | start = _lseek(self._f_image, end, whence1) 289 | if start == -1 or start >= limit or start == self.image_size: 290 | break 291 | 292 | end = _lseek(self._f_image, start, whence2) 293 | if end == -1 or end == self.image_size: 294 | end = self.blocks_cnt * self.block_size 295 | if end > limit: 296 | end = limit 297 | 298 | start_blk = start // self.block_size 299 | end_blk = end // self.block_size - 1 300 | _log.debug("FilemapSeek: yielding range (%d, %d)" % (start_blk, end_blk)) 301 | yield (start_blk, end_blk) 302 | 303 | def get_mapped_ranges(self, start, count): 304 | """Refer to the '_FilemapBase' class for the documentation.""" 305 | _log.debug( 306 | "FilemapSeek: get_mapped_ranges(%d, %d(%d))" 307 | % (start, count, start + count - 1) 308 | ) 309 | return self._get_ranges(start, count, _SEEK_DATA, _SEEK_HOLE) 310 | 311 | def get_unmapped_ranges(self, start, count): 312 | """Refer to the '_FilemapBase' class for the documentation.""" 313 | _log.debug( 314 | "FilemapSeek: get_unmapped_ranges(%d, %d(%d))" 315 | % (start, count, start + count - 1) 316 | ) 317 | return self._get_ranges(start, count, _SEEK_HOLE, _SEEK_DATA) 318 | 319 | 320 | # Below goes the FIEMAP ioctl implementation, which is not very readable 321 | # because it deals with the rather complex FIEMAP ioctl. To understand the 322 | # code, you need to know the FIEMAP interface, which is documented in the 323 | # "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources. 324 | 325 | # Format string for 'struct fiemap' 326 | _FIEMAP_FORMAT = "=QQLLLL" 327 | # sizeof(struct fiemap) 328 | _FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT) 329 | # Format string for 'struct fiemap_extent' 330 | _FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL" 331 | # sizeof(struct fiemap_extent) 332 | _FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT) 333 | # The FIEMAP ioctl number 334 | _FIEMAP_IOCTL = 0xC020660B 335 | # This FIEMAP ioctl flag which instructs the kernel to sync the file before 336 | # reading the block map 337 | _FIEMAP_FLAG_SYNC = 0x00000001 338 | # Size of the buffer for 'struct fiemap_extent' elements which will be used 339 | # when invoking the FIEMAP ioctl. With a larger buffer, the FIEMAP ioctl will 340 | # be invoked fewer times. 341 | _FIEMAP_BUFFER_SIZE = 256 * 1024 342 | 343 | 344 | class FilemapFiemap(_FilemapBase): 345 | """ 346 | This class provides API to the FIEMAP ioctl. Namely, it allows to iterate 347 | over all mapped blocks and over all holes. 348 | 349 | This class synchronizes the image file every time it invokes the FIEMAP 350 | ioctl in order to work-around early FIEMAP implementation kernel bugs. 351 | """ 352 | 353 | def __init__(self, image): 354 | """ 355 | Initialize a class instance. The 'image' argument is the file object 356 | to operate on. 357 | """ 358 | 359 | # Call the base class constructor first 360 | _FilemapBase.__init__(self, image) 361 | _log.debug("FilemapFiemap: initializing") 362 | 363 | self._buf_size = _FIEMAP_BUFFER_SIZE 364 | 365 | # Calculate how many 'struct fiemap_extent' elements fit the buffer 366 | self._buf_size -= _FIEMAP_SIZE 367 | self._fiemap_extent_cnt = self._buf_size // _FIEMAP_EXTENT_SIZE 368 | assert self._fiemap_extent_cnt > 0 369 | self._buf_size = self._fiemap_extent_cnt * _FIEMAP_EXTENT_SIZE 370 | self._buf_size += _FIEMAP_SIZE 371 | 372 | # Allocate a mutable buffer for the FIEMAP ioctl 373 | self._buf = array.array("B", [0] * self._buf_size) 374 | 375 | # Check if the FIEMAP ioctl is supported 376 | self.block_is_mapped(0) 377 | 378 | def _invoke_fiemap(self, block, count): 379 | """ 380 | Invoke the FIEMAP ioctl for 'count' blocks of the file starting from 381 | block number 'block'. 382 | 383 | The full result of the operation is stored in 'self._buf' on exit. 384 | Returns the unpacked 'struct fiemap' data structure in form of a python 385 | list (just like 'struct.upack()'). 386 | """ 387 | 388 | if self.blocks_cnt != 0 and (block < 0 or block >= self.blocks_cnt): 389 | raise Error( 390 | "bad block number %d, should be within [0, %d]" 391 | % (block, self.blocks_cnt) 392 | ) 393 | 394 | # Initialize the 'struct fiemap' part of the buffer. We use the 395 | # '_FIEMAP_FLAG_SYNC' flag in order to make sure the file is 396 | # synchronized. The reason for this is that early FIEMAP 397 | # implementations had many bugs related to cached dirty data, and 398 | # synchronizing the file is a necessary work-around. 399 | struct.pack_into( 400 | _FIEMAP_FORMAT, 401 | self._buf, 402 | 0, 403 | block * self.block_size, 404 | count * self.block_size, 405 | _FIEMAP_FLAG_SYNC, 406 | 0, 407 | self._fiemap_extent_cnt, 408 | 0, 409 | ) 410 | 411 | try: 412 | fcntl.ioctl(self._f_image, _FIEMAP_IOCTL, self._buf, 1) 413 | except IOError as err: 414 | # Note, the FIEMAP ioctl is supported by the Linux kernel starting 415 | # from version 2.6.28 (year 2008). 416 | if err.errno == errno.EOPNOTSUPP: 417 | errstr = ( 418 | "FilemapFiemap: the FIEMAP ioctl is not supported " 419 | "by the file-system" 420 | ) 421 | _log.debug(errstr) 422 | raise ErrorNotSupp(errstr) 423 | if err.errno == errno.ENOTTY: 424 | errstr = ( 425 | "FilemapFiemap: the FIEMAP ioctl is not supported " "by the kernel" 426 | ) 427 | _log.debug(errstr) 428 | raise ErrorNotSupp(errstr) 429 | raise Error( 430 | "the FIEMAP ioctl failed for '%s': %s" % (self._image_path, err) 431 | ) 432 | 433 | return struct.unpack(_FIEMAP_FORMAT, self._buf[:_FIEMAP_SIZE]) 434 | 435 | def block_is_mapped(self, block): 436 | """Refer to the '_FilemapBase' class for the documentation.""" 437 | struct_fiemap = self._invoke_fiemap(block, 1) 438 | 439 | # The 3rd element of 'struct_fiemap' is the 'fm_mapped_extents' field. 440 | # If it contains zero, the block is not mapped, otherwise it is 441 | # mapped. 442 | result = bool(struct_fiemap[3]) 443 | _log.debug("FilemapFiemap: block_is_mapped(%d) returns %s" % (block, result)) 444 | return result 445 | 446 | def block_is_unmapped(self, block): 447 | """Refer to the '_FilemapBase' class for the documentation.""" 448 | return not self.block_is_mapped(block) 449 | 450 | def _unpack_fiemap_extent(self, index): 451 | """ 452 | Unpack a 'struct fiemap_extent' structure object number 'index' from 453 | the internal 'self._buf' buffer. 454 | """ 455 | 456 | offset = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE * index 457 | return struct.unpack( 458 | _FIEMAP_EXTENT_FORMAT, self._buf[offset : offset + _FIEMAP_EXTENT_SIZE] 459 | ) 460 | 461 | def _do_get_mapped_ranges(self, start, count): 462 | """ 463 | Implements most the functionality for the 'get_mapped_ranges()' 464 | generator: invokes the FIEMAP ioctl, walks through the mapped extents 465 | and yields mapped block ranges. However, the ranges may be consecutive 466 | (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()' simply merges 467 | them. 468 | """ 469 | 470 | block = start 471 | while block < start + count: 472 | struct_fiemap = self._invoke_fiemap(block, count) 473 | 474 | mapped_extents = struct_fiemap[3] 475 | if mapped_extents == 0: 476 | # No more mapped blocks 477 | return 478 | 479 | extent = 0 480 | while extent < mapped_extents: 481 | fiemap_extent = self._unpack_fiemap_extent(extent) 482 | 483 | # Start of the extent 484 | extent_start = fiemap_extent[0] 485 | # Starting block number of the extent 486 | extent_block = extent_start // self.block_size 487 | # Length of the extent 488 | extent_len = fiemap_extent[2] 489 | # Count of blocks in the extent 490 | extent_count = extent_len // self.block_size 491 | 492 | # Extent length and offset have to be block-aligned 493 | assert extent_start % self.block_size == 0 494 | assert extent_len % self.block_size == 0 495 | 496 | if extent_block > start + count - 1: 497 | return 498 | 499 | first = max(extent_block, block) 500 | last = min(extent_block + extent_count, start + count) - 1 501 | yield (first, last) 502 | 503 | extent += 1 504 | 505 | block = extent_block + extent_count 506 | 507 | def get_mapped_ranges(self, start, count): 508 | """Refer to the '_FilemapBase' class for the documentation.""" 509 | _log.debug( 510 | "FilemapFiemap: get_mapped_ranges(%d, %d(%d))" 511 | % (start, count, start + count - 1) 512 | ) 513 | iterator = self._do_get_mapped_ranges(start, count) 514 | 515 | try: 516 | first_prev, last_prev = next(iterator) 517 | except StopIteration: 518 | return 519 | 520 | for first, last in iterator: 521 | if last_prev == first - 1: 522 | last_prev = last 523 | else: 524 | _log.debug( 525 | "FilemapFiemap: yielding range (%d, %d)" % (first_prev, last_prev) 526 | ) 527 | yield (first_prev, last_prev) 528 | first_prev, last_prev = first, last 529 | 530 | _log.debug("FilemapFiemap: yielding range (%d, %d)" % (first_prev, last_prev)) 531 | yield (first_prev, last_prev) 532 | 533 | def get_unmapped_ranges(self, start, count): 534 | """Refer to the '_FilemapBase' class for the documentation.""" 535 | _log.debug( 536 | "FilemapFiemap: get_unmapped_ranges(%d, %d(%d))" 537 | % (start, count, start + count - 1) 538 | ) 539 | hole_first = start 540 | for first, last in self._do_get_mapped_ranges(start, count): 541 | if first > hole_first: 542 | _log.debug( 543 | "FilemapFiemap: yielding range (%d, %d)" % (hole_first, first - 1) 544 | ) 545 | yield (hole_first, first - 1) 546 | 547 | hole_first = last + 1 548 | 549 | if hole_first < start + count: 550 | _log.debug( 551 | "FilemapFiemap: yielding range (%d, %d)" 552 | % (hole_first, start + count - 1) 553 | ) 554 | yield (hole_first, start + count - 1) 555 | 556 | 557 | def filemap(image): 558 | """ 559 | Create and return an instance of a Filemap class - 'FilemapFiemap' or 560 | 'FilemapSeek', depending on what the system we run on supports. If the 561 | FIEMAP ioctl is supported, an instance of the 'FilemapFiemap' class is 562 | returned. Otherwise, if 'SEEK_HOLE' is supported an instance of the 563 | 'FilemapSeek' class is returned. If none of these are supported, the 564 | function generates an 'Error' type exception. 565 | """ 566 | 567 | try: 568 | return FilemapFiemap(image) 569 | except ErrorNotSupp: 570 | return FilemapSeek(image) 571 | -------------------------------------------------------------------------------- /src/bmaptool/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoctoproject/bmaptool/618a7316102f6f81faa60537503012a419eafa06/src/bmaptool/__init__.py -------------------------------------------------------------------------------- /src/bmaptool/__main__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import re 3 | import sys 4 | 5 | from .CLI import main 6 | 7 | if __name__ == "__main__": 8 | sys.argv[0] = re.sub(r"(-script\.pyw|\.exe|\.pyz)?$", "", sys.argv[0]) 9 | sys.exit(main()) 10 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoctoproject/bmaptool/618a7316102f6f81faa60537503012a419eafa06/tests/__init__.py -------------------------------------------------------------------------------- /tests/helpers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ts=4 sw=4 tw=88 et ai si 3 | # 4 | # Copyright (c) 2012-2014 Intel, Inc. 5 | # License: GPLv2 6 | # Author: Artem Bityutskiy 7 | # 8 | # This program is free software; you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License, version 2, 10 | # as published by the Free Software Foundation. 11 | # 12 | # This program is distributed in the hope that it will be useful, but 13 | # WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | # General Public License for more details. 16 | 17 | """ 18 | This module contains independent functions shared between various 19 | tests. 20 | """ 21 | 22 | # Disable the following pylint recommendations: 23 | # * Too many statements (R0915) 24 | # pylint: disable=R0915 25 | 26 | import tempfile 27 | import random 28 | import itertools 29 | import hashlib 30 | import struct 31 | import sys 32 | import os 33 | from bmaptool import BmapHelpers, BmapCopy, TransRead 34 | 35 | 36 | def _create_random_sparse_file(file_obj, size): 37 | """ 38 | Create a sparse file with randomly distributed holes. The mapped areas are 39 | filled with semi-random data. Returns a tuple containing 2 lists: 40 | 1. a list of mapped block ranges, same as 'Filemap.get_mapped_ranges()' 41 | 2. a list of unmapped block ranges (holes), same as 42 | 'Filemap.get_unmapped_ranges()' 43 | """ 44 | 45 | file_obj.truncate(size) 46 | block_size = BmapHelpers.get_block_size(file_obj) 47 | blocks_cnt = (size + block_size - 1) // block_size 48 | 49 | def process_block(block): 50 | """ 51 | This is a helper function which processes a block. It randomly decides 52 | whether the block should be filled with random data or should become a 53 | hole. Returns 'True' if the block was mapped and 'False' otherwise. 54 | """ 55 | 56 | map_the_block = random.getrandbits(1) 57 | 58 | if map_the_block: 59 | # Randomly select how much we are going to write 60 | seek = random.randint(0, block_size - 1) 61 | write = random.randint(1, block_size - seek) 62 | assert seek + write <= block_size 63 | file_obj.seek(block * block_size + seek) 64 | file_obj.write(struct.pack("=B", random.getrandbits(8)) * write) 65 | return map_the_block 66 | 67 | mapped = [] 68 | unmapped = [] 69 | iterator = range(0, blocks_cnt) 70 | for was_mapped, group in itertools.groupby(iterator, process_block): 71 | # Start of a mapped region or a hole. Find the last element in the 72 | # group. 73 | first = next(group) 74 | last = first 75 | for last in group: 76 | pass 77 | 78 | if was_mapped: 79 | mapped.append((first, last)) 80 | else: 81 | unmapped.append((first, last)) 82 | 83 | file_obj.truncate(size) 84 | file_obj.flush() 85 | 86 | return (mapped, unmapped) 87 | 88 | 89 | def _create_random_file(file_obj, size): 90 | """ 91 | Fill the 'file_obj' file object with semi-random data up to the size 'size'. 92 | """ 93 | 94 | chunk_size = 1024 * 1024 95 | written = 0 96 | 97 | while written < size: 98 | if written + chunk_size > size: 99 | chunk_size = size - written 100 | 101 | file_obj.write(struct.pack("=B", random.getrandbits(8)) * chunk_size) 102 | 103 | written += chunk_size 104 | 105 | file_obj.flush() 106 | 107 | 108 | def generate_test_files(max_size=4 * 1024 * 1024, directory=None, delete=True): 109 | """ 110 | This is a generator which yields files which other tests use as the input 111 | for the testing. The generator tries to yield "interesting" files which 112 | cover various corner-cases. For example, a large hole file, a file with 113 | no holes, files of unaligned length, etc. 114 | 115 | The 'directory' argument specifies the directory path where the yielded 116 | test files should be created. The 'delete' argument specifies whether the 117 | yielded test files have to be automatically deleted. 118 | 119 | The generator yields tuples consisting of the following elements: 120 | 1. the test file object 121 | 2. file size in bytes 122 | 3. a list of mapped block ranges, same as 'Filemap.get_mapped_ranges()' 123 | 4. a list of unmapped block ranges (holes), same as 124 | 'Filemap.get_unmapped_ranges()' 125 | """ 126 | 127 | # 128 | # Generate sparse files with one single hole spanning the entire file 129 | # 130 | 131 | # A block-sized hole 132 | file_obj = tempfile.NamedTemporaryFile( 133 | "wb+", prefix="4Khole_", delete=delete, dir=directory, suffix=".img" 134 | ) 135 | block_size = BmapHelpers.get_block_size(file_obj) 136 | file_obj.truncate(block_size) 137 | yield (file_obj, block_size, [], [(0, 0)]) 138 | file_obj.close() 139 | 140 | # A block size + 1 byte hole 141 | file_obj = tempfile.NamedTemporaryFile( 142 | "wb+", prefix="4Khole_plus_1_", delete=delete, dir=directory, suffix=".img" 143 | ) 144 | file_obj.truncate(block_size + 1) 145 | yield (file_obj, block_size + 1, [], [(0, 1)]) 146 | file_obj.close() 147 | 148 | # A block size - 1 byte hole 149 | file_obj = tempfile.NamedTemporaryFile( 150 | "wb+", prefix="4Khole_minus_1_", delete=delete, dir=directory, suffix=".img" 151 | ) 152 | file_obj.truncate(block_size - 1) 153 | yield (file_obj, block_size - 1, [], [(0, 0)]) 154 | file_obj.close() 155 | 156 | # A 1-byte hole 157 | file_obj = tempfile.NamedTemporaryFile( 158 | "wb+", prefix="1byte_hole_", delete=delete, dir=directory, suffix=".img" 159 | ) 160 | file_obj.truncate(1) 161 | yield (file_obj, 1, [], [(0, 0)]) 162 | file_obj.close() 163 | 164 | # And 10 holes of random size 165 | for i in range(10): 166 | size = random.randint(1, max_size) 167 | file_obj = tempfile.NamedTemporaryFile( 168 | "wb+", 169 | suffix=".img", 170 | delete=delete, 171 | dir=directory, 172 | prefix="rand_hole_%d_" % i, 173 | ) 174 | file_obj.truncate(size) 175 | blocks_cnt = (size + block_size - 1) // block_size 176 | yield (file_obj, size, [], [(0, blocks_cnt - 1)]) 177 | file_obj.close() 178 | 179 | # 180 | # Generate random sparse files 181 | # 182 | 183 | # The maximum size 184 | file_obj = tempfile.NamedTemporaryFile( 185 | "wb+", prefix="sparse_", delete=delete, dir=directory, suffix=".img" 186 | ) 187 | mapped, unmapped = _create_random_sparse_file(file_obj, max_size) 188 | yield (file_obj, max_size, mapped, unmapped) 189 | file_obj.close() 190 | 191 | # The maximum size + 1 byte 192 | file_obj = tempfile.NamedTemporaryFile( 193 | "wb+", prefix="sparse_plus_1_", delete=delete, dir=directory, suffix=".img" 194 | ) 195 | mapped, unmapped = _create_random_sparse_file(file_obj, max_size + 1) 196 | yield (file_obj, max_size + 1, mapped, unmapped) 197 | file_obj.close() 198 | 199 | # The maximum size - 1 byte 200 | file_obj = tempfile.NamedTemporaryFile( 201 | "wb+", prefix="sparse_minus_1_", delete=delete, dir=directory, suffix=".img" 202 | ) 203 | mapped, unmapped = _create_random_sparse_file(file_obj, max_size - 1) 204 | yield (file_obj, max_size - 1, mapped, unmapped) 205 | file_obj.close() 206 | 207 | # And 10 files of random size 208 | for i in range(10): 209 | size = random.randint(1, max_size) 210 | file_obj = tempfile.NamedTemporaryFile( 211 | "wb+", suffix=".img", delete=delete, dir=directory, prefix="sparse_%d_" % i 212 | ) 213 | mapped, unmapped = _create_random_sparse_file(file_obj, size) 214 | yield (file_obj, size, mapped, unmapped) 215 | file_obj.close() 216 | 217 | # 218 | # Generate random fully-mapped files 219 | # 220 | 221 | # A block-sized file 222 | file_obj = tempfile.NamedTemporaryFile( 223 | "wb+", prefix="4Kmapped_", delete=delete, dir=directory, suffix=".img" 224 | ) 225 | _create_random_file(file_obj, block_size) 226 | yield (file_obj, block_size, [(0, 0)], []) 227 | file_obj.close() 228 | 229 | # A block size + 1 byte file 230 | file_obj = tempfile.NamedTemporaryFile( 231 | "wb+", prefix="4Kmapped_plus_1_", delete=delete, dir=directory, suffix=".img" 232 | ) 233 | _create_random_file(file_obj, block_size + 1) 234 | yield (file_obj, block_size + 1, [(0, 1)], []) 235 | file_obj.close() 236 | 237 | # A block size - 1 byte file 238 | file_obj = tempfile.NamedTemporaryFile( 239 | "wb+", prefix="4Kmapped_minus_1_", delete=delete, dir=directory, suffix=".img" 240 | ) 241 | _create_random_file(file_obj, block_size - 1) 242 | yield (file_obj, block_size - 1, [(0, 0)], []) 243 | file_obj.close() 244 | 245 | # A 1-byte file 246 | file_obj = tempfile.NamedTemporaryFile( 247 | "wb+", prefix="1byte_mapped_", delete=delete, dir=directory, suffix=".img" 248 | ) 249 | _create_random_file(file_obj, 1) 250 | yield (file_obj, 1, [(0, 0)], []) 251 | file_obj.close() 252 | 253 | # And 10 mapped files of random size 254 | for i in range(10): 255 | size = random.randint(1, max_size) 256 | file_obj = tempfile.NamedTemporaryFile( 257 | "wb+", 258 | suffix=".img", 259 | delete=delete, 260 | dir=directory, 261 | prefix="rand_mapped_%d_" % i, 262 | ) 263 | _create_random_file(file_obj, size) 264 | blocks_cnt = (size + block_size - 1) // block_size 265 | yield (file_obj, size, [(0, blocks_cnt - 1)], []) 266 | file_obj.close() 267 | 268 | 269 | def calculate_chksum(file_path): 270 | """Calculates checksum for the contents of file 'file_path'.""" 271 | 272 | file_obj = TransRead.TransRead(file_path) 273 | hash_obj = hashlib.new("sha256") 274 | 275 | chunk_size = 1024 * 1024 276 | 277 | while True: 278 | chunk = file_obj.read(chunk_size) 279 | if not chunk: 280 | break 281 | hash_obj.update(chunk) 282 | 283 | file_obj.close() 284 | return hash_obj.hexdigest() 285 | 286 | 287 | def copy_and_verify_image(image, dest, bmap, image_chksum, image_size): 288 | """ 289 | Copy image 'image' using bmap file 'bmap' to the destination file 'dest' 290 | and verify the resulting image checksum. 291 | """ 292 | 293 | f_image = TransRead.TransRead(image) 294 | f_dest = open(dest, "w+b") 295 | if bmap: 296 | f_bmap = open(bmap, "r") 297 | else: 298 | f_bmap = None 299 | 300 | writer = BmapCopy.BmapCopy(f_image, f_dest, f_bmap, image_size) 301 | # Randomly decide whether we want the progress bar or not 302 | if bool(random.getrandbits(1)) and sys.stdout.isatty(): 303 | writer.set_progress_indicator(sys.stdout, None) 304 | writer.copy(bool(random.getrandbits(1)), bool(random.getrandbits(1))) 305 | 306 | # Compare the original file and the copy are identical 307 | assert calculate_chksum(dest) == image_chksum 308 | 309 | if f_bmap: 310 | f_bmap.close() 311 | f_dest.close() 312 | f_image.close() 313 | -------------------------------------------------------------------------------- /tests/oldcodebase/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoctoproject/bmaptool/618a7316102f6f81faa60537503012a419eafa06/tests/oldcodebase/__init__.py -------------------------------------------------------------------------------- /tests/test-data/test.image.bmap.v1.2: -------------------------------------------------------------------------------- 1 | 2 | 23 | 24 | 25 | 26 | 821752 27 | 28 | 29 | 4096 30 | 31 | 32 | 201 33 | 34 | 37 | 38 | 0-1 39 | 3-5 40 | 9-10 41 | 12 42 | 15-18 43 | 20 44 | 22 45 | 24 46 | 30-32 47 | 34-35 48 | 40 49 | 42-43 50 | 45 51 | 47 52 | 49-50 53 | 52-53 54 | 55-56 55 | 60-63 56 | 65-67 57 | 70 58 | 72 59 | 78-80 60 | 82-83 61 | 85 62 | 88 63 | 90-91 64 | 96 65 | 98-105 66 | 111 67 | 114-116 68 | 119-133 69 | 135 70 | 137 71 | 140 72 | 142-144 73 | 146-147 74 | 150-151 75 | 155 76 | 157 77 | 159-160 78 | 163-174 79 | 177 80 | 181-186 81 | 188-189 82 | 191 83 | 193 84 | 195 85 | 198-199 86 | 87 | 88 | 89 | 117 90 | 91 | -------------------------------------------------------------------------------- /tests/test-data/test.image.bmap.v1.3: -------------------------------------------------------------------------------- 1 | 2 | 23 | 24 | 25 | 26 | 821752 27 | 28 | 29 | 4096 30 | 31 | 32 | 201 33 | 34 | 35 | 117 36 | 37 | 39 | e235f7cd0c6b8c07a2e6f2538510fb763e7790a6 40 | 41 | 44 | 45 | 0-1 46 | 3-5 47 | 9-10 48 | 12 49 | 15-18 50 | 20 51 | 22 52 | 24 53 | 30-32 54 | 34-35 55 | 40 56 | 42-43 57 | 45 58 | 47 59 | 49-50 60 | 52-53 61 | 55-56 62 | 60-63 63 | 65-67 64 | 70 65 | 72 66 | 78-80 67 | 82-83 68 | 85 69 | 88 70 | 90-91 71 | 96 72 | 98-105 73 | 111 74 | 114-116 75 | 119-133 76 | 135 77 | 137 78 | 140 79 | 142-144 80 | 146-147 81 | 150-151 82 | 155 83 | 157 84 | 159-160 85 | 163-174 86 | 177 87 | 181-186 88 | 188-189 89 | 191 90 | 193 91 | 195 92 | 198-199 93 | 94 | 95 | -------------------------------------------------------------------------------- /tests/test-data/test.image.bmap.v1.4: -------------------------------------------------------------------------------- 1 | 2 | 23 | 24 | 25 | 26 | 821752 27 | 28 | 29 | 4096 30 | 31 | 32 | 201 33 | 34 | 35 | 117 36 | 37 | 38 | sha256 39 | 40 | 42 | 4310fd457a88d307abeeb593a7888e1fa3cae0cfc01d905158967c904c5375e5 43 | 44 | 47 | 48 | 0-1 49 | 3-5 50 | 9-10 51 | 12 52 | 15-18 53 | 20 54 | 22 55 | 24 56 | 30-32 57 | 34-35 58 | 40 59 | 42-43 60 | 45 61 | 47 62 | 49-50 63 | 52-53 64 | 55-56 65 | 60-63 66 | 65-67 67 | 70 68 | 72 69 | 78-80 70 | 82-83 71 | 85 72 | 88 73 | 90-91 74 | 96 75 | 98-105 76 | 111 77 | 114-116 78 | 119-133 79 | 135 80 | 137 81 | 140 82 | 142-144 83 | 146-147 84 | 150-151 85 | 155 86 | 157 87 | 159-160 88 | 163-174 89 | 177 90 | 181-186 91 | 188-189 92 | 191 93 | 193 94 | 195 95 | 198-199 96 | 97 | 98 | -------------------------------------------------------------------------------- /tests/test-data/test.image.bmap.v2.0: -------------------------------------------------------------------------------- 1 | 2 | 23 | 24 | 25 | 26 | 821752 27 | 28 | 29 | 4096 30 | 31 | 32 | 201 33 | 34 | 35 | 117 36 | 37 | 38 | sha256 39 | 40 | 42 | d9cf7d44790d04fcbb089c5eeec7700e9233439ab6e4bd759035906e20f90070 43 | 44 | 47 | 48 | 0-1 49 | 3-5 50 | 9-10 51 | 12 52 | 15-18 53 | 20 54 | 22 55 | 24 56 | 30-32 57 | 34-35 58 | 40 59 | 42-43 60 | 45 61 | 47 62 | 49-50 63 | 52-53 64 | 55-56 65 | 60-63 66 | 65-67 67 | 70 68 | 72 69 | 78-80 70 | 82-83 71 | 85 72 | 88 73 | 90-91 74 | 96 75 | 98-105 76 | 111 77 | 114-116 78 | 119-133 79 | 135 80 | 137 81 | 140 82 | 142-144 83 | 146-147 84 | 150-151 85 | 155 86 | 157 87 | 159-160 88 | 163-174 89 | 177 90 | 181-186 91 | 188-189 92 | 191 93 | 193 94 | 195 95 | 198-199 96 | 97 | 98 | -------------------------------------------------------------------------------- /tests/test-data/test.image.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yoctoproject/bmaptool/618a7316102f6f81faa60537503012a419eafa06/tests/test-data/test.image.gz -------------------------------------------------------------------------------- /tests/test_CLI.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ts=4 sw=4 tw=88 et ai si 3 | # 4 | # Copyright (c) 2022 Benedikt Wildenhain 5 | # License: GPLv2 6 | # Author: Benedikt Wildenhain 7 | # 8 | # This program is free software; you can redistribute it and/or modify it under 9 | # the terms of the GNU General Public License, version 2 or any later version, 10 | # as published by the Free Software Foundation. 11 | # 12 | # This program is distributed in the hope that it will be useful, but 13 | # WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | # General Public License for more details. 16 | 17 | import unittest 18 | 19 | import os 20 | import subprocess 21 | import sys 22 | import tempfile 23 | import tests.helpers 24 | import shutil 25 | from dataclasses import dataclass 26 | 27 | 28 | @dataclass 29 | class Key: 30 | gnupghome: str 31 | uid: str 32 | fpr: str = None 33 | 34 | 35 | testkeys = { 36 | "correct": Key(None, "correct ", None), 37 | "unknown": Key(None, "unknown ", None), 38 | } 39 | 40 | 41 | class TestCLI(unittest.TestCase): 42 | def test_valid_signature(self): 43 | completed_process = subprocess.run( 44 | [ 45 | "bmaptool", 46 | "copy", 47 | "--bmap", 48 | "tests/test-data/test.image.bmap.v2.0", 49 | "--bmap-sig", 50 | "tests/test-data/signatures/test.image.bmap.v2.0correct.det.asc", 51 | "tests/test-data/test.image.gz", 52 | self.tmpfile, 53 | ], 54 | stdout=subprocess.PIPE, 55 | stderr=subprocess.PIPE, 56 | check=False, 57 | ) 58 | self.assertEqual(completed_process.returncode, 0) 59 | self.assertEqual(completed_process.stdout, b"") 60 | self.assertIn( 61 | b"successfully verified bmap file signature", completed_process.stderr 62 | ) 63 | 64 | def test_valid_signature_fingerprint(self): 65 | assert testkeys["correct"].fpr is not None 66 | completed_process = subprocess.run( 67 | [ 68 | "bmaptool", 69 | "copy", 70 | "--bmap", 71 | "tests/test-data/signatures/test.image.bmap.v2.0correct.asc", 72 | "--fingerprint", 73 | testkeys["correct"].fpr, 74 | "tests/test-data/test.image.gz", 75 | self.tmpfile, 76 | ], 77 | stdout=subprocess.PIPE, 78 | stderr=subprocess.PIPE, 79 | check=False, 80 | ) 81 | self.assertEqual(completed_process.returncode, 0) 82 | self.assertEqual(completed_process.stdout, b"") 83 | self.assertIn( 84 | b"successfully verified bmap file signature", completed_process.stderr 85 | ) 86 | 87 | def test_valid_signature_fingerprint_keyring(self): 88 | assert testkeys["correct"].fpr is not None 89 | completed_process = subprocess.run( 90 | [ 91 | "bmaptool", 92 | "copy", 93 | "--bmap", 94 | "tests/test-data/signatures/test.image.bmap.v2.0correct.asc", 95 | "--fingerprint", 96 | testkeys["correct"].fpr, 97 | "--keyring", 98 | testkeys["correct"].gnupghome + ".keyring", 99 | "tests/test-data/test.image.gz", 100 | self.tmpfile, 101 | ], 102 | stdout=subprocess.PIPE, 103 | stderr=subprocess.PIPE, 104 | check=False, 105 | # should work without GNUPGHOME set because we supply --keyring 106 | env={k: v for k, v in os.environ.items() if k != "GNUPGHOME"}, 107 | ) 108 | self.assertEqual(completed_process.returncode, 0) 109 | self.assertEqual(completed_process.stdout, b"") 110 | self.assertIn( 111 | b"successfully verified bmap file signature", completed_process.stderr 112 | ) 113 | 114 | def test_unknown_signer(self): 115 | completed_process = subprocess.run( 116 | [ 117 | "bmaptool", 118 | "copy", 119 | "--bmap", 120 | "tests/test-data/test.image.bmap.v2.0", 121 | "--bmap-sig", 122 | "tests/test-data/signatures/test.image.bmap.v2.0unknown.det.asc", 123 | "tests/test-data/test.image.gz", 124 | self.tmpfile, 125 | ], 126 | stdout=subprocess.PIPE, 127 | stderr=subprocess.PIPE, 128 | check=False, 129 | ) 130 | self.assertEqual(completed_process.returncode, 1) 131 | self.assertEqual(completed_process.stdout, b"") 132 | self.assertIn(b"discovered a BAD GPG signature", completed_process.stderr) 133 | 134 | def test_wrong_signature(self): 135 | completed_process = subprocess.run( 136 | [ 137 | "bmaptool", 138 | "copy", 139 | "--bmap", 140 | "tests/test-data/test.image.bmap.v1.4", 141 | "--bmap-sig", 142 | "tests/test-data/signatures/test.image.bmap.v2.0correct.det.asc", 143 | "tests/test-data/test.image.gz", 144 | self.tmpfile, 145 | ], 146 | stdout=subprocess.PIPE, 147 | stderr=subprocess.PIPE, 148 | check=False, 149 | ) 150 | self.assertEqual(completed_process.returncode, 1) 151 | self.assertEqual(completed_process.stdout, b"") 152 | self.assertIn(b"discovered a BAD GPG signature", completed_process.stderr) 153 | 154 | def test_wrong_signature_unknown_signer(self): 155 | completed_process = subprocess.run( 156 | [ 157 | "bmaptool", 158 | "copy", 159 | "--bmap", 160 | "tests/test-data/test.image.bmap.v1.4", 161 | "--bmap-sig", 162 | "tests/test-data/signatures/test.image.bmap.v2.0unknown.det.asc", 163 | "tests/test-data/test.image.gz", 164 | self.tmpfile, 165 | ], 166 | stdout=subprocess.PIPE, 167 | stderr=subprocess.PIPE, 168 | check=False, 169 | ) 170 | self.assertEqual(completed_process.returncode, 1) 171 | self.assertEqual(completed_process.stdout, b"") 172 | self.assertIn(b"discovered a BAD GPG signature", completed_process.stderr) 173 | 174 | def test_clearsign(self): 175 | completed_process = subprocess.run( 176 | [ 177 | "bmaptool", 178 | "copy", 179 | "--bmap", 180 | "tests/test-data/signatures/test.image.bmap.v2.0correct.asc", 181 | "tests/test-data/test.image.gz", 182 | self.tmpfile, 183 | ], 184 | stdout=subprocess.PIPE, 185 | stderr=subprocess.PIPE, 186 | check=False, 187 | ) 188 | self.assertEqual(completed_process.returncode, 0) 189 | self.assertEqual(completed_process.stdout, b"") 190 | self.assertIn( 191 | b"successfully verified bmap file signature", completed_process.stderr 192 | ) 193 | 194 | def test_fingerprint_without_signature(self): 195 | assert testkeys["correct"].fpr is not None 196 | completed_process = subprocess.run( 197 | [ 198 | "bmaptool", 199 | "copy", 200 | "--bmap", 201 | "tests/test-data/test.image.bmap.v2.0", 202 | "--fingerprint", 203 | testkeys["correct"].fpr, 204 | "tests/test-data/test.image.gz", 205 | self.tmpfile, 206 | ], 207 | stdout=subprocess.PIPE, 208 | stderr=subprocess.PIPE, 209 | check=False, 210 | ) 211 | self.assertEqual(completed_process.returncode, 1) 212 | self.assertEqual(completed_process.stdout, b"") 213 | self.assertIn( 214 | b"no signature found but --fingerprint given", completed_process.stderr 215 | ) 216 | 217 | def setUp(self): 218 | try: 219 | import gpg 220 | except ImportError: 221 | self.skipTest("python module 'gpg' missing") 222 | 223 | os.makedirs("tests/test-data/signatures", exist_ok=True) 224 | for key in testkeys.values(): 225 | assert key.gnupghome is None 226 | key.gnupghome = tempfile.mkdtemp(prefix="bmaptool") 227 | context = gpg.Context(home_dir=key.gnupghome) 228 | dmkey = context.create_key( 229 | key.uid, 230 | algorithm="rsa3072", 231 | expires_in=31536000, 232 | sign=True, 233 | certify=True, 234 | ) 235 | key.fpr = dmkey.fpr 236 | for bmapv in ["2.0", "1.4"]: 237 | testp = "tests/test-data" 238 | imbn = "test.image.bmap.v" 239 | with open(f"{testp}/{imbn}{bmapv}", "rb") as bmapf: 240 | bmapcontent = bmapf.read() 241 | with open( 242 | f"{testp}/signatures/{imbn}{bmapv}{key.uid.split()[0]}.asc", 243 | "wb", 244 | ) as sigf: 245 | signed_data, result = context.sign( 246 | bmapcontent, mode=gpg.constants.sig.mode.CLEAR 247 | ) 248 | sigf.write(signed_data) 249 | plaintext, sigs = context.verify(signed_data, None) 250 | with open( 251 | f"{testp}/signatures/{imbn}{bmapv}{key.uid.split()[0]}.det.asc", 252 | "wb", 253 | ) as detsigf: 254 | signed_data, result = context.sign( 255 | bmapcontent, mode=gpg.constants.sig.mode.DETACH 256 | ) 257 | detsigf.write(signed_data) 258 | # the file supplied to gpgv via --keyring must not be armored 259 | context.armor = False 260 | with open(f"{key.gnupghome}.keyring", "wb") as f: 261 | f.write(context.key_export_minimal()) 262 | 263 | self.tmpfile = tempfile.mkstemp(prefix="testfile_", dir=".")[1] 264 | os.environ["GNUPGHOME"] = testkeys["correct"].gnupghome 265 | 266 | def tearDown(self): 267 | os.unlink(self.tmpfile) 268 | for key in testkeys.values(): 269 | shutil.rmtree(key.gnupghome) 270 | os.unlink(f"{key.gnupghome}.keyring") 271 | key.gnupghome = None 272 | for bmapv in ["2.0", "1.4"]: 273 | testp = "tests/test-data" 274 | imbn = "test.image.bmap.v" 275 | os.unlink(f"{testp}/signatures/{imbn}{bmapv}{key.uid.split()[0]}.asc") 276 | os.unlink( 277 | f"{testp}/signatures/{imbn}{bmapv}{key.uid.split()[0]}.det.asc" 278 | ) 279 | os.rmdir("tests/test-data/signatures") 280 | 281 | 282 | if __name__ == "__main__": 283 | unittest.main() 284 | -------------------------------------------------------------------------------- /tests/test_api_base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ts=4 sw=4 tw=88 et ai si 3 | # 4 | # Copyright (c) 2012-2014 Intel, Inc. 5 | # License: GPLv2 6 | # Author: Artem Bityutskiy 7 | # 8 | # This program is free software; you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License, version 2, 10 | # as published by the Free Software Foundation. 11 | # 12 | # This program is distributed in the hope that it will be useful, but 13 | # WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | # General Public License for more details. 16 | 17 | """ 18 | This test verifies the base bmap creation and copying API functionality. It 19 | generates a random sparse file, then creates a bmap for this file and copies it 20 | to a different file using the bmap. Then it compares the original random sparse 21 | file and the copy and verifies that they are identical. 22 | """ 23 | 24 | # Disable the following pylint recommendations: 25 | # * Too many public methods (R0904) 26 | # * Too many local variables (R0914) 27 | # * Too many statements (R0915) 28 | # pylint: disable=R0904 29 | # pylint: disable=R0914 30 | # pylint: disable=R0915 31 | 32 | import os 33 | import sys 34 | import tempfile 35 | import filecmp 36 | import subprocess 37 | from itertools import zip_longest 38 | from tests import helpers 39 | from bmaptool import BmapHelpers, BmapCreate, Filemap 40 | 41 | # This is a work-around for Centos 6 42 | try: 43 | import unittest2 as unittest # pylint: disable=F0401 44 | except ImportError: 45 | import unittest 46 | 47 | 48 | class Error(Exception): 49 | """A class for exceptions generated by this test.""" 50 | 51 | pass 52 | 53 | 54 | def _compare_holes(file1, file2): 55 | """ 56 | Make sure that files 'file1' and 'file2' have holes at the same places. 57 | The 'file1' and 'file2' arguments may be full file paths or file objects. 58 | """ 59 | 60 | filemap1 = Filemap.filemap(file1) 61 | filemap2 = Filemap.filemap(file2) 62 | 63 | iterator1 = filemap1.get_unmapped_ranges(0, filemap1.blocks_cnt) 64 | iterator2 = filemap2.get_unmapped_ranges(0, filemap2.blocks_cnt) 65 | 66 | iterator = zip_longest(iterator1, iterator2) 67 | for range1, range2 in iterator: 68 | if range1 != range2: 69 | raise Error( 70 | "mismatch for hole %d-%d, it is %d-%d in file2" 71 | % (range1[0], range1[1], range2[0], range2[1]) 72 | ) 73 | 74 | 75 | def _generate_compressed_files(file_path, delete=True): 76 | """ 77 | This is a generator which yields compressed versions of a file 78 | 'file_path'. 79 | 80 | The 'delete' argument specifies whether the compressed files that this 81 | generator yields have to be automatically deleted. 82 | """ 83 | 84 | # Make sure the temporary files start with the same name as 'file_obj' in 85 | # order to simplify debugging. 86 | prefix = os.path.splitext(os.path.basename(file_path))[0] + "." 87 | # Put the temporary files in the directory with 'file_obj' 88 | directory = os.path.dirname(file_path) 89 | 90 | compressors = [ 91 | ("bzip2", None, ".bz2", "-c -k"), 92 | ("pbzip2", None, ".p.bz2", "-c -k"), 93 | ("gzip", None, ".gz", "-c"), 94 | ("pigz", None, ".p.gz", "-c -k"), 95 | ("xz", None, ".xz", "-c -k"), 96 | ("lzop", None, ".lzo", "-c -k"), 97 | ("lz4", None, ".lz4", "-c -k"), 98 | ("zstd", None, ".zst", "-c -k"), 99 | # The "-P -C /" trick is used to avoid silly warnings: 100 | # "tar: Removing leading `/' from member names" 101 | ("bzip2", "tar", ".tar.bz2", "-c -j -O -P -C /"), 102 | ("gzip", "tar", ".tar.gz", "-c -z -O -P -C /"), 103 | ("xz", "tar", ".tar.xz", "-c -J -O -P -C /"), 104 | ("lzop", "tar", ".tar.lzo", "-c --lzo -O -P -C /"), 105 | ("lz4", "tar", ".tar.lz4", "-c -Ilz4 -O -P -C /"), 106 | ("zstd", "tar", ".tar.zst", "-c -Izstd -O -P -C /"), 107 | ("zip", None, ".zip", "-q -j -"), 108 | ] 109 | 110 | for decompressor, archiver, suffix, options in compressors: 111 | if not BmapHelpers.program_is_available(decompressor): 112 | continue 113 | if archiver and not BmapHelpers.program_is_available(archiver): 114 | continue 115 | 116 | tmp_file_obj = tempfile.NamedTemporaryFile( 117 | "wb+", prefix=prefix, delete=delete, dir=directory, suffix=suffix 118 | ) 119 | 120 | if archiver: 121 | args = archiver + " " + options + " " + file_path 122 | else: 123 | args = decompressor + " " + options + " " + file_path 124 | child_process = subprocess.Popen( 125 | args, 126 | shell=True, 127 | stdout=tmp_file_obj, 128 | stderr=subprocess.DEVNULL, 129 | ) 130 | child_process.wait() 131 | tmp_file_obj.flush() 132 | yield tmp_file_obj.name 133 | tmp_file_obj.close() 134 | 135 | 136 | def _do_test(image, image_size, delete=True): 137 | """ 138 | A basic test for the bmap creation and copying functionality. It first 139 | generates a bmap for file 'image', and then copies the sparse file to a 140 | different file, and then checks that the original file and the copy are 141 | identical. 142 | 143 | The 'image_size' argument is size of the image in bytes. The 'delete' 144 | argument specifies whether the temporary files that this function creates 145 | have to be automatically deleted. 146 | """ 147 | 148 | try: 149 | Filemap.filemap(image) 150 | except Filemap.ErrorNotSupp as e: 151 | sys.stderr.write("%s\n" % e) 152 | return 153 | 154 | # Make sure the temporary files start with the same name as 'image' in 155 | # order to simplify debugging. 156 | prefix = os.path.splitext(os.path.basename(image))[0] + "." 157 | # Put the temporary files in the directory with the image 158 | directory = os.path.dirname(image) 159 | 160 | # Create and open a temporary file for a copy of the image 161 | f_copy = tempfile.NamedTemporaryFile( 162 | "wb+", prefix=prefix, delete=delete, dir=directory, suffix=".copy" 163 | ) 164 | 165 | # Create and open 2 temporary files for the bmap 166 | f_bmap1 = tempfile.NamedTemporaryFile( 167 | "w+", prefix=prefix, delete=delete, dir=directory, suffix=".bmap1" 168 | ) 169 | f_bmap2 = tempfile.NamedTemporaryFile( 170 | "w+", prefix=prefix, delete=delete, dir=directory, suffix=".bmap2" 171 | ) 172 | 173 | image_chksum = helpers.calculate_chksum(image) 174 | 175 | # 176 | # Pass 1: generate the bmap, copy and compare 177 | # 178 | 179 | # Create bmap for the random sparse file 180 | creator = BmapCreate.BmapCreate(image, f_bmap1.name) 181 | creator.generate() 182 | 183 | helpers.copy_and_verify_image( 184 | image, f_copy.name, f_bmap1.name, image_chksum, image_size 185 | ) 186 | 187 | # Make sure that holes in the copy are identical to holes in the random 188 | # sparse file. 189 | _compare_holes(image, f_copy.name) 190 | 191 | # 192 | # Pass 2: same as pass 1, but use file objects instead of paths 193 | # 194 | 195 | creator = BmapCreate.BmapCreate(image, f_bmap2) 196 | creator.generate() 197 | helpers.copy_and_verify_image( 198 | image, f_copy.name, f_bmap2.name, image_chksum, image_size 199 | ) 200 | _compare_holes(image, f_copy.name) 201 | 202 | # Make sure the bmap files generated at pass 1 and pass 2 are identical 203 | assert filecmp.cmp(f_bmap1.name, f_bmap2.name, False) 204 | 205 | # 206 | # Pass 3: test compressed files copying with bmap 207 | # 208 | 209 | for compressed in _generate_compressed_files(image, delete=delete): 210 | helpers.copy_and_verify_image( 211 | compressed, f_copy.name, f_bmap1.name, image_chksum, image_size 212 | ) 213 | 214 | # Test without setting the size 215 | helpers.copy_and_verify_image( 216 | compressed, f_copy.name, f_bmap1.name, image_chksum, None 217 | ) 218 | 219 | # Append a "file:" prefix to make BmapCopy use urllib 220 | compressed = "file:" + compressed 221 | helpers.copy_and_verify_image( 222 | compressed, f_copy.name, f_bmap1.name, image_chksum, image_size 223 | ) 224 | helpers.copy_and_verify_image( 225 | compressed, f_copy.name, f_bmap1.name, image_chksum, None 226 | ) 227 | 228 | # 229 | # Pass 5: copy without bmap and make sure it is identical to the original 230 | # file. 231 | 232 | helpers.copy_and_verify_image(image, f_copy.name, None, image_chksum, image_size) 233 | helpers.copy_and_verify_image(image, f_copy.name, None, image_chksum, None) 234 | 235 | # 236 | # Pass 6: test compressed files copying without bmap 237 | # 238 | 239 | for compressed in _generate_compressed_files(image, delete=delete): 240 | helpers.copy_and_verify_image( 241 | compressed, f_copy.name, f_bmap1.name, image_chksum, image_size 242 | ) 243 | 244 | # Test without setting the size 245 | helpers.copy_and_verify_image( 246 | compressed, f_copy.name, f_bmap1.name, image_chksum, None 247 | ) 248 | 249 | # Append a "file:" prefix to make BmapCopy use urllib 250 | helpers.copy_and_verify_image( 251 | compressed, f_copy.name, f_bmap1.name, image_chksum, image_size 252 | ) 253 | helpers.copy_and_verify_image( 254 | compressed, f_copy.name, f_bmap1.name, image_chksum, None 255 | ) 256 | 257 | # Close temporary files, which will also remove them 258 | f_copy.close() 259 | f_bmap1.close() 260 | f_bmap2.close() 261 | 262 | 263 | class TestCreateCopy(unittest.TestCase): 264 | """ 265 | The test class for this unit tests. Basically executes the '_do_test()' 266 | function for different sparse files. 267 | """ 268 | 269 | def test(self): # pylint: disable=R0201 270 | """ 271 | The test entry point. Executes the '_do_test()' function for files of 272 | different sizes, holes distribution and format. 273 | """ 274 | 275 | # Delete all the test-related temporary files automatically 276 | delete = True 277 | # Create all the test-related temporary files in current directory 278 | directory = "." 279 | 280 | iterator = helpers.generate_test_files(delete=delete, directory=directory) 281 | for f_image, image_size, _, _ in iterator: 282 | assert image_size == os.path.getsize(f_image.name) 283 | _do_test(f_image.name, image_size, delete=delete) 284 | -------------------------------------------------------------------------------- /tests/test_bmap_helpers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ts=4 sw=4 tw=88 et ai si 3 | # 4 | # Copyright (c) 2012-2014 Intel, Inc. 5 | # License: GPLv2 6 | # Author: Artem Bityutskiy 7 | # 8 | # This program is free software; you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License, version 2, 10 | # as published by the Free Software Foundation. 11 | # 12 | # This program is distributed in the hope that it will be useful, but 13 | # WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | # General Public License for more details. 16 | 17 | """ 18 | This test verifies 'BmapHelpers' module functionality. 19 | """ 20 | 21 | import os 22 | import sys 23 | import tempfile 24 | 25 | try: 26 | from unittest.mock import patch 27 | except ImportError: # for Python < 3.3 28 | from mock import patch 29 | try: 30 | from tempfile import TemporaryDirectory 31 | except ImportError: # for Python < 3.2 32 | from backports.tempfile import TemporaryDirectory 33 | from bmaptool import BmapHelpers 34 | 35 | 36 | # This is a work-around for Centos 6 37 | try: 38 | import unittest2 as unittest # pylint: disable=F0401 39 | except ImportError: 40 | import unittest 41 | 42 | 43 | class TestBmapHelpers(unittest.TestCase): 44 | """The test class for these unit tests.""" 45 | 46 | def test_get_file_system_type(self): 47 | """Check a file system type is returned when used with a file""" 48 | 49 | with tempfile.NamedTemporaryFile( 50 | "r", prefix="testfile_", delete=True, dir=".", suffix=".img" 51 | ) as fobj: 52 | fstype = BmapHelpers.get_file_system_type(fobj.name) 53 | self.assertTrue(fstype) 54 | 55 | def test_get_file_system_type_no_fstype_found(self): 56 | """Check error raised when supplied file doesn't exist""" 57 | 58 | directory = os.path.dirname(__file__) 59 | fobj = os.path.join(directory, "BmapHelpers/file/does/not/exist") 60 | with self.assertRaises(BmapHelpers.Error): 61 | BmapHelpers.get_file_system_type(fobj) 62 | 63 | def test_get_file_system_type_symlink(self): 64 | """Check a file system type is returned when used with a symlink""" 65 | 66 | with TemporaryDirectory(prefix="testdir_", dir=".") as directory: 67 | with tempfile.NamedTemporaryFile( 68 | "r", prefix="testfile_", delete=False, dir=directory, suffix=".img" 69 | ) as fobj: 70 | lnk = os.path.join(directory, "test_symlink") 71 | os.symlink(fobj.name, lnk) 72 | fstype = BmapHelpers.get_file_system_type(lnk) 73 | self.assertTrue(fstype) 74 | 75 | def test_is_zfs_configuration_compatible_enabled(self): 76 | """Check compatibility check is true when zfs param is set correctly""" 77 | 78 | with tempfile.NamedTemporaryFile( 79 | "w+", prefix="testfile_", delete=True, dir=".", suffix=".txt" 80 | ) as fobj: 81 | fobj.write("1") 82 | fobj.flush() 83 | mockobj = patch.object(BmapHelpers, "ZFS_COMPAT_PARAM_PATH", fobj.name) 84 | with mockobj: 85 | self.assertTrue(BmapHelpers.is_zfs_configuration_compatible()) 86 | 87 | def test_is_zfs_configuration_compatible_disabled(self): 88 | """Check compatibility check is false when zfs param is set incorrectly""" 89 | 90 | with tempfile.NamedTemporaryFile( 91 | "w+", prefix="testfile_", delete=True, dir=".", suffix=".txt" 92 | ) as fobj: 93 | fobj.write("0") 94 | fobj.flush() 95 | mockobj = patch.object(BmapHelpers, "ZFS_COMPAT_PARAM_PATH", fobj.name) 96 | with mockobj: 97 | self.assertFalse(BmapHelpers.is_zfs_configuration_compatible()) 98 | 99 | def test_is_zfs_configuration_compatible_invalid_read_value(self): 100 | """Check error raised if any content of zfs config file invalid""" 101 | 102 | with tempfile.NamedTemporaryFile( 103 | "a", prefix="testfile_", delete=True, dir=".", suffix=".txt" 104 | ) as fobj: 105 | mockobj = patch.object(BmapHelpers, "ZFS_COMPAT_PARAM_PATH", fobj.name) 106 | with self.assertRaises(BmapHelpers.Error): 107 | with mockobj: 108 | BmapHelpers.is_zfs_configuration_compatible() 109 | 110 | @patch("builtins.open" if sys.version_info[0] >= 3 else "__builtin__.open") 111 | def test_is_zfs_configuration_compatible_unreadable_file(self, mock_open): 112 | """Check error raised if any IO errors when checking zfs config file""" 113 | 114 | mock_open.side_effect = IOError 115 | with self.assertRaises(BmapHelpers.Error): 116 | if not BmapHelpers.is_zfs_configuration_compatible(): 117 | raise BmapHelpers.Error 118 | 119 | def test_is_zfs_configuration_compatible_not_installed(self): 120 | """Check compatibility check passes when zfs not installed""" 121 | 122 | directory = os.path.dirname(__file__) 123 | filepath = os.path.join(directory, "BmapHelpers/file/does/not/exist") 124 | mockobj = patch.object(BmapHelpers, "ZFS_COMPAT_PARAM_PATH", filepath) 125 | with mockobj: 126 | self.assertFalse(BmapHelpers.is_zfs_configuration_compatible()) 127 | 128 | @patch.object(BmapHelpers, "get_file_system_type", return_value="zfs") 129 | def test_is_compatible_file_system_zfs_valid( 130 | self, mock_get_fs_type 131 | ): # pylint: disable=unused-argument 132 | """Check compatibility check passes when zfs param is set correctly""" 133 | 134 | with tempfile.NamedTemporaryFile( 135 | "w+", prefix="testfile_", delete=True, dir=".", suffix=".img" 136 | ) as fobj: 137 | fobj.write("1") 138 | fobj.flush() 139 | mockobj = patch.object(BmapHelpers, "ZFS_COMPAT_PARAM_PATH", fobj.name) 140 | with mockobj: 141 | self.assertTrue(BmapHelpers.is_compatible_file_system(fobj.name)) 142 | 143 | @patch.object(BmapHelpers, "get_file_system_type", return_value="zfs") 144 | def test_is_compatible_file_system_zfs_invalid( 145 | self, mock_get_fs_type 146 | ): # pylint: disable=unused-argument 147 | """Check compatibility check fails when zfs param is set incorrectly""" 148 | 149 | with tempfile.NamedTemporaryFile( 150 | "w+", prefix="testfile_", delete=True, dir=".", suffix=".img" 151 | ) as fobj: 152 | fobj.write("0") 153 | fobj.flush() 154 | mockobj = patch.object(BmapHelpers, "ZFS_COMPAT_PARAM_PATH", fobj.name) 155 | with mockobj: 156 | self.assertFalse(BmapHelpers.is_compatible_file_system(fobj.name)) 157 | 158 | @patch.object(BmapHelpers, "get_file_system_type", return_value="ext4") 159 | def test_is_compatible_file_system_ext4( 160 | self, mock_get_fs_type 161 | ): # pylint: disable=unused-argument 162 | """Check non-zfs file systems pass compatibility checks""" 163 | 164 | with tempfile.NamedTemporaryFile( 165 | "w+", prefix="testfile_", delete=True, dir=".", suffix=".img" 166 | ) as fobj: 167 | self.assertTrue(BmapHelpers.is_compatible_file_system(fobj.name)) 168 | -------------------------------------------------------------------------------- /tests/test_compat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ts=4 sw=4 tw=88 et ai si 3 | # 4 | # Copyright (c) 2012-2014 Intel, Inc. 5 | # License: GPLv2 6 | # Author: Artem Bityutskiy 7 | # 8 | # This program is free software; you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License, version 2, 10 | # as published by the Free Software Foundation. 11 | # 12 | # This program is distributed in the hope that it will be useful, but 13 | # WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | # General Public License for more details. 16 | 17 | """ 18 | This unit test verifies various compatibility aspects of the BmapCopy module: 19 | * current BmapCopy has to handle all the older bmap formats 20 | * older BmapCopy have to handle all the newer compatible bmap formats 21 | """ 22 | 23 | # Disable the following pylint recommendations: 24 | # * Too many public methods (R0904) 25 | # * Attribute 'XYZ' defined outside __init__ (W0201), because unittest 26 | # classes are not supposed to have '__init__()' 27 | # pylint: disable=R0904 28 | # pylint: disable=W0201 29 | 30 | import os 31 | import shutil 32 | import tempfile 33 | from tests import helpers 34 | from bmaptool import TransRead, BmapCopy 35 | 36 | # This is a work-around for Centos 6 37 | try: 38 | import unittest2 as unittest # pylint: disable=F0401 39 | except ImportError: 40 | import unittest 41 | 42 | # Test image file name 43 | _IMAGE_NAME = "test.image.gz" 44 | # Test bmap file names template 45 | _BMAP_TEMPL = "test.image.bmap.v" 46 | # Name of the subdirectory where test data are stored 47 | _TEST_DATA_SUBDIR = "test-data" 48 | # Name of the subdirectory where old BmapCopy modules are stored 49 | _OLDCODEBASE_SUBDIR = "oldcodebase" 50 | 51 | 52 | class TestCompat(unittest.TestCase): 53 | """The test class for this unit test.""" 54 | 55 | def test(self): 56 | """The test entry point.""" 57 | 58 | test_data_dir = os.path.join(os.path.dirname(__file__), _TEST_DATA_SUBDIR) 59 | image_path = os.path.join(test_data_dir, _IMAGE_NAME) 60 | 61 | # Construct the list of bmap files to test 62 | self._bmap_paths = [] 63 | for dentry in os.listdir(test_data_dir): 64 | dentry_path = os.path.join(test_data_dir, dentry) 65 | if os.path.isfile(dentry_path) and dentry.startswith(_BMAP_TEMPL): 66 | self._bmap_paths.append(dentry_path) 67 | 68 | # Create and open a temporary file for uncompressed image and its copy 69 | self._f_image = tempfile.NamedTemporaryFile( 70 | "wb+", prefix=_IMAGE_NAME, suffix=".image" 71 | ) 72 | self._f_copy = tempfile.NamedTemporaryFile( 73 | "wb+", prefix=_IMAGE_NAME, suffix=".copy" 74 | ) 75 | 76 | # Uncompress the test image into 'self._f_image' 77 | f_tmp_img = TransRead.TransRead(image_path) 78 | shutil.copyfileobj(f_tmp_img, self._f_image) 79 | f_tmp_img.close() 80 | self._f_image.flush() 81 | 82 | image_chksum = helpers.calculate_chksum(self._f_image.name) 83 | image_size = os.path.getsize(self._f_image.name) 84 | 85 | # Test the current version of BmapCopy 86 | for bmap_path in self._bmap_paths: 87 | helpers.copy_and_verify_image( 88 | image_path, self._f_copy.name, bmap_path, image_chksum, image_size 89 | ) 90 | 91 | # Test the older versions of BmapCopy 92 | self._test_older_bmapcopy() 93 | 94 | self._f_copy.close() 95 | self._f_image.close() 96 | 97 | def _test_older_bmapcopy(self): 98 | """Test older than the current versions of the BmapCopy class.""" 99 | 100 | def import_module(searched_module): 101 | """Search and import a module by its name.""" 102 | 103 | modref = __import__(searched_module) 104 | for name in searched_module.split(".")[1:]: 105 | modref = getattr(modref, name) 106 | return modref 107 | 108 | oldcodebase_dir = os.path.join(os.path.dirname(__file__), _OLDCODEBASE_SUBDIR) 109 | 110 | # Construct the list of old BmapCopy modules 111 | old_modules = [] 112 | for dentry in os.listdir(oldcodebase_dir): 113 | if dentry.startswith("BmapCopy") and dentry.endswith(".py"): 114 | old_modules.append("tests." + _OLDCODEBASE_SUBDIR + "." + dentry[:-3]) 115 | 116 | for old_module in old_modules: 117 | modref = import_module(old_module) 118 | 119 | for bmap_path in self._bmap_paths: 120 | self._do_test_older_bmapcopy(bmap_path, modref) 121 | 122 | def _do_test_older_bmapcopy(self, bmap_path, modref): 123 | """ 124 | Test an older version of BmapCopy class, referenced by the 'modref' 125 | argument. The 'bmap_path' argument is the bmap file path to test with. 126 | """ 127 | 128 | # Get a reference to the older BmapCopy class object to test with 129 | old_bmapcopy_class = getattr(modref, "BmapCopy") 130 | supported_ver = getattr(modref, "SUPPORTED_BMAP_VERSION") 131 | 132 | f_bmap = open(bmap_path, "r") 133 | 134 | # Find the version of the bmap file. The easiest is to simply use the 135 | # latest BmapCopy. 136 | bmapcopy = BmapCopy.BmapCopy(self._f_image, self._f_copy, f_bmap) 137 | bmap_version = bmapcopy.bmap_version 138 | bmap_version_major = bmapcopy.bmap_version_major 139 | 140 | try: 141 | if supported_ver >= bmap_version: 142 | writer = old_bmapcopy_class(self._f_image, self._f_copy, f_bmap) 143 | writer.copy(True, True) 144 | except: # pylint: disable=W0702 145 | if supported_ver >= bmap_version_major: 146 | # The BmapCopy which we are testing is supposed to support this 147 | # version of bmap file format. However, bmap format version 1.4 148 | # was a screw-up, because it actually had incompatible changes, 149 | # so old versions of BmapCopy are supposed to fail. 150 | if not (supported_ver == 1 and bmap_version == "1.4"): 151 | print( 152 | 'Module "%s" failed to handle "%s"' 153 | % (modref.__name__, bmap_path) 154 | ) 155 | raise 156 | 157 | f_bmap.close() 158 | -------------------------------------------------------------------------------- /tests/test_filemap.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ts=4 sw=4 tw=88 et ai si 3 | # 4 | # Copyright (c) 2012-2014 Intel, Inc. 5 | # License: GPLv2 6 | # Author: Artem Bityutskiy 7 | # 8 | # This program is free software; you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License, version 2, 10 | # as published by the Free Software Foundation. 11 | # 12 | # This program is distributed in the hope that it will be useful, but 13 | # WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | # General Public License for more details. 16 | 17 | """ 18 | This test verifies 'Filemap' module functionality. It generates random sparse 19 | files and makes sure the module returns correct information about the holes. 20 | """ 21 | 22 | # Disable the following pylint recommendations: 23 | # * Too many public methods - R0904 24 | # * Too many arguments - R0913 25 | # pylint: disable=R0904 26 | # pylint: disable=R0913 27 | 28 | import sys 29 | import random 30 | import itertools 31 | import tests.helpers 32 | from itertools import zip_longest 33 | from bmaptool import Filemap 34 | 35 | # This is a work-around for Centos 6 36 | try: 37 | import unittest2 as unittest # pylint: disable=F0401 38 | except ImportError: 39 | import unittest 40 | 41 | 42 | class Error(Exception): 43 | """A class for exceptions generated by this test.""" 44 | 45 | pass 46 | 47 | 48 | def _check_ranges(f_image, filemap, first_block, blocks_cnt, ranges, ranges_type): 49 | """ 50 | This is a helper function for '_do_test()' which compares the correct 51 | 'ranges' list of mapped or unmapped blocks ranges for file object 'f_image' 52 | with what the 'Filemap' module reports. The 'ranges_type' argument defines 53 | whether the 'ranges' list is a list of mapped or unmapped blocks. The 54 | 'first_block' and 'blocks_cnt' define the subset of blocks in 'f_image' 55 | that should be verified by this function. 56 | """ 57 | 58 | if ranges_type == "mapped": 59 | filemap_iterator = filemap.get_mapped_ranges(first_block, blocks_cnt) 60 | elif ranges_type == "unmapped": 61 | filemap_iterator = filemap.get_unmapped_ranges(first_block, blocks_cnt) 62 | else: 63 | raise Error("incorrect list type") 64 | 65 | last_block = first_block + blocks_cnt - 1 66 | 67 | # The 'ranges' list contains all ranges, from block zero to the last 68 | # block. However, we are conducting a test for 'blocks_cnt' of blocks 69 | # starting from block 'first_block'. Create an iterator which filters 70 | # those block ranges from the 'ranges' list, that are out of the 71 | # 'first_block'/'blocks_cnt' file region. 72 | ranges_iterator = (x for x in ranges if x[1] >= first_block and x[0] <= last_block) 73 | iterator = zip_longest(ranges_iterator, filemap_iterator) 74 | 75 | # Iterate over both - the (filtered) 'ranges' list which contains correct 76 | # ranges and the Filemap generator, and verify the mapped/unmapped ranges 77 | # returned by the 'Filemap' module. 78 | for correct, check in iterator: 79 | # The first and the last range of the filtered 'ranges' list may still 80 | # be out of the limit - correct them in this case 81 | if correct[0] < first_block: 82 | correct = (first_block, correct[1]) 83 | if correct[1] > last_block: 84 | correct = (correct[0], last_block) 85 | 86 | if check[0] > check[1] or check != correct: 87 | raise Error( 88 | "bad or mismatching %s range for file '%s': correct " 89 | "is %d-%d, get_%s_ranges(%d, %d) returned %d-%d" 90 | % ( 91 | ranges_type, 92 | f_image.name, 93 | correct[0], 94 | correct[1], 95 | ranges_type, 96 | first_block, 97 | blocks_cnt, 98 | check[0], 99 | check[1], 100 | ) 101 | ) 102 | 103 | for block in range(correct[0], correct[1] + 1): 104 | if ranges_type == "mapped" and filemap.block_is_unmapped(block): 105 | raise Error( 106 | "range %d-%d of file '%s' is mapped, but" 107 | "'block_is_unmapped(%d) returned 'True'" 108 | % (correct[0], correct[1], f_image.name, block) 109 | ) 110 | if ranges_type == "unmapped" and filemap.block_is_mapped(block): 111 | raise Error( 112 | "range %d-%d of file '%s' is unmapped, but" 113 | "'block_is_mapped(%d) returned 'True'" 114 | % (correct[0], correct[1], f_image.name, block) 115 | ) 116 | 117 | 118 | def _do_test(f_image, filemap, mapped, unmapped): 119 | """ 120 | Verify that the 'Filemap' module provides correct mapped and unmapped areas 121 | for the 'f_image' file object. The 'mapped' and 'unmapped' lists contain 122 | the correct ranges. The 'filemap' is one of the classed from the 'Filemap' 123 | module. 124 | """ 125 | 126 | # Check both 'get_mapped_ranges()' and 'get_unmapped_ranges()' for the 127 | # entire file. 128 | first_block = 0 129 | blocks_cnt = filemap.blocks_cnt 130 | _check_ranges(f_image, filemap, first_block, blocks_cnt, mapped, "mapped") 131 | _check_ranges(f_image, filemap, first_block, blocks_cnt, unmapped, "unmapped") 132 | 133 | # Select a random area in the file and repeat the test few times 134 | for _ in range(0, 10): 135 | first_block = random.randint(0, filemap.blocks_cnt - 1) 136 | blocks_cnt = random.randint(1, filemap.blocks_cnt - first_block) 137 | _check_ranges(f_image, filemap, first_block, blocks_cnt, mapped, "mapped") 138 | _check_ranges(f_image, filemap, first_block, blocks_cnt, unmapped, "unmapped") 139 | 140 | 141 | class TestFilemap(unittest.TestCase): 142 | """ 143 | The test class for this unit tests. Basically executes the '_do_test()' 144 | function for different sparse files. 145 | """ 146 | 147 | def test(self): # pylint: disable=R0201 148 | """ 149 | The test entry point. Executes the '_do_test()' function for files of 150 | different sizes, holes distribution and format. 151 | """ 152 | 153 | # Delete all the test-related temporary files automatically 154 | delete = True 155 | # Create all the test-related temporary files in current directory 156 | directory = "." 157 | # Maximum size of the random files used in this test 158 | max_size = 16 * 1024 * 1024 159 | 160 | iterator = tests.helpers.generate_test_files(max_size, directory, delete) 161 | for f_image, _, mapped, unmapped in iterator: 162 | try: 163 | fiemap = Filemap.FilemapFiemap(f_image) 164 | _do_test(f_image, fiemap, mapped, unmapped) 165 | 166 | seek = Filemap.FilemapSeek(f_image) 167 | _do_test(f_image, seek, mapped, unmapped) 168 | except Filemap.ErrorNotSupp: 169 | pass 170 | --------------------------------------------------------------------------------