├── .gitignore ├── .gitmodules ├── .travis.yml ├── COPYING ├── MANIFEST.in ├── README.rst ├── bedup ├── __init__.py ├── __main__.py ├── compat.py ├── datetime.py ├── dedup.py ├── filesystem.py ├── hashing.py ├── main.py ├── migrations.py ├── model.py ├── platform │ ├── __init__.py │ ├── btrfs.py │ ├── cffi_support.py │ ├── chattr.py │ ├── fiemap.py │ ├── futimens.py │ ├── ioprio.py │ ├── openat.py │ ├── syncfs.py │ ├── time.py │ └── unshare.py ├── termupdates.py ├── test_bedup.py └── tracking.py ├── setup.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[oc] 2 | /.coverage 3 | /.tox/ 4 | /MANIFEST 5 | /bedup.egg-info/ 6 | /bedup/platform/__pycache__/ 7 | # These are created under ext_package in develop mode: 8 | /bedup/platform/_cffi__*.so 9 | # these can be removed manually: 10 | /bedup/_cffi__*.so 11 | /build/ 12 | /dist/ 13 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "btrfs"] 2 | path = btrfs 3 | url = https://git.kernel.org/pub/scm/linux/kernel/git/mason/btrfs-progs.git 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - "3.3" 5 | - "3.4" 6 | - "3.5" 7 | - "pypy3" 8 | 9 | matrix: 10 | allow_failures: 11 | # Neither vido nor bedup run on 3.2, PyPy stdlib is still 3.2 12 | - python: pypy3 13 | 14 | install: 15 | - uname -a 16 | - lsb_release -a 17 | - python3 --version 18 | - pypy3 --version 19 | - sudo touch /etc/suid-debug 20 | # Travis has a python3, but the /usr/bin/python3 command is Python 3.2, beurk 21 | - sudo apt-get -y install libffi-dev btrfs-tools 22 | - which python3 23 | - which pypy3 24 | - pip install cffi pytest pytest-cov coveralls git+https://github.com/g2p/vido.git@for-travis 25 | - curl --compressed -LO https://github.com/g2p/kernels/raw/master/linux.uml 26 | - chmod +x linux.uml 27 | - pip install . 28 | - sudo locale-gen en_US.UTF-8 29 | 30 | # tox has some advantages over travis runners: 31 | # it tests installation from the sdist, which will 32 | # report things like missing header files. 33 | 34 | script: 35 | - time LC_ALL=en_US.UTF-8 vido --pass-env LC_ALL --kernel=./linux.uml -- python3 -Wd -bb -Wignore:::site -Wignore:::cffi.verifier -Wignore:::_pytest.main -Wignore:::_pytest.assertion.oldinterpret -m pytest --cov=bedup --capture=no bedup/ 36 | 37 | after_success: 38 | - coveralls 39 | 40 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # We only include the first two directly, 2 | # but ctree.h pulls in a lot more stuff. 3 | include btrfs/ioctl.h 4 | include btrfs/ctree.h 5 | include btrfs/list.h 6 | include btrfs/kerncompat.h 7 | include btrfs/radix-tree.h 8 | include btrfs/extent-cache.h 9 | include btrfs/rbtree.h 10 | include btrfs/extent_io.h 11 | 12 | include COPYING 13 | include README.rst 14 | 15 | include tox.ini 16 | 17 | # contains cffi-generated C files 18 | prune bedup/__pycache__ 19 | prune bedup/platform/__pycache__ 20 | 21 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Deduplication for Btrfs. 2 | 3 | bedup looks for new and changed files, making sure that multiple copies of 4 | identical files share space on disk. It integrates deeply with btrfs so that 5 | scans are incremental and low-impact. 6 | 7 | Requirements 8 | ============ 9 | 10 | You need Python 3.3 or newer, and Linux 3.3 or newer. 11 | Linux 3.9.4 or newer is recommended, because it fixes a scanning bug 12 | and is compatible with cross-volume deduplication. 13 | 14 | This should get you started on Ubuntu 16.04: 15 | 16 | :: 17 | 18 | sudo aptitude install python3-pip python3-dev python3-cffi libffi-dev build-essential git 19 | 20 | This should get you started on earlier versions of Debian/Ubuntu: 21 | 22 | :: 23 | 24 | sudo aptitude install python3-pip python3-dev libffi-dev build-essential git 25 | 26 | This should get you started on Fedora: 27 | 28 | :: 29 | 30 | yum install python3-pip python3-devel libffi-devel gcc git 31 | 32 | Installation 33 | ============ 34 | 35 | On systems other than Ubuntu 16.04 you need to install CFFI: 36 | 37 | :: 38 | 39 | pip3 install --user cffi 40 | 41 | Option 1 (recommended): from a git clone 42 | ---------------------------------------- 43 | 44 | Enable submodules (this will pull headers from btrfs-progs) 45 | 46 | :: 47 | 48 | git submodule update --init 49 | 50 | Complete the installation. This will compile some code with CFFI and 51 | pull the rest of our Python dependencies: 52 | 53 | :: 54 | 55 | python3 setup.py install --user 56 | cp -lt ~/bin ~/.local/bin/bedup 57 | 58 | Option 2: from a PyPI release 59 | ----------------------------- 60 | 61 | :: 62 | 63 | pip3 install --user bedup 64 | cp -lt ~/bin ~/.local/bin/bedup 65 | 66 | Running 67 | ======= 68 | 69 | :: 70 | 71 | bedup --help 72 | bedup --help 73 | 74 | On Debian and Fedora, you may need to use `sudo -E ~/bin/bedup` or install cffi 75 | and bedup as root (bedup and its dependencies will get installed to /usr/local). 76 | 77 | You'll see a list of supported commands. 78 | 79 | - **scan** scans volumes to keep track of potentially duplicated files. 80 | - **dedup** runs scan, then deduplicates identical files. 81 | - **show** shows btrfs filesystems and their tracking status. 82 | - **dedup-files** takes a list of identical files and deduplicates them. 83 | - **find-new** reimplements the ``btrfs subvolume find-new`` command 84 | with a few extra options. 85 | 86 | To deduplicate all filesystems: :: 87 | 88 | sudo bedup dedup 89 | 90 | Unmounted or read-only filesystems are excluded if they aren't listed 91 | on the command line. 92 | Filesystems can be referenced by uuid or by a path in /dev: :: 93 | 94 | sudo bedup dedup /dev/disks/by-label/Btrfs 95 | 96 | Giving a subvolume path also works, and will include subvolumes by default. 97 | 98 | Since cross-subvolume deduplication requires Linux 3.6, users of older 99 | kernels should use the ``--no-crossvol`` flag. 100 | 101 | Hacking 102 | ======= 103 | 104 | :: 105 | 106 | pip3 install --user pytest tox ipdb https://github.com/jbalogh/check 107 | 108 | To run the tests:: 109 | 110 | sudo python3 -m pytest -s bedup 111 | 112 | To test compatibility and packaging as well:: 113 | 114 | GETROOT=/usr/bin/sudo tox 115 | 116 | Run a style check on edited files:: 117 | 118 | check.py 119 | 120 | Implementation 121 | ============== 122 | 123 | Deduplication is implemented using a Btrfs feature that allows for 124 | cloning data from one file to the other. The cloned ranges become shared 125 | on disk, saving space. 126 | 127 | File metadata isn't affected, and later changes to one file won't affect 128 | the other (this is unlike hard-linking). 129 | 130 | This approach doesn't require special kernel support, but it has two 131 | downsides: locking has to be done in userspace, and there is no way to 132 | free space within read-only (frozen) snapshots. 133 | 134 | Scanning 135 | -------- 136 | 137 | Scanning is done incrementally, the technique is similar to ``btrfs subvolume 138 | find-new``. You need an up-to-date kernel (3.10, 3.9.4, 3.8.13.1, 3.6.11.5, 139 | 3.5.7.14, 3.4.47) to index all files; earlier releases have a bug that 140 | causes find-new to end prematurely. The fix can also be cherry-picked 141 | from `this commit 142 | `_. 143 | 144 | Locking 145 | ------- 146 | 147 | Before cloning, we need to lock the files so that their contents don't 148 | change from the time the data is compared to the time it is cloned. 149 | Implementation note: This is done by setting the immutable attribute on 150 | the file, scanning /proc to see if some processes still have write 151 | access to the file (via preexisting file descriptors or memory 152 | mappings), bailing if the file is in write use. If all is well, the 153 | comparison and cloning steps can proceed. The immutable attribute is 154 | then reverted. 155 | 156 | This locking process might not be fool-proof in all cases; for example a 157 | malicious application might manage to bypass it, which would allow it to 158 | change the contents of files it doesn't have access to. 159 | 160 | There is also a small time window when an application will get 161 | permission errors, if it tries to get write access to a file we have 162 | already started to deduplicate. 163 | 164 | Finally, a system crash at the wrong time could leave some files immutable. 165 | They will be reported at the next run; fix them using the ``chattr -i`` 166 | command. 167 | 168 | Subvolumes 169 | ---------- 170 | 171 | The clone call is considered a write operation and won't work on 172 | read-only snapshots. 173 | 174 | Before Linux 3.6, the clone call didn't work across subvolumes. 175 | 176 | Defragmentation 177 | --------------- 178 | 179 | Before Linux 3.9, defragmentation could break copy-on-write sharing, 180 | which made it inadvisable when snapshots or deduplication are used. 181 | Btrfs defragmentation has to be explicitly requested (or background 182 | defragmentation enabled), so this generally shouldn't be a problem for 183 | users who were unaware of the feature. 184 | 185 | Users of Linux 3.9 or newer can safely pass the `--defrag` option to 186 | `bedup dedup`, which will defragment files before deduplicating them. 187 | 188 | Reporting bugs 189 | ============== 190 | 191 | Be sure to mention the following: 192 | 193 | - Linux kernel version: uname -rv 194 | - Python version 195 | - Distribution 196 | 197 | And give some of the program output. 198 | 199 | Build status 200 | ============ 201 | 202 | .. image:: https://travis-ci.org/g2p/bedup.png 203 | :target: https://travis-ci.org/g2p/bedup 204 | 205 | -------------------------------------------------------------------------------- /bedup/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g2p/bedup/9694f6f718844c33017052eb271f68b6c0d0b7d3/bedup/__init__.py -------------------------------------------------------------------------------- /bedup/__main__.py: -------------------------------------------------------------------------------- 1 | # vim: set fileencoding=utf-8 sw=4 ts=4 et : 2 | 3 | # bedup - Btrfs deduplication 4 | # Copyright (C) 2015 Gabriel de Perthuis 5 | # 6 | # This file is part of bedup. 7 | # 8 | # bedup is free software: you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by 10 | # the Free Software Foundation, either version 2 of the License, or 11 | # (at your option) any later version. 12 | # 13 | # bedup is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License 19 | # along with bedup. If not, see . 20 | 21 | 22 | 23 | import argparse 24 | import codecs 25 | import errno 26 | import locale 27 | import os 28 | import sqlalchemy 29 | import sys 30 | import warnings 31 | import xdg.BaseDirectory # pyxdg, apt:python-xdg 32 | 33 | from collections import defaultdict, OrderedDict 34 | from contextlib import closing, ExitStack 35 | from sqlalchemy.orm import sessionmaker 36 | from sqlalchemy.pool import SingletonThreadPool 37 | from uuid import UUID 38 | 39 | from .platform.btrfs import find_new, get_root_generation 40 | from .platform.ioprio import set_idle_priority 41 | from .platform.syncfs import syncfs 42 | 43 | from .dedup import dedup_same, FilesInUseError 44 | from .filesystem import show_vols, WholeFS, NotAVolume 45 | from .migrations import upgrade_schema 46 | from .termupdates import TermTemplate 47 | from .tracking import ( 48 | track_updated_files, dedup_tracked, reset_vol, fake_updates, 49 | annotated_inodes_by_size) 50 | 51 | 52 | APP_NAME = 'bedup' 53 | 54 | 55 | def cmd_dedup_files(args): 56 | try: 57 | return dedup_same(args.source, args.dests, args.defrag) 58 | except FilesInUseError as exn: 59 | exn.describe(sys.stderr) 60 | return 1 61 | 62 | 63 | def cmd_find_new(args): 64 | volume_fd = os.open(args.volume, os.O_DIRECTORY) 65 | if args.zero_terminated: 66 | sep = '\0' 67 | else: 68 | sep = '\n' 69 | find_new(volume_fd, args.generation, sys.stdout, terse=args.terse, sep=sep) 70 | 71 | 72 | def cmd_show_vols(args): 73 | sess = get_session(args) 74 | whole_fs = WholeFS(sess) 75 | show_vols(whole_fs, args.fsuuid_or_device, args.show_deleted) 76 | 77 | 78 | def sql_setup(dbapi_con, con_record): 79 | cur = dbapi_con.cursor() 80 | # Uncripple the SQL implementation 81 | cur.execute('PRAGMA foreign_keys = ON') 82 | cur.execute('PRAGMA foreign_keys') 83 | val = cur.fetchone() 84 | assert val == (1,), val 85 | 86 | # So that writers do not block readers 87 | # https://www.sqlite.org/wal.html 88 | cur.execute('PRAGMA journal_mode = WAL') 89 | cur.execute('PRAGMA journal_mode') 90 | val = cur.fetchone() 91 | # SQLite 3.7 is required 92 | assert val == ('wal',), val 93 | 94 | 95 | def get_session(args): 96 | if args.db_path is None: 97 | data_dir = xdg.BaseDirectory.save_data_path(APP_NAME) 98 | args.db_path = os.path.join(data_dir, 'db.sqlite') 99 | url = sqlalchemy.engine.url.URL('sqlite', database=args.db_path) 100 | engine = sqlalchemy.engine.create_engine( 101 | url, echo=args.verbose_sql, poolclass=SingletonThreadPool) 102 | sqlalchemy.event.listen(engine, 'connect', sql_setup) 103 | upgrade_schema(engine) 104 | Session = sessionmaker(bind=engine) 105 | sess = Session() 106 | return sess 107 | 108 | 109 | def vol_cmd(args): 110 | if args.command == 'dedup-vol': 111 | sys.stderr.write( 112 | "The dedup-vol command is deprecated, please use dedup.\n") 113 | args.command = 'dedup' 114 | args.defrag = False 115 | elif args.command == 'reset' and not args.filter: 116 | sys.stderr.write("You need to list volumes explicitly.\n") 117 | return 1 118 | 119 | with ExitStack() as stack: 120 | tt = stack.enter_context(closing(TermTemplate())) 121 | # Adds about 1s to cold startup 122 | sess = get_session(args) 123 | whole_fs = WholeFS(sess, size_cutoff=args.size_cutoff) 124 | stack.enter_context(closing(whole_fs)) 125 | 126 | if not args.filter: 127 | vols = whole_fs.load_all_writable_vols(tt) 128 | else: 129 | vols = OrderedDict() 130 | for filt in args.filter: 131 | if filt.startswith('vol:/'): 132 | volpath = filt[4:] 133 | try: 134 | filt_vols = whole_fs.load_vols( 135 | [volpath], tt, recurse=False) 136 | except NotAVolume: 137 | sys.stderr.write( 138 | 'Path doesn\'t point to a btrfs volume: %r\n' 139 | % (volpath,)) 140 | return 1 141 | elif filt.startswith('/'): 142 | if os.path.realpath(filt).startswith('/dev/'): 143 | filt_vols = whole_fs.load_vols_for_device(filt, tt) 144 | else: 145 | volpath = filt 146 | try: 147 | filt_vols = whole_fs.load_vols( 148 | [volpath], tt, recurse=True) 149 | except NotAVolume: 150 | sys.stderr.write( 151 | 'Path doesn\'t point to a btrfs volume: %r\n' 152 | % (volpath,)) 153 | return 1 154 | else: 155 | try: 156 | uuid = UUID(hex=filt) 157 | except ValueError: 158 | sys.stderr.write( 159 | 'Filter format not recognised: %r\n' % filt) 160 | return 1 161 | filt_vols = whole_fs.load_vols_for_fs( 162 | whole_fs.get_fs(uuid), tt) 163 | for vol in filt_vols: 164 | vols[vol] = True 165 | 166 | # XXX should group by mountpoint instead. 167 | # Only a problem when called with volume names instead of an fs filter. 168 | vols_by_fs = defaultdict(list) 169 | 170 | if args.command == 'reset': 171 | for vol in vols: 172 | if user_confirmation( 173 | 'Reset tracking status of {}?'.format(vol), False 174 | ): 175 | reset_vol(sess, vol) 176 | print('Reset of {} done'.format(vol)) 177 | 178 | if args.command in ('scan', 'dedup'): 179 | set_idle_priority() 180 | for vol in vols: 181 | if args.flush: 182 | tt.format('{elapsed} Flushing %s' % (vol,)) 183 | syncfs(vol.fd) 184 | tt.format(None) 185 | track_updated_files(sess, vol, tt) 186 | vols_by_fs[vol.fs].append(vol) 187 | 188 | if args.command == 'dedup': 189 | if args.groupby == 'vol': 190 | for vol in vols: 191 | tt.notify('Deduplicating volume %s' % vol) 192 | dedup_tracked(sess, [vol], tt, defrag=args.defrag) 193 | elif args.groupby == 'mpoint': 194 | for fs, volset in vols_by_fs.items(): 195 | tt.notify('Deduplicating filesystem %s' % fs) 196 | dedup_tracked(sess, volset, tt, defrag=args.defrag) 197 | else: 198 | assert False, args.groupby 199 | 200 | # For safety only. 201 | # The methods we call from the tracking module are expected to commit. 202 | sess.commit() 203 | 204 | 205 | def cmd_generation(args): 206 | volume_fd = os.open(args.volume, os.O_DIRECTORY) 207 | if args.flush: 208 | syncfs(volume_fd) 209 | generation = get_root_generation(volume_fd) 210 | print('%d' % generation) 211 | 212 | 213 | def user_confirmation(message, default): 214 | # default='n' would be an easy mistake to make 215 | assert default is bool(default) 216 | 217 | yes_values = 'y yes'.split() 218 | no_values = 'n no'.split() 219 | if default: 220 | choices = 'Y/n' 221 | yes_values.append('') 222 | else: 223 | choices = 'y/N' 224 | no_values.append('') 225 | 226 | while True: 227 | try: 228 | choice = input("%s (%s) " % (message, choices)).lower().strip() 229 | except EOFError: 230 | # non-interactive 231 | choice = '' 232 | if choice in yes_values: 233 | return True 234 | elif choice in no_values: 235 | return False 236 | 237 | 238 | def cmd_forget_fs(args): 239 | sess = get_session(args) 240 | whole_fs = WholeFS(sess) 241 | filesystems = [ 242 | whole_fs.get_fs_existing(UUID(hex=uuid)) for uuid in args.uuid] 243 | for fs in filesystems: 244 | if not user_confirmation('Wipe all data about fs %s?' % fs, False): 245 | continue 246 | for vol in fs._impl.volumes: 247 | # A lot of things will cascade 248 | sess.delete(vol) 249 | sess.delete(fs._impl) 250 | sess.commit() 251 | print('Wiped all data about %s' % fs) 252 | 253 | 254 | def cmd_size_lookup(args): 255 | sess = get_session(args) 256 | whole_fs = WholeFS(sess) 257 | if args.zero_terminated: 258 | end ='\0' 259 | else: 260 | end = '\n' 261 | for vol, rp, inode in annotated_inodes_by_size(whole_fs, args.size): 262 | print(vol.describe_path(rp), end=end) 263 | 264 | # We've deleted some stale inodes 265 | sess.commit() 266 | 267 | 268 | def cmd_shell(args): 269 | sess = get_session(args) 270 | whole_fs = WholeFS(sess) 271 | from . import model 272 | try: 273 | from IPython import embed 274 | except ImportError: 275 | sys.stderr.write( 276 | 'Please install bedup[interactive] for this feature\n') 277 | return 1 278 | with warnings.catch_warnings(): 279 | warnings.simplefilter('default') 280 | warnings.filterwarnings('ignore', module='IPython') 281 | embed() 282 | 283 | 284 | def cmd_fake_updates(args): 285 | sess = get_session(args) 286 | faked = fake_updates(sess, args.max_events) 287 | sess.commit() 288 | print('Faked about %d commonality clusters' % faked) 289 | 290 | 291 | def sql_flags(parser): 292 | parser.add_argument( 293 | '--db-path', dest='db_path', 294 | help='Override the location of the sqlite database') 295 | parser.add_argument( 296 | '--verbose-sql', action='store_true', dest='verbose_sql', 297 | help='Print SQL statements being executed') 298 | 299 | 300 | def vol_flags(parser): 301 | parser.add_argument( 302 | 'filter', nargs='*', 303 | help='List filesystem uuids, devices, or volume mountpoints to ' 304 | 'select which volumes are included. ' 305 | 'Prefix a volume mountpoint with vol: if you do not want ' 306 | 'subvolumes to be included.') 307 | sql_flags(parser) 308 | parser.add_argument( 309 | '--size-cutoff', type=int, dest='size_cutoff', 310 | help='Change the minimum size (in bytes) of tracked files ' 311 | 'for the listed volumes. ' 312 | 'Lowering the cutoff will trigger a partial rescan of older files.') 313 | parser.add_argument( 314 | '--no-crossvol', action='store_const', 315 | const='vol', default='mpoint', dest='groupby', 316 | help='This option disables cross-volume deduplication. ' 317 | 'This may be useful with pre-3.6 kernels.') 318 | 319 | 320 | def scan_flags(parser): 321 | vol_flags(parser) 322 | parser.add_argument( 323 | '--flush', action='store_true', dest='flush', 324 | help='Flush outstanding data using syncfs before scanning volumes') 325 | 326 | 327 | def is_in_path(cmd): 328 | # See shutil.which in Python 3.3 329 | return any( 330 | os.path.exists(el + '/' + cmd) for el in os.environ['PATH'].split(':')) 331 | 332 | 333 | def main(argv): 334 | progname = 'bedup' if is_in_path('bedup') else 'python3 -m bedup' 335 | io_enc = codecs.lookup(locale.getpreferredencoding()).name 336 | if io_enc == 'ascii': 337 | print( 338 | 'bedup will abort because Python was configured to use ASCII ' 339 | 'for console I/O.\nSee https://git.io/vnzk6 which ' 340 | 'explains how to use a UTF-8 locale.', file=sys.stderr) 341 | return 1 342 | parser = argparse.ArgumentParser(prog=progname) 343 | parser.add_argument( 344 | '--debug', action='store_true', help=argparse.SUPPRESS) 345 | commands = parser.add_subparsers(dest='command', metavar='command') 346 | 347 | sp_scan_vol = commands.add_parser( 348 | 'scan', help='Scan', description=""" 349 | Scans volumes to keep track of potentially duplicated files.""") 350 | sp_scan_vol.set_defaults(action=vol_cmd) 351 | scan_flags(sp_scan_vol) 352 | 353 | # In Python 3.2+ we can add aliases here. 354 | # Hidden aliases doesn't seem supported though. 355 | sp_dedup_vol = commands.add_parser( 356 | 'dedup', help='Scan and deduplicate', description=""" 357 | Runs scan, then deduplicates identical files.""") 358 | sp_dedup_vol.set_defaults(action=vol_cmd) 359 | scan_flags(sp_dedup_vol) 360 | sp_dedup_vol.add_argument( 361 | '--defrag', action='store_true', 362 | help='Defragment files that are going to be deduplicated') 363 | 364 | # An alias so as not to break btrfs-time-machine. 365 | # help='' is unset, which should make it (mostly) invisible. 366 | sp_dedup_vol_compat = commands.add_parser( 367 | 'dedup-vol', description=""" 368 | A deprecated alias for the 'dedup' command.""") 369 | sp_dedup_vol_compat.set_defaults(action=vol_cmd) 370 | scan_flags(sp_dedup_vol_compat) 371 | 372 | sp_reset_vol = commands.add_parser( 373 | 'reset', help='Reset tracking metadata', description=""" 374 | Reset tracking data for the listed volumes. Mostly useful for testing.""") 375 | sp_reset_vol.set_defaults(action=vol_cmd) 376 | vol_flags(sp_reset_vol) 377 | 378 | sp_show_vols = commands.add_parser( 379 | 'show', help='Show metadata overview', description=""" 380 | Shows filesystems and volumes with their tracking status.""") 381 | sp_show_vols.set_defaults(action=cmd_show_vols) 382 | sp_show_vols.add_argument('fsuuid_or_device', nargs='?') 383 | sp_show_vols.add_argument( 384 | '--show-deleted', dest='show_deleted', action='store_true', 385 | help='Show volumes that have been deleted') 386 | sql_flags(sp_show_vols) 387 | 388 | sp_find_new = commands.add_parser( 389 | 'find-new', help='List changed files', description=""" 390 | lists changes to volume since generation 391 | 392 | This is a reimplementation of btrfs find-new, 393 | modified to include directories as well.""") 394 | sp_find_new.set_defaults(action=cmd_find_new) 395 | sp_find_new.add_argument( 396 | '-0|--zero-terminated', dest='zero_terminated', action='store_true', 397 | help='Use a NUL character as the line separator') 398 | sp_find_new.add_argument( 399 | '--terse', dest='terse', action='store_true', help='Print names only') 400 | sp_find_new.add_argument('volume', help='Volume to search') 401 | sp_find_new.add_argument( 402 | 'generation', type=int, nargs='?', default=0, 403 | help='Only show items modified at generation or a newer transaction') 404 | 405 | sp_forget_fs = commands.add_parser( 406 | 'forget-fs', help='Wipe all metadata', description=""" 407 | Wipe all metadata for the listed filesystems. 408 | Useful if the filesystems don't exist anymore.""") 409 | sp_forget_fs.set_defaults(action=cmd_forget_fs) 410 | sp_forget_fs.add_argument('uuid', nargs='+', help='Btrfs filesystem uuids') 411 | sql_flags(sp_forget_fs) 412 | 413 | sp_dedup_files = commands.add_parser( 414 | 'dedup-files', help='Deduplicate listed', description=""" 415 | Freezes listed files, checks them for being identical, 416 | and projects the extents of the first file onto the other files. 417 | 418 | The effects are visible with filefrag -v (apt:e2fsprogs), 419 | which displays the extent map of files. 420 | """.strip()) 421 | sp_dedup_files.set_defaults(action=cmd_dedup_files) 422 | sp_dedup_files.add_argument('source', metavar='SRC', help='Source file') 423 | sp_dedup_files.add_argument( 424 | 'dests', metavar='DEST', nargs='+', help='Dest files') 425 | # Don't forget to also set new options in the dedup-vol test in vol_cmd 426 | sp_dedup_files.add_argument( 427 | '--defrag', action='store_true', 428 | help='Defragment the source file first') 429 | 430 | sp_generation = commands.add_parser( 431 | 'generation', help='Display volume generation', description=""" 432 | Display the btrfs generation of VOLUME.""") 433 | sp_generation.set_defaults(action=cmd_generation) 434 | sp_generation.add_argument('volume', help='Btrfs volume') 435 | sp_generation.add_argument( 436 | '--flush', action='store_true', dest='flush', 437 | help='Flush outstanding data using syncfs before lookup') 438 | 439 | sp_size_lookup = commands.add_parser( 440 | 'size-lookup', help='Look up inodes by size', description=""" 441 | List tracked inodes with a given size.""") 442 | sp_size_lookup.set_defaults(action=cmd_size_lookup) 443 | sp_size_lookup.add_argument('size', type=int) 444 | sp_size_lookup.add_argument( 445 | '-0|--zero-terminated', dest='zero_terminated', action='store_true', 446 | help='Use a NUL character as the line separator') 447 | sql_flags(sp_size_lookup) 448 | 449 | sp_shell = commands.add_parser( 450 | 'shell', description=""" 451 | Run an interactive shell (useful for prototyping).""") 452 | sp_shell.set_defaults(action=cmd_shell) 453 | sql_flags(sp_shell) 454 | 455 | sp_fake_updates = commands.add_parser( 456 | 'fake-updates', description=""" 457 | Fake inode updates from the latest dedup events (useful for benchmarking).""") 458 | sp_fake_updates.set_defaults(action=cmd_fake_updates) 459 | sp_fake_updates.add_argument('max_events', type=int) 460 | sql_flags(sp_fake_updates) 461 | 462 | # Give help when no subcommand is given 463 | if not argv[1:]: 464 | parser.print_help() 465 | return 466 | 467 | args = parser.parse_args(argv[1:]) 468 | 469 | if args.debug: 470 | try: 471 | from ipdb import launch_ipdb_on_exception 472 | except ImportError: 473 | sys.stderr.write( 474 | 'Please install bedup[interactive] for this feature\n') 475 | return 1 476 | with launch_ipdb_on_exception(): 477 | # Handle all warnings as errors. 478 | # Overrides the default filter that ignores deprecations 479 | # and prints the rest. 480 | warnings.simplefilter('error') 481 | warnings.filterwarnings('ignore', module='IPython\..*') 482 | warnings.filterwarnings('ignore', module='alembic\..*') 483 | return args.action(args) 484 | else: 485 | try: 486 | return args.action(args) 487 | except IOError as err: 488 | if err.errno == errno.EPERM: 489 | sys.stderr.write( 490 | "You need to run this command as root.\n") 491 | return 1 492 | raise 493 | 494 | 495 | def script_main(): 496 | # site.py takes about 1s before main gets called 497 | sys.exit(main(sys.argv)) 498 | 499 | 500 | if __name__ == '__main__': 501 | script_main() 502 | 503 | -------------------------------------------------------------------------------- /bedup/compat.py: -------------------------------------------------------------------------------- 1 | # bedup - Btrfs deduplication 2 | # Copyright (C) 2015 Gabriel de Perthuis 3 | # 4 | # This file is part of bedup. 5 | # 6 | # bedup is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 2 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # bedup is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with bedup. If not, see . 18 | 19 | 20 | # CFFI 0.4 reimplements Python 2 buffers on Python 3 21 | def buffer_to_bytes(buf): 22 | return buf[:] 23 | 24 | -------------------------------------------------------------------------------- /bedup/datetime.py: -------------------------------------------------------------------------------- 1 | # bedup - Btrfs deduplication 2 | # Copyright (C) 2012 Gabriel de Perthuis 3 | # 4 | # This file is part of bedup. 5 | # 6 | # bedup is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 2 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # bedup is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with bedup. If not, see . 18 | 19 | from __future__ import absolute_import 20 | import datetime 21 | 22 | ZERO = datetime.timedelta(0) 23 | 24 | 25 | class Utc(datetime.tzinfo): 26 | def utcoffset(self, dt): 27 | return ZERO 28 | 29 | def tzname(self, dt): 30 | return 'UTC' 31 | 32 | def dst(self, dt): 33 | return ZERO 34 | 35 | UTC = Utc() 36 | 37 | 38 | def system_now(): 39 | # datetime.utcnow is broken 40 | return datetime.datetime.now(tz=UTC) 41 | 42 | -------------------------------------------------------------------------------- /bedup/dedup.py: -------------------------------------------------------------------------------- 1 | # vim: set fileencoding=utf-8 sw=4 ts=4 et : 2 | 3 | # bedup - Btrfs deduplication 4 | # Copyright (C) 2015 Gabriel de Perthuis 5 | # 6 | # This file is part of bedup. 7 | # 8 | # bedup is free software: you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by 10 | # the Free Software Foundation, either version 2 of the License, or 11 | # (at your option) any later version. 12 | # 13 | # bedup is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License 19 | # along with bedup. If not, see . 20 | 21 | import collections 22 | import errno 23 | import glob 24 | import os 25 | import re 26 | import stat 27 | 28 | from .platform.btrfs import clone_data, defragment as btrfs_defragment 29 | from .platform.chattr import editflags, FS_IMMUTABLE_FL 30 | from .platform.futimens import fstat_ns, futimens 31 | 32 | 33 | BUFSIZE = 8192 34 | 35 | 36 | class FilesDifferError(ValueError): 37 | pass 38 | 39 | 40 | class FilesInUseError(RuntimeError): 41 | def describe(self, ofile): 42 | for (fi, users) in self.args[1].items(): 43 | ofile.write('File %s is in use\n' % fi) 44 | for use_info in users: 45 | ofile.write(' used as %r\n' % (use_info,)) 46 | 47 | 48 | ProcUseInfo = collections.namedtuple( 49 | 'ProcUseInfo', 'proc_path is_readable is_writable') 50 | 51 | 52 | def proc_use_info(proc_path): 53 | try: 54 | mode = os.lstat(proc_path).st_mode 55 | except OSError as e: 56 | if e.errno == errno.ENOENT: 57 | return 58 | raise 59 | else: 60 | return ProcUseInfo( 61 | proc_path=proc_path, 62 | is_readable=bool(mode & stat.S_IRUSR), 63 | is_writable=bool(mode & stat.S_IWUSR)) 64 | 65 | 66 | def cmp_fds(fd1, fd2): 67 | # Python 3 can take closefd=False instead of a duplicated fd. 68 | fi1 = os.fdopen(os.dup(fd1), 'rb') 69 | fi2 = os.fdopen(os.dup(fd2), 'rb') 70 | return cmp_files(fi1, fi2) 71 | 72 | 73 | def cmp_files(fi1, fi2): 74 | fi1.seek(0) 75 | fi2.seek(0) 76 | while True: 77 | b1 = fi1.read(BUFSIZE) 78 | b2 = fi2.read(BUFSIZE) 79 | if b1 != b2: 80 | return False 81 | if not b1: 82 | return True 83 | 84 | 85 | def dedup_same(source, dests, defragment=False): 86 | if defragment: 87 | source_fd = os.open(source, os.O_RDWR) 88 | else: 89 | source_fd = os.open(source, os.O_RDONLY) 90 | dest_fds = [os.open(dname, os.O_RDWR) for dname in dests] 91 | fds = [source_fd] + dest_fds 92 | fd_names = dict(zip(fds, [source] + dests)) 93 | 94 | with ImmutableFDs(fds) as immutability: 95 | if immutability.fds_in_write_use: 96 | raise FilesInUseError( 97 | 'Some of the files to deduplicate ' 98 | 'are open for writing elsewhere', 99 | dict( 100 | (fd_names[fd], tuple(immutability.write_use_info(fd))) 101 | for fd in immutability.fds_in_write_use)) 102 | 103 | if defragment: 104 | btrfs_defragment(source_fd) 105 | for fd in dest_fds: 106 | if not cmp_fds(source_fd, fd): 107 | raise FilesDifferError(fd_names[source_fd], fd_names[fd]) 108 | clone_data(dest=fd, src=source_fd, check_first=not defragment) 109 | 110 | 111 | PROC_PATH_RE = re.compile(r'^/proc/(\d+)/fd/(\d+)$') 112 | 113 | 114 | def find_inodes_in_write_use(fds): 115 | for (fd, use_info) in find_inodes_in_use(fds): 116 | if use_info.is_writable: 117 | yield (fd, use_info) 118 | 119 | 120 | def find_inodes_in_use(fds): 121 | """ 122 | Find which of these inodes are in use, and give their open modes. 123 | 124 | Does not count the passed fds as an use of the inode they point to, 125 | but if the current process has the same inodes open with different 126 | file descriptors these will be listed. 127 | 128 | Looks at /proc/*/fd and /proc/*/map_files (Linux 3.3). 129 | Conceivably there are other uses we're missing, to be foolproof 130 | will require support in btrfs itself; a share-same-range ioctl 131 | would work well. 132 | """ 133 | 134 | self_pid = os.getpid() 135 | id_fd_assoc = collections.defaultdict(list) 136 | 137 | for fd in fds: 138 | st = os.fstat(fd) 139 | id_fd_assoc[(st.st_dev, st.st_ino)].append(fd) 140 | 141 | def st_id_candidates(it): 142 | # map proc paths to stat identifiers (devno and ino) 143 | for proc_path in it: 144 | try: 145 | st = os.stat(proc_path) 146 | except OSError as e: 147 | # glob opens directories during matching, 148 | # and other processes might close their fds in the meantime. 149 | # This isn't a problem for the immutable-locked use case. 150 | # ESTALE could happen with NFS or Docker 151 | if e.errno in (errno.ENOENT, errno.ESTALE): 152 | continue 153 | raise 154 | 155 | st_id = (st.st_dev, st.st_ino) 156 | if st_id not in id_fd_assoc: 157 | continue 158 | 159 | yield proc_path, st_id 160 | 161 | for proc_path, st_id in st_id_candidates(glob.glob('/proc/[1-9]*/fd/*')): 162 | other_pid, other_fd = map( 163 | int, PROC_PATH_RE.match(proc_path).groups()) 164 | original_fds = id_fd_assoc[st_id] 165 | if other_pid == self_pid: 166 | if other_fd in original_fds: 167 | continue 168 | 169 | use_info = proc_use_info(proc_path) 170 | if not use_info: 171 | continue 172 | 173 | for fd in original_fds: 174 | yield (fd, use_info) 175 | 176 | # Requires Linux 3.3 177 | for proc_path, st_id in st_id_candidates( 178 | glob.glob('/proc/[1-9]*/map_files/*') 179 | ): 180 | use_info = proc_use_info(proc_path) 181 | if not use_info: 182 | continue 183 | 184 | original_fds = id_fd_assoc[st_id] 185 | for fd in original_fds: 186 | yield (fd, use_info) 187 | 188 | 189 | RestoreInfo = collections.namedtuple( 190 | 'RestoreInfo', ('fd', 'immutable', 'atime', 'mtime')) 191 | 192 | 193 | class ImmutableFDs(object): 194 | """A context manager to mark a set of fds immutable. 195 | 196 | Actually works at the inode level, fds are just to make sure 197 | inodes can be referenced unambiguously. 198 | 199 | This also restores atime and mtime when leaving. 200 | """ 201 | 202 | # Alternatives: mandatory locking. 203 | # Needs -o remount,mand + a metadata update + the same scan 204 | # for outstanding fds (although the race window is smaller). 205 | # The only real advantage is portability to more filesystems. 206 | # Since mandatory locking is a mount option, chances are 207 | # it is scoped to a mount namespace, which would complicate 208 | # attempts to enforce it with a remount. 209 | 210 | def __init__(self, fds): 211 | self.__fds = fds 212 | self.__revert_list = [] 213 | self.__in_use = None 214 | self.__writable_fds = None 215 | 216 | def __enter__(self): 217 | for fd in self.__fds: 218 | # Prevents anyone from creating write-mode file descriptors, 219 | # but the ones that already exist remain valid. 220 | was_immutable = editflags(fd, add_flags=FS_IMMUTABLE_FL) 221 | # editflags doesn't change atime or mtime; 222 | # measure after locking then. 223 | atime, mtime = fstat_ns(fd) 224 | self.__revert_list.append( 225 | RestoreInfo(fd, was_immutable, atime, mtime)) 226 | return self 227 | 228 | def __exit__(self, exc_type, exc_value, traceback): 229 | for (fd, immutable, atime, mtime) in reversed(self.__revert_list): 230 | if not immutable: 231 | editflags(fd, remove_flags=FS_IMMUTABLE_FL) 232 | # XXX Someone might modify the file between editflags 233 | # and futimens; oh well. 234 | # Needs kernel changes either way, either a dedup ioctl 235 | # or mandatory locking that doesn't touch file metadata. 236 | futimens(fd, (atime, mtime)) 237 | 238 | def __require_use_info(self): 239 | # We only track write use, other uses can appear after the /proc scan 240 | if self.__in_use is None: 241 | self.__in_use = collections.defaultdict(list) 242 | for (fd, use_info) in find_inodes_in_write_use(self.__fds): 243 | self.__in_use[fd].append(use_info) 244 | self.__writable_fds = frozenset(self.__in_use.keys()) 245 | 246 | def write_use_info(self, fd): 247 | self.__require_use_info() 248 | # A quick check to prevent unnecessary list instanciation 249 | if fd in self.__in_use: 250 | return tuple(self.__in_use[fd]) 251 | else: 252 | return tuple() 253 | 254 | @property 255 | def fds_in_write_use(self): 256 | self.__require_use_info() 257 | return self.__writable_fds 258 | 259 | -------------------------------------------------------------------------------- /bedup/filesystem.py: -------------------------------------------------------------------------------- 1 | # vim: set fileencoding=utf-8 sw=4 ts=4 et : 2 | # bedup - Btrfs deduplication 3 | # Copyright (C) 2015 Gabriel de Perthuis 4 | # 5 | # This file is part of bedup. 6 | # 7 | # bedup is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # bedup is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with bedup. If not, see . 19 | 20 | import errno 21 | import os 22 | import re 23 | import subprocess 24 | import sys 25 | import tempfile 26 | 27 | from collections import namedtuple, defaultdict, OrderedDict, Counter 28 | from uuid import UUID 29 | 30 | from sqlalchemy.util import memoized_property 31 | from sqlalchemy.orm.exc import NoResultFound 32 | 33 | from .platform.btrfs import ( 34 | get_fsid, get_root_id, lookup_ino_path_one, 35 | read_root_tree, BTRFS_FIRST_FREE_OBJECTID) 36 | from .platform.openat import openat 37 | from .platform.unshare import unshare, CLONE_NEWNS 38 | 39 | from .model import ( 40 | BtrfsFilesystem, Volume, get_or_create, VolumePathHistory) 41 | 42 | 43 | # 32MiB, initial scan takes about 12', might gain 15837689948, 44 | # sqlite takes 256k 45 | DEFAULT_SIZE_CUTOFF = 32 * 1024 ** 2 46 | # about 12' again, might gain 25807974687 47 | DEFAULT_SIZE_CUTOFF = 16 * 1024 ** 2 48 | # 13'40" (36' with a backup job running in parallel), might gain 26929240347, 49 | # sqlite takes 758k 50 | DEFAULT_SIZE_CUTOFF = 8 * 1024 ** 2 51 | 52 | 53 | DeviceInfo = namedtuple('DeviceInfo', 'label devices') 54 | MountInfo = namedtuple('MountInfo', 'internal_path mpoint readonly private') 55 | 56 | # A description, which may or may not be a path in the global filesystem 57 | VolDesc = namedtuple('VolDesc', 'description is_fs_path') 58 | 59 | 60 | class NotMounted(RuntimeError): 61 | pass 62 | 63 | 64 | class NotPlugged(RuntimeError): 65 | pass 66 | 67 | 68 | class BadDevice(RuntimeError): 69 | pass 70 | 71 | 72 | class NotAVolume(RuntimeError): 73 | # Not a BtrFS volume 74 | # For example: not btrfs, or normal directory within a btrfs fs 75 | pass 76 | 77 | 78 | def path_isprefix(prefix, path): 79 | # prefix and path must be absolute and normalised, 80 | # including symlink resolution. 81 | return prefix == '/' or path == prefix or path.startswith(prefix + '/') 82 | 83 | 84 | class BtrfsFilesystem2(object): 85 | """Augments the db-persisted BtrfsFilesystem with some live metadata. 86 | """ 87 | 88 | def __init__(self, whole_fs, impl, uuid): 89 | self._whole_fs = whole_fs 90 | self._impl = impl 91 | self._uuid = uuid 92 | self._root_info = None 93 | self._mpoints = None 94 | self._best_desc = {} 95 | self._priv_mpoint = None 96 | 97 | self._minfos = None 98 | 99 | try: 100 | # XXX Not in the db schema yet 101 | self._impl.label = self.label 102 | except NotPlugged: 103 | # XXX No point creating a live object in this case 104 | pass 105 | 106 | @property 107 | def impl(self): 108 | return self._impl 109 | 110 | @property 111 | def uuid(self): 112 | return self._uuid 113 | 114 | def iter_open_vols(self): 115 | for vol in self._whole_fs.iter_open_vols(): 116 | if vol._fs == self: 117 | yield vol 118 | 119 | def clean_up_mpoints(self): 120 | if self._priv_mpoint is None: 121 | return 122 | for vol in self.iter_open_vols(): 123 | if vol._fd is not None: 124 | vol.close() 125 | subprocess.check_call('umount -n -- '.split() + [self._priv_mpoint]) 126 | os.rmdir(self._priv_mpoint) 127 | self._priv_mpoint = None 128 | 129 | def __str__(self): 130 | return self.desc 131 | 132 | @memoized_property 133 | def desc(self): 134 | try: 135 | if self.label and self._whole_fs._label_occurs[self.label] == 1: 136 | return '<%s>' % self.label 137 | except NotPlugged: 138 | # XXX Keep the label in the db? 139 | pass 140 | return '{%s}' % self.uuid 141 | 142 | def best_desc(self, root_id): 143 | if root_id not in self._best_desc: 144 | intpath = self.root_info[root_id].path 145 | candidate_mis = [ 146 | mi for mi in self.minfos 147 | if not mi.private and path_isprefix(mi.internal_path, intpath)] 148 | if candidate_mis: 149 | mi = max( 150 | candidate_mis, key=lambda mi: len(mi.internal_path)) 151 | base = mi.mpoint 152 | intbase = mi.internal_path 153 | is_fs_path = True 154 | else: 155 | base = self.desc 156 | intbase = '/' 157 | is_fs_path = False 158 | self._best_desc[root_id] = VolDesc( 159 | os.path.normpath( 160 | os.path.join(base, os.path.relpath(intpath, intbase))), 161 | is_fs_path) 162 | return self._best_desc[root_id] 163 | 164 | def ensure_private_mpoint(self): 165 | # Create a private mountpoint with: 166 | # noatime,noexec,nodev 167 | # subvol=/ 168 | if self._priv_mpoint is not None: 169 | return 170 | 171 | self.require_plugged() 172 | self._whole_fs.ensure_unshared() 173 | pm = tempfile.mkdtemp(suffix='.privmnt') 174 | subprocess.check_call( 175 | 'mount -t btrfs -o subvol=/,noatime,noexec,nodev -nU'.split() 176 | + [str(self.uuid), pm]) 177 | self._priv_mpoint = pm 178 | self.add_minfo(MountInfo( 179 | internal_path='/', mpoint=pm, readonly=False, private=True)) 180 | 181 | def load_vol_by_root_id(self, root_id): 182 | self.ensure_private_mpoint() 183 | ri = self.root_info[root_id] 184 | return self._whole_fs._get_vol_by_path( 185 | self._priv_mpoint + ri.path, desc=None) 186 | 187 | @memoized_property 188 | def root_info(self): 189 | if not self.minfos: 190 | raise NotMounted 191 | fd = os.open(self.minfos[0].mpoint, os.O_DIRECTORY) 192 | try: 193 | return read_root_tree(fd) 194 | finally: 195 | os.close(fd) 196 | 197 | @memoized_property 198 | def device_info(self): 199 | try: 200 | return self._whole_fs.device_info[self.uuid] 201 | except KeyError: 202 | raise NotPlugged(self) 203 | 204 | def require_plugged(self): 205 | if self.uuid not in self._whole_fs.device_info: 206 | raise NotPlugged(self) 207 | 208 | @memoized_property 209 | def label(self): 210 | return self.device_info.label 211 | 212 | @property 213 | def minfos(self): 214 | # Not memoised, some may be added later 215 | if self._minfos is None: 216 | mps = [] 217 | try: 218 | for dev in self.device_info.devices: 219 | dev_canonical = os.path.realpath(dev) 220 | if dev_canonical in self._whole_fs.mpoints_by_dev: 221 | mps.extend(self._whole_fs.mpoints_by_dev[dev_canonical]) 222 | except NotPlugged: 223 | pass 224 | self._minfos = mps 225 | return tuple(self._minfos) 226 | 227 | def add_minfo(self, mi): 228 | if mi not in self.minfos: 229 | self._minfos.append(mi) 230 | 231 | def _iter_subvols(self, start_root_ids): 232 | child_id_map = defaultdict(list) 233 | 234 | for root_id, ri in self.root_info.items(): 235 | if ri.parent_root_id is not None: 236 | child_id_map[ri.parent_root_id].append(root_id) 237 | 238 | def _iter_children(root_id, top_level): 239 | yield (root_id, self.root_info[root_id], top_level) 240 | for child_id in child_id_map[root_id]: 241 | for item in _iter_children(child_id, False): 242 | yield item 243 | 244 | for root_id in start_root_ids: 245 | for item in _iter_children(root_id, True): 246 | yield item 247 | 248 | def _load_visible_vols(self, start_paths, nest_desc): 249 | # Use dicts, there may be repetitions under multiple mountpoints 250 | loaded = OrderedDict() 251 | 252 | start_vols = OrderedDict( 253 | (vol.root_id, vol) 254 | for vol in ( 255 | self._whole_fs._get_vol_by_path(start_fspath, desc=None) 256 | for start_fspath in start_paths)) 257 | 258 | for (root_id, ri, top_level) in self._iter_subvols(start_vols): 259 | if top_level: 260 | start_vol = start_vols[root_id] 261 | if start_vol not in loaded: 262 | loaded[start_vol] = True 263 | start_desc = start_vol.desc 264 | start_intpath = ri.path 265 | start_fd = start_vol.fd 266 | # relpath is more predictable with absolute paths; 267 | # otherwise it relies on getcwd (via abspath) 268 | assert os.path.isabs(start_intpath) 269 | else: 270 | relpath = os.path.relpath(ri.path, start_intpath) 271 | if nest_desc: 272 | desc = VolDesc( 273 | os.path.join(start_desc.description, relpath), 274 | start_desc.is_fs_path) 275 | else: 276 | desc = None 277 | vol = self._whole_fs._get_vol_by_relpath( 278 | start_fd, relpath, desc=desc) 279 | if vol not in loaded: 280 | loaded[vol] = True 281 | return loaded.keys(), start_vols.values() 282 | 283 | 284 | def impl_property(name): 285 | def getter(inst): 286 | return getattr(inst._impl, name) 287 | 288 | def setter(inst, val): 289 | setattr(inst._impl, name, val) 290 | 291 | return property(getter, setter) 292 | 293 | 294 | class Volume2(object): 295 | def __init__(self, whole_fs, fs, impl, desc, fd): 296 | self._whole_fs = whole_fs 297 | self._fs = fs 298 | self._impl = impl 299 | self._desc = desc 300 | self._fd = fd 301 | 302 | self.st_dev = os.fstat(self._fd).st_dev 303 | 304 | self._impl.live = self 305 | 306 | last_tracked_generation = impl_property('last_tracked_generation') 307 | last_tracked_size_cutoff = impl_property('last_tracked_size_cutoff') 308 | size_cutoff = impl_property('size_cutoff') 309 | 310 | def __str__(self): 311 | return self.desc.description 312 | 313 | @property 314 | def impl(self): 315 | return self._impl 316 | 317 | @property 318 | def root_info(self): 319 | return self._fs.root_info[self._impl.root_id] 320 | 321 | @property 322 | def root_id(self): 323 | return self._impl.root_id 324 | 325 | @property 326 | def desc(self): 327 | return self._desc 328 | 329 | @property 330 | def fd(self): 331 | return self._fd 332 | 333 | @property 334 | def fs(self): 335 | return self._fs 336 | 337 | @classmethod 338 | def vol_id_of_fd(cls, fd): 339 | try: 340 | return get_fsid(fd), get_root_id(fd) 341 | except IOError as err: 342 | if err.errno == errno.ENOTTY: 343 | raise NotAVolume(fd) 344 | raise 345 | 346 | def close(self): 347 | os.close(self._fd) 348 | self._fd = None 349 | 350 | def lookup_one_path(self, inode): 351 | return lookup_ino_path_one(self.fd, inode.ino) 352 | 353 | def describe_path(self, relpath): 354 | return os.path.join(self.desc.description, relpath) 355 | 356 | 357 | class WholeFS(object): 358 | """A singleton representing the local filesystem""" 359 | 360 | def __init__(self, sess, size_cutoff=None): 361 | # Public functions that rely on sess: 362 | # get_fs, iter_fs, load_all_writable_vols, load_vols, 363 | # Requiring root: 364 | # load_all_writable_vols, load_vols. 365 | self.sess = sess 366 | self._unshared = False 367 | self._size_cutoff = size_cutoff 368 | self._fs_map = {} 369 | # keyed on fs_uuid, vol.root_id 370 | self._vol_map = {} 371 | self._label_occurs = None 372 | 373 | def get_fs_existing(self, uuid): 374 | assert isinstance(uuid, UUID) 375 | if uuid not in self._fs_map: 376 | try: 377 | db_fs = self.sess.query( 378 | BtrfsFilesystem).filter_by(uuid=str(uuid)).one() 379 | except NoResultFound: 380 | raise KeyError(uuid) 381 | fs = BtrfsFilesystem2(self, db_fs, uuid) 382 | self._fs_map[uuid] = fs 383 | return self._fs_map[uuid] 384 | 385 | def get_fs(self, uuid): 386 | assert isinstance(uuid, UUID) 387 | if uuid not in self._fs_map: 388 | if uuid in self.device_info: 389 | db_fs, fs_created = get_or_create( 390 | self.sess, BtrfsFilesystem, uuid=str(uuid)) 391 | else: 392 | # Don't create a db object without a live fs backing it 393 | try: 394 | db_fs = self.sess.query( 395 | BtrfsFilesystem).filter_by(uuid=str(uuid)).one() 396 | except NoResultFound: 397 | raise NotPlugged(uuid) 398 | fs = BtrfsFilesystem2(self, db_fs, uuid) 399 | self._fs_map[uuid] = fs 400 | return self._fs_map[uuid] 401 | 402 | def iter_fs(self): 403 | seen_fs_ids = [] 404 | for (uuid, di) in self.device_info.items(): 405 | fs = self.get_fs(uuid) 406 | seen_fs_ids.append(fs._impl.id) 407 | yield fs, di 408 | 409 | extra_fs_query = self.sess.query(BtrfsFilesystem.uuid) 410 | if seen_fs_ids: 411 | # Conditional because we get a performance SAWarning otherwise 412 | extra_fs_query = extra_fs_query.filter( 413 | ~ BtrfsFilesystem.id.in_(seen_fs_ids)) 414 | for uuid, in extra_fs_query: 415 | yield self.get_fs(UUID(hex=uuid)), None 416 | 417 | def iter_open_vols(self): 418 | return iter(self._vol_map.values()) 419 | 420 | def _get_vol_by_path(self, volpath, desc): 421 | volpath = os.path.normpath(volpath) 422 | fd = os.open(volpath, os.O_DIRECTORY) 423 | return self._get_vol(fd, desc) 424 | 425 | def _get_vol_by_relpath(self, base_fd, relpath, desc): 426 | fd = openat(base_fd, relpath, os.O_DIRECTORY) 427 | return self._get_vol(fd, desc) 428 | 429 | def _get_vol(self, fd, desc): 430 | if not is_subvolume(fd): 431 | raise NotAVolume(fd, desc) 432 | vol_id = Volume2.vol_id_of_fd(fd) 433 | 434 | # If a volume was given multiple times on the command line, 435 | # keep the first name and fd for it. 436 | if vol_id in self._vol_map: 437 | os.close(fd) 438 | return self._vol_map[vol_id] 439 | 440 | fs_uuid, root_id = vol_id 441 | 442 | fs = self.get_fs(uuid=fs_uuid) 443 | db_vol, db_vol_created = get_or_create( 444 | self.sess, Volume, fs=fs._impl, root_id=root_id) 445 | 446 | if self._size_cutoff is not None: 447 | db_vol.size_cutoff = self._size_cutoff 448 | elif db_vol_created: 449 | db_vol.size_cutoff = DEFAULT_SIZE_CUTOFF 450 | 451 | if desc is None: 452 | desc = fs.best_desc(root_id) 453 | 454 | vol = Volume2(self, fs=fs, impl=db_vol, desc=desc, fd=fd) 455 | 456 | if desc.is_fs_path: 457 | path_history, ph_created = get_or_create( 458 | self.sess, VolumePathHistory, 459 | vol=db_vol, path=desc.description) 460 | 461 | self._vol_map[vol_id] = vol 462 | return vol 463 | 464 | @memoized_property 465 | def mpoints_by_dev(self): 466 | assert not self._unshared 467 | mbd = defaultdict(list) 468 | with open('/proc/self/mountinfo') as mounts: 469 | for line in mounts: 470 | items = line.split() 471 | idx = items.index('-') 472 | fs_type = items[idx + 1] 473 | opts1 = items[5].split(',') 474 | opts2 = items[idx + 3].split(',') 475 | readonly = 'ro' in opts1 + opts2 476 | if fs_type != 'btrfs': 477 | continue 478 | intpath = items[3] 479 | mpoint = items[4] 480 | dev = os.path.realpath(items[idx + 2]) 481 | mbd[dev].append(MountInfo(intpath, mpoint, readonly, False)) 482 | return dict(mbd) 483 | 484 | @memoized_property 485 | def device_info(self): 486 | di = {} 487 | lbls = Counter() 488 | cmd = 'blkid -s LABEL -s UUID -t TYPE=btrfs'.split() 489 | subp = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True) 490 | for line in subp.stdout: 491 | dev, label, uuid = BLKID_RE.match(line).groups() 492 | uuid = UUID(hex=uuid) 493 | if uuid in di: 494 | # btrfs raid 495 | assert di[uuid].label == label 496 | di[uuid].devices.append(dev) 497 | else: 498 | lbls[label] += 1 499 | di[uuid] = DeviceInfo(label, [dev]) 500 | rc = subp.wait() 501 | # 2 means there is no btrfs filesystem 502 | if rc not in (0, 2): 503 | raise subprocess.CalledProcessError(rc, cmd) 504 | self._label_occurs = dict(lbls) 505 | return di 506 | 507 | def ensure_unshared(self): 508 | if not self._unshared: 509 | # Make sure we read mountpoints before creating ours, 510 | # so that ours won't appear on the list. 511 | self.mpoints_by_dev 512 | unshare(CLONE_NEWNS) 513 | self._unshared = True 514 | 515 | def clean_up_mpoints(self): 516 | if not self._unshared: 517 | return 518 | for fs, di in self.iter_fs(): 519 | fs.clean_up_mpoints() 520 | 521 | def close(self): 522 | # For context managers 523 | self.clean_up_mpoints() 524 | 525 | def load_vols_for_device(self, devpath, tt): 526 | for uuid, di in self.device_info.items(): 527 | if any(os.path.samefile(dp, devpath) for dp in di.devices): 528 | fs = self.get_fs(uuid) 529 | return self.load_vols_for_fs(fs, tt) 530 | raise BadDevice('No Btrfs filesystem detected by blkid', devpath) 531 | 532 | def load_vols_for_fs(self, fs, tt): 533 | # Check that the filesystem is plugged 534 | fs.device_info 535 | 536 | loaded = [] 537 | fs.ensure_private_mpoint() 538 | lo, sta = fs._load_visible_vols([fs._priv_mpoint], nest_desc=False) 539 | assert self._vol_map 540 | frozen_skipped = 0 541 | for vol in lo: 542 | if vol.root_info.is_frozen: 543 | vol.close() 544 | frozen_skipped += 1 545 | else: 546 | loaded.append(vol) 547 | if frozen_skipped: 548 | tt.notify( 549 | 'Skipped %d frozen volumes in filesystem %s' % ( 550 | frozen_skipped, fs)) 551 | return loaded 552 | 553 | def load_all_writable_vols(self, tt): 554 | # All non-frozen volumes that are on a 555 | # filesystem that has a non-ro mountpoint. 556 | loaded = [] 557 | for (uuid, di) in self.device_info.items(): 558 | fs = self.get_fs(uuid) 559 | try: 560 | fs.root_info 561 | except NotMounted: 562 | tt.notify('Skipping filesystem %s, not mounted' % fs) 563 | continue 564 | if all(mi.readonly for mi in fs.minfos): 565 | tt.notify('Skipping filesystem %s, not mounted rw' % fs) 566 | continue 567 | loaded.extend(self.load_vols_for_fs(fs, tt)) 568 | return loaded 569 | 570 | def load_vols(self, volpaths, tt, recurse): 571 | # The volume at volpath, plus all its visible non-frozen descendants 572 | # XXX Some of these may fail if other filesystems 573 | # are mounted on top of them. 574 | loaded = OrderedDict() 575 | for volpath in volpaths: 576 | vol = self._get_vol_by_path(volpath, desc=VolDesc(volpath, True)) 577 | if recurse: 578 | if vol.root_info.path != '/': 579 | tt.notify( 580 | '%s isn\'t the root volume, ' 581 | 'use the filesystem uuid for maximum efficiency.' % vol) 582 | lo, sta = vol._fs._load_visible_vols([volpath], nest_desc=True) 583 | skipped = 0 584 | for vol in lo: 585 | if vol in loaded: 586 | continue 587 | if vol.root_info.is_frozen and vol not in sta: 588 | vol.close() 589 | skipped += 1 590 | else: 591 | loaded[vol] = True 592 | if skipped: 593 | tt.notify( 594 | 'Skipped %d frozen volumes in filesystem %s' % ( 595 | skipped, vol.fs)) 596 | else: 597 | if vol not in loaded: 598 | loaded[vol] = True 599 | return loaded.keys() 600 | 601 | 602 | BLKID_RE = re.compile( 603 | r'^(?P/dev/.*):' 604 | r'(?:\s+LABEL="(?P