├── blocks ├── __init__.py ├── maintboot.init ├── augeas │ ├── lvm.aug │ └── test_lvm.aug └── __main__.py ├── tests ├── Makefile ├── run-tests └── run-tests.inside ├── MANIFEST.in ├── requirements.txt ├── .gitignore ├── setup.py ├── .travis.yml ├── README.md └── COPYING /blocks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | 2 | check: 3 | ./run-tests 4 | 5 | .PHONY: check 6 | 7 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include COPYING 2 | include README.md 3 | 4 | include blocks/augeas/*.aug 5 | include blocks/maintboot.init 6 | 7 | -------------------------------------------------------------------------------- /tests/run-tests: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | rm -f test.disk 5 | # sparse 6 | truncate -s1G test.disk 7 | 8 | vido --mem=192M --kernel ./linux.uml --disk test.disk \ 9 | -- ./run-tests.inside 10 | 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -e git+https://github.com/g2p/blocks.git@master#egg=blocks 2 | -e git+https://github.com/g2p/python-augeas.git@v0.4.2a1#egg=python-augeas 3 | https://github.com/g2p/pyparted/archive/pyparted-3.10a1.tar.gz#egg=pyparted 4 | 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[oc] 2 | /MANIFEST 3 | /*.egg-info/ 4 | /build/ 5 | /dist/ 6 | /tests/linux.uml 7 | /tests/before* 8 | /tests/after* 9 | /tests/keyfile 10 | /tests/whispers 11 | /tests/test.disk 12 | /tests/.coverage 13 | *.py,cover 14 | -------------------------------------------------------------------------------- /blocks/maintboot.init: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -x 3 | mount -t devtmpfs devtmpfs /dev 4 | mount -t proc proc /proc 5 | mount -t sysfs sysfs /sys 6 | echo root:x:0:0:root:/root:/bin/sh > /etc/passwd 7 | export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin 8 | ldconfig 9 | blocks maintboot-impl 10 | echo s > /proc/sysrq-trigger 11 | echo b > /proc/sysrq-trigger 12 | # wait for system to reboot 13 | read foo 14 | 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.3 2 | 3 | from setuptools import setup 4 | 5 | setup( 6 | name='blocks', 7 | version='0.1.4', 8 | author='Gabriel de Perthuis', 9 | author_email='g2p.code+blocks@gmail.com', 10 | url='https://github.com/g2p/blocks', 11 | license='GNU GPL', 12 | keywords='bcache lvm storage partitioning ssd', 13 | description='Conversion tools for block devices', 14 | entry_points={ 15 | 'console_scripts': [ 16 | 'blocks = blocks.__main__:script_main']}, 17 | packages=[ 18 | 'blocks', 19 | ], 20 | include_package_data=True, 21 | # See requirements.txt for installable versions 22 | install_requires=[ 23 | 'maintboot', 24 | 'python-augeas >= 0.4.2a0', 25 | 'pyparted > 3.10a0'], 26 | classifiers=''' 27 | Programming Language :: Python :: 3 28 | Programming Language :: Python :: 3.3 29 | License :: OSI Approved :: GNU General Public License (GPL) 30 | Operating System :: POSIX :: Linux 31 | Intended Audience :: System Administrators 32 | Intended Audience :: End Users/Desktop 33 | Topic :: System :: Filesystems 34 | Topic :: Utilities 35 | Environment :: Console 36 | '''.strip().splitlines(), 37 | long_description=''' 38 | Conversion tools for block devices. 39 | 40 | Convert between raw partitions, logical volumes, and bcache 41 | devices witout moving data. 42 | 43 | See `github.com/g2p/blocks `_ 44 | for installation and usage instructions.''') 45 | 46 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | install: 3 | - lsb_release -a 4 | - sudo apt-get update 5 | - sudo apt-get install 6 | python3-minimal libparted-dev libaugeas0 7 | pkg-config gcc cryptsetup lvm2 liblzo2-dev 8 | nilfs-tools reiserfsprogs xfsprogs e2fsprogs btrfs-tools gdisk 9 | - sudo apt-get build-dep btrfs-tools 10 | - which python3 11 | - wget 12 | https://bitbucket.org/pypa/setuptools/raw/bootstrap/ez_setup.py 13 | https://raw.github.com/pypa/pip/master/contrib/get-pip.py 14 | https://mirrors.kernel.org/ubuntu/pool/main/a/augeas/libaugeas0_1.0.0-0ubuntu1_amd64.deb 15 | - curl --compressed https://raw.github.com/g2p/kernels/master/linux.uml -o tests/linux.uml 16 | - chmod +x tests/linux.uml 17 | - sudo dpkg -i libaugeas0*deb 18 | - export PATH=$PATH:$HOME/.local/bin:$PWD/btrfs-progs 19 | - export PIP_USE_MIRRORS=true PIP_DOWNLOAD_CACHE=~/.cache/pip 20 | - python3.3 ez_setup.py --user 21 | - easy_install-3.3 --user pip 22 | - python3.3 -m pip install --user 23 | -e. git+https://github.com/g2p/python-augeas.git@v0.4.2a1#egg=python-augeas 24 | https://github.com/g2p/pyparted/archive/pyparted-3.10a1.tar.gz#egg=pyparted 25 | git+https://github.com/g2p/vido.git#egg=vido 26 | coverage 27 | - git clone https://github.com/g2p/bcache-tools.git 28 | - git clone https://git.kernel.org/pub/scm/linux/kernel/git/mason/btrfs-progs.git 29 | - make -C btrfs-progs version.h btrfs-show-super 30 | - make -C bcache-tools 31 | - sudo make -C bcache-tools install 32 | - sudo mkdir -vp /etc/lvm/backup /etc/lvm/archive 33 | - sudo ln -Tsf /proc/mounts /etc/mtab 34 | 35 | script: 36 | - make -C tests 37 | 38 | -------------------------------------------------------------------------------- /blocks/augeas/lvm.aug: -------------------------------------------------------------------------------- 1 | (* 2 | Module: LVM 3 | Parses LVM metadata. 4 | 5 | Author: Gabriel de Perthuis 6 | 7 | About: License 8 | This file is licensed under the LGPL v2+. 9 | 10 | About: Configuration files 11 | This lens applies to files in /etc/lvm/backup and /etc/lvm/archive. 12 | 13 | About: Examples 14 | The file contains various examples and tests. 15 | *) 16 | 17 | module LVM = 18 | autoload xfm 19 | 20 | (* See lvm2/libdm/libdm-config.c for tokenisation; 21 | * libdm uses a blacklist but I prefer the safer whitelist approach. *) 22 | (* View: identifier 23 | * The left hand side of a definition *) 24 | let identifier = /[a-zA-Z0-9_.+-]+/ 25 | 26 | (* strings can contain backslash-escaped dquotes, but I don't know 27 | * how to get the message across to augeas *) 28 | let str = [label "str". 29 | Util.del_str "\"" . store /[^"]*/ . Util.del_str "\""] 30 | let int = [label "int". store Rx.integer] 31 | (* View: flat_literal 32 | * A literal without structure *) 33 | let flat_literal = int|str 34 | 35 | (* allow multiline and mixed int/str, used for raids and stripes *) 36 | (* View: list 37 | * A list containing flat literals *) 38 | let list = [ 39 | label "list" . counter "list" 40 | . del /\[[ \t\n]*/ "[" 41 | .([seq "list". flat_literal . del /,[ \t\n]*/ ", "]* 42 | . [seq "list". flat_literal . del /[ \t\n]*/ ""])? 43 | . Util.del_str "]"] 44 | 45 | (* View: val 46 | * Any value that appears on the right hand side of an assignment *) 47 | let val = flat_literal | list 48 | 49 | (* View: nondef 50 | * A line that doesn't contain a statement *) 51 | let nondef = 52 | Util.empty 53 | | Util.comment 54 | 55 | (* Build.block couldn't be reused, because of recursion and 56 | * a different philosophy of whitespace handling. *) 57 | (* View: def 58 | * An assignment, or a block containing definitions *) 59 | let rec def = [ 60 | Util.indent . key identifier . ( 61 | del /[ \t]*\{\n/ " {\n" 62 | .[label "dict".(nondef | def)*] 63 | . Util.indent . Util.del_str "}\n" 64 | |del /[ \t]*=[ \t]*/ " = " . val . Util.comment_or_eol)] 65 | 66 | (* View: lns 67 | * The main lens *) 68 | let lns = (nondef | def)* 69 | 70 | let filter = 71 | incl "/etc/lvm/archive/*.vg" 72 | . incl "/etc/lvm/backup/*" 73 | . Util.stdexcl 74 | 75 | let xfm = transform lns filter 76 | 77 | -------------------------------------------------------------------------------- /tests/run-tests.inside: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | # Precise compat 5 | mkdir -vp /run/lock /var/lock 6 | if test -e /etc/lvm/archive; then mount -t tmpfs tmpfs /etc/lvm/archive; fi 7 | if test -e /etc/lvm/backup; then mount -t tmpfs tmpfs /etc/lvm/backup; fi 8 | 9 | DISK=$VIDO_DISK0 10 | P1NAME=$(basename ${DISK}1) 11 | 12 | function coverage { python3.3 -m coverage "$@"; } 13 | coverage erase 14 | # shopt -s expand_aliases also works 15 | function blocks { 16 | coverage run --append --source=blocks -m blocks "$@" 17 | } 18 | 19 | ! blocks 20 | 21 | sgdisk --new=1:1M:+127M --new=2:129M:+127M \ 22 | --new=3:256M:+128M --new=4:384M:+128M \ 23 | --new=5:512M:+260M --new=6:772M:+124M \ 24 | -p $DISK 25 | 26 | 27 | mkfs.ext4 ${DISK}1 28 | blocks to-lvm ${DISK}1 --vg-name=VG 29 | ! blocks to-lvm ${DISK}1 30 | 31 | dd if=/dev/urandom of=keyfile bs=64k count=1 32 | 33 | # Doing this outside of a partition, otherwise the part-to-bcache 34 | # strategy would be used 35 | truncate -s1G whispers 36 | loop=$(losetup -f --show -- whispers) 37 | cryptsetup --batch-mode luksFormat -- "$loop" keyfile 38 | cryptsetup luksOpen --key-file=keyfile -- "$loop" whispers 39 | mkfs.ext4 /dev/mapper/whispers 40 | blocks to-bcache -- "$loop" 41 | 42 | cryptsetup --batch-mode luksFormat ${DISK}1 keyfile 43 | cryptsetup luksOpen --key-file=keyfile ${DISK}1 whispers 44 | mkfs.ext4 /dev/mapper/whispers 45 | blocks to-lvm ${DISK}1 46 | 47 | # go back to unencrypted, otherwise some tests would ask for passphrases 48 | mkfs.ext4 ${DISK}1 49 | # resize should require fsck if the fs has been mounted 50 | # XXX doesn't check in practice 51 | mkdir -p mnt 52 | mount ${DISK}1 mnt 53 | touch mnt/ts 54 | umount mnt 55 | blocks to-lvm ${DISK}1 --vg-name=VG 56 | ! blocks to-lvm ${DISK}1 57 | 58 | vgchange -ay 59 | blocks resize --resize-device /dev/mapper/VG-$P1NAME 64M 60 | # will round up 61 | blocks resize --resize-device /dev/mapper/VG-$P1NAME 95M 62 | blocks resize /dev/mapper/VG-$P1NAME 64M 63 | blocks resize /dev/mapper/VG-$P1NAME 96M 64 | vgchange -an 65 | 66 | mkfs.xfs ${DISK}6 67 | blocks resize --resize-device ${DISK}6 128M 68 | ! blocks resize --resize-device ${DISK}6 120M 69 | mkfs.reiserfs -q ${DISK}6 70 | blocks resize --resize-device ${DISK}6 132M 71 | mkfs.ext4 ${DISK}6 72 | blocks resize --resize-device ${DISK}6 128M 73 | mkswap ${DISK}6 74 | blocks resize --resize-device ${DISK}6 120M 75 | 76 | # Btrfs sb identification fails, but _BHRfS_M does appear 77 | # in the strace, what drugs is blkid on? 78 | # blkid won't recognise a btrfs filesystem smaller than 256m 79 | # (also btrfs can't shrink a 256m fs, though it can create a smaller one) 80 | mkfs.btrfs ${DISK}5 81 | #strace -s1024 -f blkid -p -c /dev/null ${DISK}5 82 | blocks to-lvm ${DISK}5 --join=VG 83 | ! blocks to-lvm ${DISK}5 84 | 85 | blocks to-bcache ${DISK}2 86 | ! blocks to-bcache ${DISK}2 87 | 88 | # Can't shrink an unrecognised partition 89 | ! blocks to-bcache ${DISK}4 90 | #mkfs.btrfs ${DISK}3 91 | # ReiserFS has too much interactivity 92 | #mkfs.reiserfs -q ${DISK}3 93 | mkfs.nilfs2 -B 1024 ${DISK}3 94 | blkid ${DISK}3 95 | 96 | blocks to-bcache ${DISK}4 97 | ! blocks to-bcache ${DISK}4 98 | 99 | vgchange -ay 100 | #ls -Artlh /dev/mapper 101 | 102 | vgcfgbackup --file before VG 103 | blkid /dev/mapper/VG-$P1NAME 104 | blocks --debug to-bcache /dev/mapper/VG-$P1NAME 105 | vgcfgbackup --file after VG 106 | bcache-super-show /dev/mapper/VG-$P1NAME 107 | ! blocks to-bcache /dev/mapper/VG-$P1NAME 108 | blocks --debug rotate /dev/mapper/VG-$P1NAME 109 | vgcfgbackup --file before.restored VG 110 | ! blocks rotate /dev/mapper/VG-$P1NAME 111 | lvchange --refresh VG/$P1NAME 112 | blkid /dev/mapper/VG-$P1NAME 113 | blocks --debug to-bcache /dev/mapper/VG-$P1NAME 114 | vgcfgbackup --file after.twice VG 115 | 116 | vgchange -an 117 | sgdisk --zap $DISK 118 | sfdisk -uS --quiet --Linux $DISK <<'ZZ' 119 | unit: sectors 120 | 121 | 2048,260096,L 122 | 262144,1835008,E 123 | 0,0,0 124 | 0,0,0 125 | 264192,260096,L 126 | 526336,1570816,L 127 | ZZ 128 | 129 | mkfs.ext4 ${DISK}1 130 | mkfs.ext4 ${DISK}5 131 | mkfs.ext4 ${DISK}6 132 | ! blocks to-bcache ${DISK}6 133 | ! blocks to-bcache ${DISK}5 134 | 135 | # Those must be identical except for seqno and various timestamps 136 | git --no-pager diff --no-index --color --patience before before.restored ||: 137 | git --no-pager diff --no-index --color --patience after after.twice ||: 138 | 139 | coverage-3.3 report -m 140 | coverage-3.3 annotate 141 | 142 | -------------------------------------------------------------------------------- /blocks/augeas/test_lvm.aug: -------------------------------------------------------------------------------- 1 | (* 2 | Module: Test_LVM 3 | Provides unit tests and examples for the lens. 4 | *) 5 | 6 | module Test_LVM = 7 | 8 | (* Variable: conf 9 | A full configuration file *) 10 | let conf = "# Generated by LVM2: date 11 | 12 | contents = \"Text Format Volume Group\" 13 | version = 1 14 | 15 | description = \"Created *after* executing 'eek'\" 16 | 17 | creation_host = \"eek\" # Linux eek 18 | creation_time = 6666666666 # eeeek 19 | 20 | VG1 { 21 | id = \"uuid-uuid-uuid-uuid\" 22 | seqno = 2 23 | status = [\"RESIZEABLE\", \"READ\", \"WRITE\"] 24 | extent_size = 8192 # 4 Megabytes 25 | max_lv = 0 26 | max_pv = 0 27 | 28 | physical_volumes { 29 | pv0 { 30 | id = \"uuid-uuid-uuid-uuid\" 31 | device = \"/dev/sda6\" # Hint only 32 | 33 | status = [\"ALLOCATABLE\"] 34 | pe_start = 123 35 | pe_count = 123456 # many Gigabytes 36 | } 37 | } 38 | 39 | logical_volumes { 40 | LogicalEek { 41 | id = \"uuid-uuid-uuid-uuid\" 42 | status = [\"READ\", \"WRITE\", \"VISIBLE\"] 43 | segment_count = 1 44 | 45 | segment1 { 46 | start_extent = 0 47 | extent_count = 123456 # beaucoup Gigabytes 48 | 49 | type = \"striped\" 50 | stripe_count = 1 # linear 51 | 52 | stripes = [ 53 | \"pv0\", 0 54 | ] 55 | } 56 | } 57 | } 58 | } 59 | " 60 | 61 | test LVM.int get "5" = { "int" = "5" } 62 | test LVM.str get "\"abc\"" = { "str" = "abc"} 63 | test LVM.lns get "\n" = {} 64 | test LVM.lns get "#foo\n" = { "#comment" = "foo"} 65 | 66 | test LVM.lns get "# Generated by LVM2: date 67 | 68 | contents = \"Text Format Volume Group\" 69 | version = 1 70 | 71 | description = \"Created *after* executing 'eek'\" 72 | 73 | creation_host = \"eek\" # Linux eek 74 | creation_time = 6666666666 # eeeek\n" = 75 | { "#comment" = "Generated by LVM2: date" } 76 | {} 77 | { "contents" 78 | { "str" = "Text Format Volume Group" } 79 | } 80 | { "version" 81 | { "int" = "1" } 82 | } 83 | {} 84 | { "description" 85 | { "str" = "Created *after* executing 'eek'" } 86 | } 87 | {} 88 | { "creation_host" 89 | { "str" = "eek" } 90 | { "#comment" = "Linux eek" } 91 | } 92 | { "creation_time" 93 | { "int" = "6666666666" } 94 | { "#comment" = "eeeek" } 95 | } 96 | 97 | (* Test: LVM.lns 98 | Test the full *) 99 | test LVM.lns get conf = 100 | { "#comment" = "Generated by LVM2: date" } 101 | {} 102 | { "contents" 103 | { "str" = "Text Format Volume Group" } 104 | } 105 | { "version" 106 | { "int" = "1" } 107 | } 108 | {} 109 | { "description" 110 | { "str" = "Created *after* executing 'eek'" } 111 | } 112 | {} 113 | { "creation_host" 114 | { "str" = "eek" } 115 | { "#comment" = "Linux eek" } 116 | } 117 | { "creation_time" 118 | { "int" = "6666666666" } 119 | { "#comment" = "eeeek" } 120 | } 121 | {} 122 | { "VG1" 123 | { "dict" 124 | { "id" 125 | { "str" = "uuid-uuid-uuid-uuid" } 126 | } 127 | { "seqno" 128 | { "int" = "2" } 129 | } 130 | { "status" 131 | { "list" 132 | { "1" 133 | { "str" = "RESIZEABLE" } 134 | } 135 | { "2" 136 | { "str" = "READ" } 137 | } 138 | { "3" 139 | { "str" = "WRITE" } 140 | } 141 | } 142 | } 143 | { "extent_size" 144 | { "int" = "8192" } 145 | { "#comment" = "4 Megabytes" } 146 | } 147 | { "max_lv" 148 | { "int" = "0" } 149 | } 150 | { "max_pv" 151 | { "int" = "0" } 152 | } 153 | {} 154 | { "physical_volumes" 155 | { "dict" 156 | { "pv0" 157 | { "dict" 158 | { "id" 159 | { "str" = "uuid-uuid-uuid-uuid" } 160 | } 161 | { "device" 162 | { "str" = "/dev/sda6" } 163 | { "#comment" = "Hint only" } 164 | } 165 | {} 166 | { "status" 167 | { "list" 168 | { "1" 169 | { "str" = "ALLOCATABLE" } 170 | } 171 | } 172 | } 173 | { "pe_start" 174 | { "int" = "123" } 175 | } 176 | { "pe_count" 177 | { "int" = "123456" } 178 | { "#comment" = "many Gigabytes" } 179 | } 180 | } 181 | } 182 | } 183 | } 184 | {} 185 | { "logical_volumes" 186 | { "dict" 187 | { "LogicalEek" 188 | { "dict" 189 | { "id" 190 | { "str" = "uuid-uuid-uuid-uuid" } 191 | } 192 | { "status" 193 | { "list" 194 | { "1" 195 | { "str" = "READ" } 196 | } 197 | { "2" 198 | { "str" = "WRITE" } 199 | } 200 | { "3" 201 | { "str" = "VISIBLE" } 202 | } 203 | } 204 | } 205 | { "segment_count" 206 | { "int" = "1" } 207 | } 208 | {} 209 | { "segment1" 210 | { "dict" 211 | { "start_extent" 212 | { "int" = "0" } 213 | } 214 | { "extent_count" 215 | { "int" = "123456" } 216 | { "#comment" = "beaucoup Gigabytes" } 217 | } 218 | {} 219 | { "type" 220 | { "str" = "striped" } 221 | } 222 | { "stripe_count" 223 | { "int" = "1" } 224 | { "#comment" = "linear" } 225 | } 226 | {} 227 | { "stripes" 228 | { "list" 229 | { "1" 230 | { "str" = "pv0" } 231 | } 232 | { "2" 233 | { "int" = "0" } 234 | } 235 | } 236 | } 237 | } 238 | } 239 | } 240 | } 241 | } 242 | } 243 | } 244 | } 245 | 246 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # blocks 2 | 3 | Conversion tools for block devices. 4 | 5 | Convert between raw partitions, logical volumes, and bcache devices 6 | without moving data. `blocks` shuffles blocks and sprouts superblocks. 7 | 8 | ## LVM conversion 9 | 10 | `blocks to-lvm` (alias: `lvmify`) takes a block device (partition or 11 | whole disk) containing a filesystem, shrinks the filesystem by a small 12 | amount, and converts it to LVM in place. 13 | 14 | The block device is converted to a physical volume and the filesystem is 15 | converted to a logical volume. If `--join=` is used the volumes 16 | join an existing volume group. 17 | 18 | An LVM conversion can be followed by other changes to the volume, 19 | growing it to multiple disks with `vgextend` and `lvextend`, or 20 | converting it to various RAID levels with `lvconvert --type=raidN 21 | -m`. 22 | 23 | ## bcache conversion 24 | 25 | `blocks to-bcache` converts a block device (partition, logical volume, 26 | LUKS device) to use bcache. If `--join=` is used the device 27 | joins an existing cache set. Otherwise you will need to [create 28 | and attach the cache device 29 | manually](http://evilpiepirate.org/git/linux-bcache.git/tree/Documentation/bcache.txt?h=bcache-dev#n80). 30 | 31 | `blocks` will pick one of several conversion strategies: 32 | 33 | * one for partitions, which requires a shrinkable filesystem or free space 34 | immediately before the partition to convert. Converting a [logical partition]( 35 | https://en.wikipedia.org/wiki/Extended_boot_record) 36 | to bcache is not supported: if `blocks` complains about overlapping metadata 37 | in the middle of the disk, please [use gdisk to convert your MBR disk to GPT]( 38 | http://falstaff.agner.ch/2012/11/20/convert-mbr-partition-table-to-gpt-ubuntu/) 39 | and reinstall your bootloader before proceeding with the bcache conversion. 40 | * one for LUKS volumes 41 | * one for LVM logical volumes 42 | 43 | When the first two strategies are unavailable, you can still convert 44 | to bcache by converting to LVM first, then converting the new LV to 45 | bcache. 46 | 47 | You will need to install bcache-tools, which is available here: 48 | 49 | * 50 | * (`sudo add-apt-repository ppa:g2p/storage`; for ubuntu 13.10 and newer) 51 | 52 | Conversion makes no demands on the kernel, but to use bcache, you need 53 | Linux 3.10 or newer. [My own branch](https://github.com/g2p/linux/commits/for-3.11/bcache) currently adds 54 | resizing support on top of [Kent Overstreet's upstream branch](http://evilpiepirate.org/git/linux-bcache.git/). 55 | 56 | ### maintboot mode 57 | 58 | Maintboot mode (`blocks to-bcache --maintboot`) is an easier way 59 | to convert in-use devices that doesn't require a LiveCD. 60 | [maintboot](https://github.com/g2p/maintboot) will run 61 | the conversion from an in-memory boot environment. 62 | This is currently tested on Ubuntu; ports to other 63 | distributions are welcome. 64 | 65 | # Ubuntu PPA (13.10 and newer) 66 | 67 | You can install python3-blocks from a PPA and skip the rest 68 | of the installation section. 69 | 70 | sudo apt-get install software-properties-common 71 | sudo add-apt-repository ppa:g2p/storage 72 | sudo apt-get update 73 | sudo apt-get install python3-blocks bcache-tools 74 | 75 | # Requirements 76 | 77 | Python 3.3, pip and Git are required before installing. 78 | 79 | You will also need libparted (2.3 or newer, library and headers) and 80 | libaugeas (library only, 1.0 or newer). 81 | 82 | On Debian/Ubuntu (Ubuntu 13.04 or newer is recommended): 83 | 84 | sudo aptitude install python3.3 python3-pip git libparted-dev libaugeas0 \ 85 | pkg-config libpython3.3-dev gcc 86 | sudo aptitude install cryptsetup lvm2 liblzo2-dev \ 87 | nilfs-tools reiserfsprogs xfsprogs e2fsprogs btrfs-tools # optional 88 | type pip-3.3 || alias pip-3.3='python3.3 -m pip.runner' 89 | 90 | Command-line tools for LVM2, LUKS, bcache (see above), filesystem 91 | resizing (see below for btrfs) are needed if those formats are involved. 92 | Kernel support isn't required however, so you can do bcache conversions 93 | from a live-cd/live-usb for example. 94 | 95 | For btrfs resizing, you need a package that provides `btrfs-show-super`, 96 | or you can install from source: 97 | 98 | * 99 | 100 | # Installation 101 | 102 | pip-3.3 install --user -r <(wget -O- https://raw.github.com/g2p/blocks/master/requirements.txt) 103 | cp -lt ~/bin ~/.local/bin/blocks 104 | 105 | # Usage 106 | 107 | ## Converting your root filesystem to LVM 108 | 109 | Install LVM. 110 | 111 | Edit your `/etc/fstab` to refer to filesystems by UUID, and regenerate 112 | your initramfs so that it picks up the new tools. 113 | 114 | With grub2, you don't need to switch to a separate boot 115 | partition, but make sure grub2 installs `lvm.mod` inside your `/boot`. 116 | 117 | Make sure your backups are up to date, boot to live media ([Ubuntu raring 118 | liveusb](http://cdimage.ubuntu.com/daily-live/current/) is a good 119 | choice), install blocks, and convert. 120 | 121 | ## Converting your root filesystem to bcache 122 | 123 | Install bcache-tools and a recent kernel (3.10 or newer). 124 | If your distribution uses Dracut (Fedora), you need Dracut 0.31 or newer. 125 | 126 | Edit your `/etc/fstab` to refer to filesystems by UUID, and regenerate 127 | your initramfs so that it picks up the new tools. 128 | On Debian and Ubuntu, this is done with `update-initramfs -u -k all`. 129 | With Dracut (Fedora), this is done with `dracut -f`. 130 | Arch Linux users should enable the bcache hook in `mkinitcpio.conf` 131 | and rerun `mkinitcpio`. 132 | If you don't see your distribution in this list, you are welcome to 133 | port [this hook](https://github.com/g2p/bcache-tools/blob/master/initcpio/install) 134 | to your distribution's preferred tools and contribute a patch to bcache-tools. 135 | Having working bcache support in your initramfs is important, as your system 136 | will be unbootable without. 137 | 138 | Edit your `grub.cfg` to refer to filesystems by UUID on the kernel 139 | command-line (this is often the case, except when you are already using 140 | LVM, in which case `update-grub` tends to write a logical path). Make 141 | sure you have a separate `/boot` partition. 142 | 143 | 1. If you don't have a cache device yet, create it on an empty SSD (or on a 144 | properly aligned partition or LV on top of it; LVM's 4MiB alignment is 145 | sufficient, as is the 1MiB alignment of modern partitioning tools). 146 | 147 | sudo make-bcache -C /dev/ 148 | This will give you a cache-set uuid. 149 | 150 | 2. If you already have a cache device 151 | 152 | ls /sys/fs/bcache 153 | And copy the cache-set uuid. 154 | 155 | 3. Finally, if you have a maintboot-compatible distribution, run: 156 | 157 | sudo blocks to-bcache --maintboot /dev/ --join 158 | If you are using encryption, use the encrypted device as the root device so 159 | that cache contents are also encrypted. 160 | 161 | 4. Otherwise, 162 | make sure your backups are up to date, boot to live media ([Ubuntu raring 163 | liveusb](http://cdimage.ubuntu.com/daily-live/current/) is a good 164 | choice), install blocks, and convert. 165 | 166 | ## bcache on a fresh install 167 | 168 | When using a distribution installer that doesn't support bcache 169 | at the partitioning stage, make sure the installer creates a 170 | separate `/boot` partition. Install everything on the HDD, 171 | using whatever layout you prefer (but I suggest LVM if you want 172 | multiple partitions). 173 | 174 | Once the installer is done, you can follow the steps at 175 | [converting your root filesystem to bcache](#converting-your-root-filesystem-to-bcache). 176 | 177 | ## Subcommand help 178 | 179 | blocks --help 180 | blocks --help 181 | 182 | If `blocks` isn't in the shell's command path, replace with: 183 | 184 | sudo python3.3 -m blocks 185 | 186 | # Build status 187 | 188 | [![Build Status](https://travis-ci.org/g2p/blocks.png)](https://travis-ci.org/g2p/blocks) 189 | 190 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /blocks/__main__.py: -------------------------------------------------------------------------------- 1 | # Python 3.3 2 | 3 | import argparse 4 | import contextlib 5 | import json 6 | import os 7 | import re 8 | import shutil 9 | import stat 10 | import string 11 | import struct 12 | import subprocess 13 | import sys 14 | import tempfile 15 | import textwrap 16 | import time 17 | import types 18 | import urllib.parse 19 | import uuid 20 | 21 | import pkg_resources 22 | 23 | 24 | # 4MiB PE, for vgmerge compatibility 25 | LVM_PE_SIZE = 4 * 1024 ** 2 26 | 27 | ASCII_ALNUM_WHITELIST = string.ascii_letters + string.digits + '.' 28 | 29 | BCACHE_MAGIC = bytes.fromhex('c6 85 73 f6 4e 1a 45 ca 82 65 f5 7f 48 ba 6d 81') 30 | 31 | 32 | # Fairly strict, snooping an incorrect mapping would be bad 33 | dm_crypt_re = re.compile( 34 | r'^0 (?P\d+) crypt (?P[a-z0-9:-]+) 0+ 0' 35 | ' (?P\d+):(?P\d+) (?P\d+)(?P [^\n]*)?\n\Z', 36 | re.ASCII) 37 | 38 | dm_kpartx_re = re.compile( 39 | r'^0 (?P\d+) linear' 40 | ' (?P\d+):(?P\d+) (?P\d+)\n\Z', 41 | re.ASCII) 42 | 43 | 44 | def bytes_to_sector(by): 45 | sectors, rem = divmod(by, 512) 46 | assert rem == 0 47 | return sectors 48 | 49 | 50 | def intdiv_up(num, denom): 51 | return (num - 1) // denom + 1 52 | 53 | 54 | def align_up(size, align): 55 | return intdiv_up(size, align) * align 56 | 57 | 58 | def align(size, align): 59 | return (size // align) * align 60 | 61 | 62 | class UnsupportedSuperblock(Exception): 63 | def __init__(self, *, device, **kwargs): 64 | self.device = device 65 | super().__init__(repr(dict(device=device, **kwargs))) 66 | 67 | 68 | class UnsupportedLayout(Exception): 69 | pass 70 | 71 | 72 | class CantShrink(Exception): 73 | pass 74 | 75 | 76 | class OverlappingPartition(Exception): 77 | pass 78 | 79 | 80 | # SQLa, compatible license 81 | class memoized_property(object): 82 | """A read-only @property that is only evaluated once.""" 83 | def __init__(self, fget, doc=None): 84 | self.fget = fget 85 | self.__doc__ = doc or fget.__doc__ 86 | self.__name__ = fget.__name__ 87 | 88 | def __get__(self, obj, cls): 89 | if obj is None: 90 | return self 91 | obj.__dict__[self.__name__] = result = self.fget(obj) 92 | return result 93 | 94 | def _reset(self, obj): 95 | obj.__dict__.pop(self.__name__, None) 96 | 97 | 98 | def quiet_call(cmd, *args, **kwargs): 99 | # universal_newlines is used to enable io decoding in the current locale 100 | proc = subprocess.Popen( 101 | cmd, *args, universal_newlines=True, stdin=subprocess.DEVNULL, 102 | stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) 103 | odat, edat = proc.communicate() 104 | if proc.returncode != 0: 105 | print( 106 | 'Command {!r} has failed with status {}\n' 107 | 'Standard output:\n{}\n' 108 | 'Standard error:\n{}'.format( 109 | cmd, proc.returncode, odat, edat), file=sys.stderr) 110 | raise subprocess.CalledProcessError(proc.returncode, cmd, odat) 111 | 112 | 113 | class MissingRequirement(Exception): 114 | pass 115 | 116 | 117 | class Requirement: 118 | @classmethod 119 | def require(self, progress): 120 | assert '/' not in self.cmd 121 | if shutil.which(self.cmd) is None: 122 | progress.bail( 123 | 'Command {!r} not found, please install the {} package' 124 | .format(self.cmd, self.pkg), MissingRequirement(self)) 125 | 126 | 127 | class LVMReq(Requirement): 128 | cmd = 'lvm' 129 | pkg = 'lvm2' 130 | 131 | 132 | class BCacheReq(Requirement): 133 | cmd = 'make-bcache' 134 | pkg = 'bcache-tools' 135 | 136 | 137 | def mk_dm(devname, table, readonly, exit_stack): 138 | needs_udev_fallback = False 139 | cmd = 'dmsetup create --noudevsync --'.split() + [devname] 140 | if readonly: 141 | cmd[3:3] = ['--readonly'] 142 | proc = subprocess.Popen(cmd, stdin=subprocess.PIPE) 143 | proc.communicate(table.encode('ascii')) 144 | if proc.returncode != 0: 145 | needs_udev_fallback = True 146 | # dmsetup 1.02.65, wheezy/quantal 147 | cmd[3:3] = ['--verifyudev'] 148 | proc = subprocess.Popen(cmd, stdin=subprocess.PIPE) 149 | proc.communicate(table.encode('ascii')) 150 | assert proc.returncode == 0, 'Report to https://github.com/g2p/blocks/issues/8 if you see this' 151 | cmd = 'dmsetup remove --noudevsync --'.split() + [devname] 152 | if needs_udev_fallback: 153 | cmd[3:3] = ['--verifyudev'] 154 | exit_stack.callback(lambda: quiet_call(cmd)) 155 | 156 | 157 | def aftersep(line, sep): 158 | return line.split(sep, 1)[1].rstrip('\n') 159 | 160 | 161 | def devpath_from_sysdir(sd): 162 | with open(sd + '/uevent') as fi: 163 | for line in fi: 164 | if line.startswith('DEVNAME='): 165 | return '/dev/' + aftersep(line, '=') 166 | 167 | 168 | class BlockDevice: 169 | def __init__(self, devpath): 170 | assert os.path.exists(devpath), devpath 171 | self.devpath = devpath 172 | 173 | @classmethod 174 | def by_uuid(cls, uuid): 175 | return cls(devpath=subprocess.check_output( 176 | ['blkid', '-U', uuid]).rstrip()) 177 | 178 | def open_excl(self): 179 | # O_EXCL on a block device takes the device lock, 180 | # exclusive against mounts and the like. 181 | # O_SYNC on a block device provides durability, see: 182 | # http://www.codeproject.com/Articles/460057/HDD-FS-O_SYNC-Throughput-vs-Integrity 183 | # O_DIRECT would bypass the block cache, which is irrelevant here 184 | return os.open( 185 | self.devpath, os.O_SYNC | os.O_RDWR | os.O_EXCL) 186 | 187 | @contextlib.contextmanager 188 | def open_excl_ctx(self): 189 | dev_fd = self.open_excl() 190 | yield dev_fd 191 | os.close(dev_fd) 192 | 193 | @memoized_property 194 | def ptable_type(self): 195 | # TODO: also detect an MBR other than protective, 196 | # and refuse to edit that. 197 | rv = subprocess.check_output( 198 | 'blkid -p -o value -s PTTYPE --'.split() + [self.devpath] 199 | ).rstrip().decode('ascii') 200 | if rv: 201 | return rv 202 | 203 | @memoized_property 204 | def superblock_type(self): 205 | return self.superblock_at(0) 206 | 207 | def superblock_at(self, offset): 208 | try: 209 | return subprocess.check_output( 210 | 'blkid -p -o value -s TYPE -O'.split() 211 | + ['%d' % offset, '--', self.devpath] 212 | ).rstrip().decode('ascii') 213 | except subprocess.CalledProcessError as err: 214 | # No recognised superblock 215 | assert err.returncode == 2, err 216 | 217 | @memoized_property 218 | def has_bcache_superblock(self): 219 | # blkid doesn't detect bcache, so special-case it. 220 | # To keep dependencies light, don't use bcache-tools for detection, 221 | # only require the tools after a successful detection. 222 | if self.size <= 8192: 223 | return False 224 | sbfd = os.open(self.devpath, os.O_RDONLY) 225 | magic, = struct.unpack('16s', os.pread(sbfd, 16, 4096 + 24)) 226 | os.close(sbfd) 227 | return magic == BCACHE_MAGIC 228 | 229 | @memoized_property 230 | def size(self): 231 | rv = int(subprocess.check_output( 232 | 'blockdev --getsize64'.split() + [self.devpath])) 233 | assert rv % 512 == 0 234 | return rv 235 | 236 | def reset_size(self): 237 | type(self).size._reset(self) 238 | 239 | @property 240 | def sysfspath(self): 241 | # pyudev would also work 242 | st = os.stat(self.devpath) 243 | assert stat.S_ISBLK(st.st_mode) 244 | return '/sys/dev/block/%d:%d' % self.devnum 245 | 246 | @property 247 | def devnum(self): 248 | st = os.stat(self.devpath) 249 | assert stat.S_ISBLK(st.st_mode) 250 | return (os.major(st.st_rdev), os.minor(st.st_rdev)) 251 | 252 | def iter_holders(self): 253 | for hld in os.listdir(self.sysfspath + '/holders'): 254 | yield BlockDevice('/dev/' + hld) 255 | 256 | @memoized_property 257 | def is_dm(self): 258 | return os.path.exists(self.sysfspath + '/dm') 259 | 260 | @memoized_property 261 | def is_lv(self): 262 | if not self.is_dm: 263 | return False 264 | try: 265 | pe_size = int(subprocess.check_output( 266 | 'lvm lvs --noheadings --rows --units=b --nosuffix ' 267 | '-o vg_extent_size --'.split() 268 | + [self.devpath], universal_newlines=True)) 269 | except subprocess.CalledProcessError: 270 | return False 271 | else: 272 | return True 273 | 274 | 275 | def dm_table(self): 276 | return subprocess.check_output( 277 | 'dmsetup table --'.split() + [self.devpath], 278 | universal_newlines=True) 279 | 280 | def dm_deactivate(self): 281 | return quiet_call( 282 | 'dmsetup remove --'.split() + [self.devpath]) 283 | 284 | def dm_setup(self, table, readonly): 285 | cmd = 'dmsetup create --'.split() + [self.devpath] 286 | if readonly: 287 | cmd[2:2] = ['--readonly'] 288 | proc = subprocess.Popen(cmd, stdin=subprocess.PIPE) 289 | proc.communicate(table.encode('ascii')) 290 | assert proc.returncode == 0 291 | 292 | @memoized_property 293 | def is_partition(self): 294 | return ( 295 | os.path.exists(self.sysfspath + '/partition') and 296 | bool(int(open(self.sysfspath + '/partition').read()))) 297 | 298 | def ptable_context(self): 299 | # the outer ptable and our offset within that 300 | import parted.disk 301 | 302 | assert self.is_partition 303 | 304 | ptable_device = PartitionedDevice( 305 | devpath_from_sysdir(self.sysfspath + '/..')) 306 | 307 | part_start = int(open(self.sysfspath + '/start').read()) * 512 308 | ptable = PartitionTable( 309 | device=ptable_device, 310 | parted_disk=parted.disk.Disk(ptable_device.parted_device)) 311 | return ptable, part_start 312 | 313 | def dev_resize(self, newsize, shrink): 314 | newsize = align_up(newsize, 512) 315 | # Be explicit about the intended direction; 316 | # shrink is more dangerous 317 | if self.is_partition: 318 | ptable, part_start = self.ptable_context() 319 | ptable.part_resize(part_start, newsize, shrink) 320 | elif self.is_lv: 321 | if shrink: 322 | cmd = ['lvm', 'lvreduce', '-f'] 323 | else: 324 | # Alloc policy / dest PVs might be useful here, 325 | # but difficult to expose cleanly. 326 | # Just don't use --resize-device and do it manually. 327 | cmd = ['lvm', 'lvextend'] 328 | quiet_call(cmd + ['--size=%db' % newsize, '--', self.devpath]) 329 | else: 330 | raise NotImplementedError('Only partitions and LVs can be resized') 331 | self.reset_size() 332 | 333 | 334 | class PartitionedDevice(BlockDevice): 335 | @memoized_property 336 | def parted_device(self): 337 | import parted.device 338 | return parted.device.Device(self.devpath) 339 | 340 | 341 | class BlockData: 342 | def __init__(self, device): 343 | self.device = device 344 | 345 | 346 | class PartitionTable(BlockData): 347 | def __init__(self, device, parted_disk): 348 | super(PartitionTable, self).__init__(device=device) 349 | self.parted_disk = parted_disk 350 | 351 | @classmethod 352 | def mkgpt(cls, device): 353 | import parted 354 | ptable_device = PartitionedDevice(device.devpath) 355 | return cls( 356 | device=ptable_device, 357 | parted_disk=parted.freshDisk(ptable_device.parted_device, 'gpt')) 358 | 359 | def _iter_range(self, start_sector, end_sector): 360 | # Loop on partitions overlapping with the range, excluding free space 361 | 362 | # Careful: end_sector is exclusive here, 363 | # but parted geometry uses inclusive ends. 364 | 365 | import _ped 366 | while start_sector < end_sector: 367 | part = self.parted_disk.getPartitionBySector(start_sector) 368 | if not (part.type & _ped.PARTITION_FREESPACE): 369 | yield part 370 | # inclusive, so add one 371 | start_sector = part.geometry.end + 1 372 | 373 | def _reserve_range(self, start, end, progress): 374 | import _ped 375 | assert 0 <= start <= end 376 | 377 | # round down 378 | start_sector = start // 512 379 | 380 | # round up 381 | end_sector = intdiv_up(end, 512) 382 | 383 | part = None 384 | for part in self._iter_range(start_sector, end_sector): 385 | if part.geometry.start >= start_sector: 386 | if part.number == -1: 387 | progress.bail( 388 | 'The range we want to reserve overlaps with ' 389 | 'the start of a reserved area at [{}, {}] ({}), ' 390 | 'the shrinking strategy will not work.'.format( 391 | part.geometry.start, part.geometry.end, 392 | _ped.partition_type_get_name(part.type)), 393 | OverlappingPartition(start, end, part)) 394 | progress.bail( 395 | 'The range we want to reserve overlaps with ' 396 | 'the start of partition {} ({}), the shrinking strategy ' 397 | 'will not work.'.format( 398 | part.path, _ped.partition_type_get_name(part.type)), 399 | OverlappingPartition(start, end, part)) 400 | 401 | if part is None: 402 | # No partitions inside the range, we're good 403 | return 404 | 405 | # There's a single overlapping partition, 406 | # and it starts outside the range. Shrink it. 407 | 408 | part_newsize = (start_sector - part.geometry.start) * 512 409 | 410 | block_stack = get_block_stack(BlockDevice(part.path), progress) 411 | 412 | block_stack.read_superblocks() 413 | block_stack.stack_reserve_end_area(part_newsize, progress) 414 | 415 | def reserve_space_before(self, part_start, length, progress): 416 | assert part_start >= length, (part_start, length) 417 | 418 | start_sector = bytes_to_sector(part_start) 419 | 420 | # Just check part_start is indeed the start of a partition 421 | part = self.parted_disk.getPartitionBySector(start_sector) 422 | if part.geometry.start != start_sector: 423 | raise KeyError(part_start, self) 424 | 425 | return self._reserve_range(part_start - length, part_start, progress) 426 | 427 | def part_resize(self, part_start, newsize, shrink): 428 | import parted.geometry 429 | import parted.constraint 430 | 431 | start_sector = bytes_to_sector(part_start) 432 | part = self.parted_disk.getPartitionBySector(start_sector) 433 | # Parted uses inclusive ends, so substract one 434 | new_end = part.geometry.start + bytes_to_sector(newsize) - 1 435 | if shrink: 436 | assert new_end < part.geometry.end 437 | else: 438 | assert new_end > part.geometry.end 439 | geom = parted.geometry.Geometry( 440 | device=self.device.parted_device, 441 | start=part.geometry.start, 442 | end=new_end) 443 | # We want an aligned region at least as large as newsize 444 | # TODO: add a CLI arg for simply getting the max region 445 | # The user could get the max region wrong if aligning 446 | # makes it slightly smaller for example 447 | optim = self.device.parted_device.optimalAlignedConstraint 448 | solve_max = parted.constraint.Constraint(minGeom=geom) 449 | cons = optim.intersect(solve_max) 450 | assert self.parted_disk.setPartitionGeometry( 451 | part, cons, geom.start, geom.end) is True 452 | self.parted_disk.commit() 453 | 454 | def shift_left(self, part_start, part_start1): 455 | assert part_start1 < part_start 456 | start_sector = bytes_to_sector(part_start) 457 | start_sector1 = bytes_to_sector(part_start1) 458 | 459 | import parted.geometry 460 | import parted.constraint 461 | import _ped 462 | 463 | left_part = self.parted_disk.getPartitionBySector(start_sector1) 464 | right_part = self.parted_disk.getPartitionBySector(start_sector) 465 | 466 | if left_part.type != _ped.PARTITION_FREESPACE: 467 | geom = parted.geometry.Geometry( 468 | device=self.device.parted_device, 469 | start=left_part.geometry.start, 470 | end=start_sector1 - 1) 471 | cons = parted.constraint.Constraint(exactGeom=geom) 472 | assert self.parted_disk.setPartitionGeometry( 473 | left_part, cons, geom.start, geom.end) is True 474 | 475 | geom = parted.geometry.Geometry( 476 | device=self.device.parted_device, 477 | start=start_sector1, 478 | end=right_part.geometry.end) 479 | cons = parted.constraint.Constraint(exactGeom=geom) 480 | assert self.parted_disk.setPartitionGeometry( 481 | right_part, cons, geom.start, geom.end) is True 482 | 483 | # commitToDevice (atomic) + commitToOS (not atomic, less important) 484 | self.parted_disk.commit() 485 | 486 | 487 | class Filesystem(BlockData): 488 | resize_needs_mpoint = False 489 | sb_size_in_bytes = False 490 | 491 | def reserve_end_area_nonrec(self, pos): 492 | # align to a block boundary that doesn't encroach 493 | pos = align(pos, self.block_size) 494 | 495 | if self.fssize <= pos: 496 | return 497 | 498 | if not self.can_shrink: 499 | raise CantShrink(self) 500 | 501 | self._mount_and_resize(pos) 502 | return pos 503 | 504 | @contextlib.contextmanager 505 | def temp_mount(self): 506 | # Don't use TemporaryDirectory, recursive cleanup 507 | # on a mountpoint would be bad 508 | mpoint = tempfile.mkdtemp(suffix='.privmnt') 509 | # Don't pass -n, Nilfs relies on /etc/mtab to find its mountpoint 510 | # TODO: use unshare() here 511 | quiet_call( 512 | ['mount', '-t', self.vfstype, '-o', 'noatime,noexec,nodev', 513 | '--', self.device.devpath, mpoint]) 514 | try: 515 | yield mpoint 516 | finally: 517 | quiet_call('umount -- '.split() + [mpoint]) 518 | os.rmdir(mpoint) 519 | 520 | def is_mounted(self): 521 | dn = '%d:%d' % self.device.devnum 522 | with open('/proc/self/mountinfo') as mounts: 523 | for line in mounts: 524 | items = line.split() 525 | if False: 526 | idx = items.index('-') 527 | fs_type = items[idx + 1] 528 | opts1 = items[5].split(',') 529 | opts2 = items[idx + 3].split(',') 530 | readonly = 'ro' in opts1 + opts2 531 | intpath = items[3] 532 | mpoint = items[4] 533 | dev = os.path.realpath(items[idx + 2]) 534 | devnum = items[2] 535 | if dn == devnum: 536 | return True 537 | return False 538 | 539 | 540 | def _mount_and_resize(self, pos): 541 | if self.resize_needs_mpoint and not self.is_mounted(): 542 | with self.temp_mount(): 543 | self._resize(pos) 544 | else: 545 | self._resize(pos) 546 | 547 | # measure size again 548 | self.read_superblock() 549 | assert self.fssize == pos 550 | 551 | def grow_nonrec(self, upper_bound): 552 | newsize = align(upper_bound, self.block_size) 553 | assert self.fssize <= newsize 554 | if self.fssize == newsize: 555 | return 556 | self._mount_and_resize(newsize) 557 | return newsize 558 | 559 | @property 560 | def fssize(self): 561 | if self.sb_size_in_bytes: 562 | assert self.size_bytes % self.block_size == 0 563 | return self.size_bytes 564 | else: 565 | return self.block_size * self.block_count 566 | 567 | @memoized_property 568 | def fslabel(self): 569 | return subprocess.check_output( 570 | 'blkid -o value -s LABEL --'.split() + [self.device.devpath] 571 | ).rstrip().decode('ascii') 572 | 573 | @memoized_property 574 | def fsuuid(self): 575 | return subprocess.check_output( 576 | 'blkid -o value -s UUID --'.split() + [self.device.devpath] 577 | ).rstrip().decode('ascii') 578 | 579 | 580 | class SimpleContainer(BlockData): 581 | # A single block device that wraps a single block device 582 | # (luks is one, but not lvm, lvm is m2m) 583 | 584 | offset = None 585 | 586 | 587 | def starts_with_word(line, word): 588 | return line.startswith(word) and line.split(maxsplit=1)[0] == word 589 | 590 | 591 | class BCacheBacking(SimpleContainer): 592 | def read_superblock(self): 593 | self.offset = None 594 | self.version = None 595 | 596 | proc = subprocess.Popen( 597 | ['bcache-super-show', '--', self.device.devpath], 598 | stdout=subprocess.PIPE) 599 | for line in proc.stdout: 600 | if starts_with_word(line, b'sb.version'): 601 | line = line.decode('ascii') 602 | self.version = int(line.split()[1]) 603 | elif starts_with_word(line, b'dev.data.first_sector'): 604 | line = line.decode('ascii') 605 | self.offset = int(line.split(maxsplit=1)[1]) * 512 606 | proc.wait() 607 | assert proc.returncode == 0 608 | assert self.offset is not None 609 | 610 | @property 611 | def is_backing(self): 612 | # Whitelist versions, just in case newer backing devices 613 | # are too different 614 | return self.version in (1, 4) 615 | 616 | def is_activated(self): 617 | return os.path.exists(self.device.sysfspath + '/bcache') 618 | 619 | @memoized_property 620 | def cached_device(self): 621 | if not self.is_activated(): 622 | # XXX How synchronous is this? 623 | with open('/sys/fs/bcache/register', 'w') as br: 624 | br.write(self.devpath + '\n') 625 | return BlockDevice(devpath_from_sysdir(self.device.sysfspath + '/bcache/dev')) 626 | 627 | def deactivate(self): 628 | with open(self.device.sysfspath + '/bcache/stop', 'w') as sf: 629 | # XXX Asynchronous 630 | sf.write('stop\n') 631 | assert not self.is_activated() 632 | type(self).cached_device._reset(self) 633 | 634 | def grow_nonrec(self, upper_bound): 635 | if upper_bound != self.device.size: 636 | raise NotImplementedError 637 | if not self.is_activated(): 638 | # Nothing to do, bcache will pick up the size on activation 639 | return 640 | with open(self.device.sysfspath + '/bcache/resize', 'w') as sf: 641 | # XXX How synchronous is this? 642 | sf.write('max\n') 643 | self.cached_device.reset_size() 644 | assert self.cached_device.size + self.offset == upper_bound, (self.cached_device.size, self.offset, upper_bound) 645 | return upper_bound 646 | 647 | 648 | class LUKS(SimpleContainer): 649 | """ 650 | pycryptsetup isn't used because: 651 | it isn't in PyPI, or in Debian or Ubuntu 652 | it isn't Python 3 653 | it's incomplete (resize not included) 654 | """ 655 | 656 | _superblock_read = False 657 | 658 | def activate(self, dmname): 659 | # cs.activate 660 | subprocess.check_call( 661 | ['cryptsetup', 'luksOpen', '--', self.device.devpath, dmname]) 662 | 663 | def deactivate(self): 664 | while True: 665 | dev = self.snoop_activated() 666 | if dev is None: 667 | break 668 | subprocess.check_call( 669 | ['cryptsetup', 'remove', '--', dev.devpath]) 670 | type(self).cleartext_device._reset(self) 671 | 672 | def snoop_activated(self): 673 | for hld in self.device.iter_holders(): 674 | if not self._superblock_read: 675 | self.read_superblock() 676 | match = dm_crypt_re.match(hld.dm_table()) 677 | # Having the correct offset ensures we're not getting 678 | # the size of a smaller filesystem inside the partition 679 | if ( 680 | match and 681 | int(match.group('offset')) == bytes_to_sector(self.offset) 682 | ): 683 | return hld 684 | 685 | @memoized_property 686 | def cleartext_device(self): 687 | # If the device is already activated we won't have 688 | # to prompt for a passphrase. 689 | dev = self.snoop_activated() 690 | if dev is None: 691 | dmname = 'cleartext-{}'.format(uuid.uuid1()) 692 | self.activate(dmname) 693 | dev = BlockDevice('/dev/mapper/' + dmname) 694 | return dev 695 | 696 | def read_superblock(self): 697 | # read the cyphertext's luks superblock 698 | #self.offset = cs.info()['offset'] # pycryptsetup 699 | self.offset = None 700 | 701 | proc = subprocess.Popen( 702 | ['cryptsetup', 'luksDump', '--', self.device.devpath], 703 | stdout=subprocess.PIPE) 704 | for line in proc.stdout: 705 | if line.startswith(b'Payload offset:'): 706 | line = line.decode('ascii') 707 | self.offset = int(aftersep(line, ':')) * 512 708 | proc.wait() 709 | assert proc.returncode == 0 710 | self._superblock_read = True 711 | 712 | def read_superblock_ll(self, fd): 713 | # Low-level 714 | # https://cryptsetup.googlecode.com/git/docs/on-disk-format.pdf 715 | 716 | self.sb_end = None 717 | magic, version = struct.unpack('>6sH', os.pread(fd, 8, 0)) 718 | assert magic == b'LUKS\xBA\xBE', magic 719 | assert version == 1 720 | 721 | payload_start_sectors, key_bytes = struct.unpack( 722 | '>2I', os.pread(fd, 8, 104)) 723 | sb_end = 592 724 | 725 | for key_slot in range(8): 726 | key_offset, key_stripes = struct.unpack( 727 | '>2I', os.pread(fd, 8, 208 + 48 * key_slot + 40)) 728 | assert key_stripes == 4000 729 | key_size = key_stripes * key_bytes 730 | key_end = key_offset * 512 + key_size 731 | if key_end > sb_end: 732 | sb_end = key_end 733 | 734 | ll_offset = payload_start_sectors * 512 735 | assert ll_offset == self.offset, (ll_offset, self.offset) 736 | assert ll_offset >= sb_end 737 | self.sb_end = sb_end 738 | 739 | def shift_sb(self, fd, shift_by): 740 | assert shift_by > 0 741 | assert shift_by % 512 == 0 742 | assert self.offset % 512 == 0 743 | assert self.sb_end + shift_by <= self.offset 744 | 745 | # Read the superblock 746 | sb = os.pread(fd, self.sb_end, 0) 747 | assert len(sb) == self.sb_end 748 | 749 | # Edit the sb 750 | offset_sectors, = struct.unpack_from('>I', sb, 104) 751 | assert offset_sectors * 512 == self.offset 752 | sb = bytearray(sb) 753 | struct.pack_into( 754 | '>I', sb, 104, offset_sectors - shift_by // 512) 755 | sb = bytes(sb) 756 | 757 | # Wipe the magic and write the shifted, edited superblock 758 | wr_len = os.pwrite(fd, b'\0' * shift_by + sb, 0) 759 | assert wr_len == shift_by + self.sb_end 760 | 761 | # Wipe the results of read_superblock_ll 762 | # Keep self.offset for now 763 | del self.sb_end 764 | 765 | def grow_nonrec(self, upper_bound): 766 | return self.reserve_end_area_nonrec(upper_bound) 767 | 768 | def reserve_end_area_nonrec(self, pos): 769 | # cryptsetup uses the inner size 770 | inner_size = pos - self.offset 771 | sectors = bytes_to_sector(inner_size) 772 | 773 | # pycryptsetup is useless, no resize support 774 | # otoh, size doesn't appear in the superblock, 775 | # and updating the dm table is only useful if 776 | # we want to do some fsck before deactivating 777 | subprocess.check_call( 778 | ['cryptsetup', 'resize', '--size=%d' % sectors, 779 | '--', self.cleartext_device.devpath]) 780 | if self.snoop_activated(): 781 | self.cleartext_device.reset_size() 782 | assert self.cleartext_device.size == inner_size 783 | return pos 784 | 785 | 786 | class XFS(Filesystem): 787 | can_shrink = False 788 | resize_needs_mpoint = True 789 | vfstype = 'xfs' 790 | 791 | def read_superblock(self): 792 | self.block_size = None 793 | self.block_count = None 794 | 795 | proc = subprocess.Popen( 796 | ['xfs_db', '-c', 'sb 0', '-c', 'p dblocks blocksize', 797 | '--', self.device.devpath], stdout=subprocess.PIPE) 798 | for line in proc.stdout: 799 | if line.startswith(b'dblocks ='): 800 | line = line.decode('ascii') 801 | self.block_count = int(aftersep(line, '=')) 802 | elif line.startswith(b'blocksize ='): 803 | line = line.decode('ascii') 804 | self.block_size = int(aftersep(line, '=')) 805 | proc.wait() 806 | assert proc.returncode == 0 807 | 808 | def _resize(self, target_size): 809 | assert target_size % self.block_size == 0 810 | target_blocks = target_size // self.block_size 811 | quiet_call( 812 | ['xfs_growfs', '-D', '%d' % target_blocks, 813 | '--', self.device.devpath]) 814 | 815 | 816 | class NilFS(Filesystem): 817 | can_shrink = True 818 | sb_size_in_bytes = True 819 | resize_needs_mpoint = True 820 | vfstype = 'nilfs2' 821 | 822 | def read_superblock(self): 823 | self.block_size = None 824 | self.size_bytes = None 825 | 826 | proc = subprocess.Popen( 827 | 'nilfs-tune -l --'.split() + [self.device.devpath], 828 | stdout=subprocess.PIPE) 829 | 830 | for line in proc.stdout: 831 | if line.startswith(b'Block size:'): 832 | line = line.decode('ascii') 833 | self.block_size = int(aftersep(line, ':')) 834 | elif line.startswith(b'Device size:'): 835 | line = line.decode('ascii') 836 | self.size_bytes = int(aftersep(line, ':')) 837 | proc.wait() 838 | assert proc.returncode == 0 839 | 840 | def _resize(self, target_size): 841 | assert target_size % self.block_size == 0 842 | quiet_call( 843 | ['nilfs-resize', '--yes', '--', 844 | self.device.devpath, '%d' % target_size]) 845 | 846 | 847 | class BtrFS(Filesystem): 848 | can_shrink = True 849 | sb_size_in_bytes = True 850 | # We'll get the mpoint ourselves 851 | resize_needs_mpoint = False 852 | vfstype = 'btrfs' 853 | 854 | def read_superblock(self): 855 | self.block_size = None 856 | self.size_bytes = None 857 | self.devid = None 858 | 859 | proc = subprocess.Popen( 860 | 'btrfs-show-super --'.split() + [self.device.devpath], 861 | stdout=subprocess.PIPE) 862 | 863 | for line in proc.stdout: 864 | if starts_with_word(line, b'dev_item.devid'): 865 | line = line.decode('ascii') 866 | self.devid = int(line.split(maxsplit=1)[1]) 867 | elif starts_with_word(line, b'sectorsize'): 868 | line = line.decode('ascii') 869 | self.block_size = int(line.split(maxsplit=1)[1]) 870 | elif starts_with_word(line, b'dev_item.total_bytes'): 871 | line = line.decode('ascii') 872 | self.size_bytes = int(line.split(maxsplit=1)[1]) 873 | proc.wait() 874 | assert proc.returncode == 0 875 | 876 | def _resize(self, target_size): 877 | assert target_size % self.block_size == 0 878 | # XXX The device is unavailable (EBUSY) 879 | # immediately after unmounting. 880 | # Bug introduced in Linux 3.0, fixed in 3.9. 881 | # Tracked down by Eric Sandeen in 882 | # http://comments.gmane.org/gmane.comp.file-systems.btrfs/23987 883 | with self.temp_mount() as mpoint: 884 | quiet_call( 885 | 'btrfs filesystem resize'.split() 886 | + ['{}:{}'.format(self.devid, target_size), mpoint]) 887 | 888 | 889 | class ReiserFS(Filesystem): 890 | can_shrink = True 891 | 892 | def read_superblock(self): 893 | self.block_size = None 894 | self.block_count = None 895 | 896 | proc = subprocess.Popen( 897 | 'reiserfstune --'.split() + [self.device.devpath], 898 | stdout=subprocess.PIPE) 899 | 900 | for line in proc.stdout: 901 | if line.startswith(b'Blocksize:'): 902 | line = line.decode('ascii') 903 | self.block_size = int(aftersep(line, ':')) 904 | elif line.startswith(b'Count of blocks on the device:'): 905 | line = line.decode('ascii') 906 | self.block_count = int(aftersep(line, ':')) 907 | proc.wait() 908 | assert proc.returncode == 0 909 | 910 | def _resize(self, target_size): 911 | assert target_size % self.block_size == 0 912 | subprocess.check_call( 913 | ['resize_reiserfs', '-q', '-s', '%d' % target_size, 914 | '--', self.device.devpath]) 915 | 916 | 917 | class ExtFS(Filesystem): 918 | can_shrink = True 919 | 920 | def read_superblock(self): 921 | self.block_size = None 922 | self.block_count = None 923 | self.state = None 924 | self.mount_tm = None 925 | self.check_tm = None 926 | 927 | proc = subprocess.Popen( 928 | 'tune2fs -l --'.split() + [self.device.devpath], 929 | stdout=subprocess.PIPE) 930 | 931 | for line in proc.stdout: 932 | if line.startswith(b'Block size:'): 933 | line = line.decode('ascii') 934 | self.block_size = int(aftersep(line, ':')) 935 | elif line.startswith(b'Block count:'): 936 | line = line.decode('ascii') 937 | self.block_count = int(aftersep(line, ':')) 938 | elif line.startswith(b'Filesystem state:'): 939 | line = line.decode('ascii') 940 | self.state = aftersep(line, ':').lstrip() 941 | elif line.startswith(b'Last mount time:'): 942 | line = line.decode('ascii') 943 | date = aftersep(line, ':').lstrip() 944 | if date == 'n/a': 945 | self.mount_tm = time.gmtime(0) 946 | else: 947 | self.mount_tm = time.strptime(date) 948 | elif line.startswith(b'Last checked:'): 949 | line = line.decode('ascii') 950 | self.check_tm = time.strptime(aftersep(line, ':').lstrip()) 951 | proc.wait() 952 | assert proc.returncode == 0 953 | 954 | def _resize(self, target_size): 955 | block_count, rem = divmod(target_size, self.block_size) 956 | assert rem == 0 957 | 958 | # resize2fs requires that the filesystem was checked 959 | if not self.is_mounted() and (self.state != 'clean' or self.check_tm < self.mount_tm): 960 | print('Checking the filesystem before resizing it') 961 | # Can't use the -n flag, it is strictly read-only and won't 962 | # update check_tm in the superblock 963 | # XXX Without either of -n -p -y, e2fsck will require a 964 | # terminal on stdin 965 | subprocess.check_call( 966 | 'e2fsck -f --'.split() + [self.device.devpath]) 967 | # Another option: 968 | #quiet_call('e2fsck -fp --'.split() + [self.device.devpath]) 969 | self.check_tm = self.mount_tm 970 | quiet_call( 971 | 'resize2fs --'.split() + [self.device.devpath, '%d' % block_count]) 972 | 973 | 974 | class Swap(Filesystem): 975 | # Not exactly a filesystem 976 | can_shrink = True 977 | 978 | def is_mounted(self): 979 | # parse /proc/swaps, see tab_parse.c 980 | raise NotImplementedError 981 | 982 | def read_superblock(self): 983 | # No need to do anything, UUID and LABEL are already 984 | # exposed through blkid 985 | with self.device.open_excl_ctx() as dev_fd: 986 | big_endian, version, last_page = self.__read_sb(dev_fd) 987 | self.block_size = 4096 988 | self.block_count = last_page + 1 989 | self.big_endian = big_endian 990 | self.version = version 991 | 992 | 993 | def __read_sb(self, dev_fd): 994 | # Assume 4k pages, bail otherwise 995 | # XXX The SB checks should be done before calling the constructor 996 | magic, = struct.unpack('10s', os.pread(dev_fd, 10, 4096 - 10)) 997 | if magic != b'SWAPSPACE2': 998 | # Might be suspend data 999 | raise UnsupportedSuperblock(device=self.device, magic=magic) 1000 | version, last_page = struct.unpack('>II', os.pread(dev_fd, 8, 1024)) 1001 | big_endian = True 1002 | if version != 1: 1003 | version0 = version 1004 | version, last_page = struct.unpack(' inner_pos: 1079 | if self.topmost.can_shrink: 1080 | progress.notify( 1081 | 'Will shrink the filesystem ({}) by {} bytes' 1082 | .format(fstype, shrink_size)) 1083 | else: 1084 | progress.bail( 1085 | 'Can\'t shrink filesystem ({}), but need another {} bytes ' 1086 | 'at the end'.format(fstype, shrink_size), 1087 | CantShrink(self.topmost)) 1088 | else: 1089 | progress.notify( 1090 | 'The filesystem ({}) leaves enough room, ' 1091 | 'no need to shrink it'.format(fstype)) 1092 | 1093 | # While there may be no need to shrink the topmost fs, 1094 | # the wrapper stack needs to be updated for the new size 1095 | for inner_pos, block_data in reversed(list(self.iter_pos(pos))): 1096 | block_data.reserve_end_area_nonrec(inner_pos) 1097 | 1098 | def read_superblocks(self): 1099 | for wrapper in self.wrappers: 1100 | wrapper.read_superblock() 1101 | self.topmost.read_superblock() 1102 | 1103 | def deactivate(self): 1104 | for wrapper in reversed(self.wrappers): 1105 | wrapper.deactivate() 1106 | # Salt the earth, our devpaths are obsolete now 1107 | del self.stack 1108 | 1109 | 1110 | def get_block_stack(device, progress): 1111 | # this cries for a conslist 1112 | stack = [] 1113 | while True: 1114 | if device.superblock_type == 'crypto_LUKS': 1115 | wrapper = LUKS(device) 1116 | stack.append(wrapper) 1117 | device = wrapper.cleartext_device 1118 | continue 1119 | elif device.has_bcache_superblock: 1120 | wrapper = BCacheBacking(device) 1121 | wrapper.read_superblock() 1122 | if not wrapper.is_backing: 1123 | # We only want backing, not all bcache superblocks 1124 | progress.bail( 1125 | 'BCache device isn\'t a backing device', 1126 | UnsupportedSuperblock(device=device)) 1127 | stack.append(wrapper) 1128 | device = wrapper.cached_device 1129 | continue 1130 | 1131 | if device.superblock_type in {'ext2', 'ext3', 'ext4'}: 1132 | stack.append(ExtFS(device)) 1133 | elif device.superblock_type == 'reiserfs': 1134 | stack.append(ReiserFS(device)) 1135 | elif device.superblock_type == 'btrfs': 1136 | stack.append(BtrFS(device)) 1137 | elif device.superblock_type == 'nilfs2': 1138 | stack.append(NilFS(device)) 1139 | elif device.superblock_type == 'xfs': 1140 | stack.append(XFS(device)) 1141 | elif device.superblock_type == 'swap': 1142 | stack.append(Swap(device)) 1143 | else: 1144 | err = UnsupportedSuperblock(device=device) 1145 | if device.superblock_type is None: 1146 | progress.bail('Unrecognised superblock', err) 1147 | else: 1148 | progress.bail( 1149 | 'Unsupported superblock type: {}' 1150 | .format(err.device.superblock_type), err) 1151 | 1152 | # only reached when we ended on a filesystem 1153 | return BlockStack(stack) 1154 | 1155 | 1156 | class SyntheticDevice(BlockDevice): 1157 | def copy_to_physical( 1158 | self, dev_fd, *, shift_by=0, reserved_area=None, other_device=False 1159 | ): 1160 | assert ( 1161 | len(self.data) == self.writable_hdr_size + self.writable_end_size) 1162 | start_data = self.data[:self.writable_hdr_size] 1163 | end_data = self.data[self.writable_hdr_size:] 1164 | wrend_offset = self.writable_hdr_size + self.rz_size + shift_by 1165 | size = self.writable_hdr_size + self.rz_size + self.writable_end_size 1166 | 1167 | if shift_by < 0: 1168 | assert not other_device 1169 | 1170 | # Means we should rotate to the left 1171 | # Update shift_by *after* setting wrend_offset 1172 | shift_by += size 1173 | 1174 | if reserved_area is not None: 1175 | assert shift_by >= reserved_area 1176 | assert wrend_offset >= reserved_area 1177 | 1178 | if not other_device: 1179 | assert 0 <= shift_by < shift_by + self.writable_hdr_size <= size 1180 | if self.writable_end_size != 0: 1181 | assert 0 <= wrend_offset < ( 1182 | wrend_offset + self.writable_end_size) <= size 1183 | 1184 | # Write then read back 1185 | written = os.pwrite( 1186 | dev_fd, start_data, shift_by) 1187 | assert written == self.writable_hdr_size, (written, self.writable_hdr_size) 1188 | assert os.pread(dev_fd, self.writable_hdr_size, shift_by) == start_data 1189 | 1190 | if self.writable_end_size != 0: 1191 | assert os.pwrite( 1192 | dev_fd, end_data, wrend_offset) == self.writable_end_size 1193 | assert os.pread( 1194 | dev_fd, self.writable_end_size, wrend_offset) == end_data 1195 | 1196 | 1197 | class ProgressListener: 1198 | pass 1199 | 1200 | 1201 | class DefaultProgressHandler(ProgressListener): 1202 | """A progress listener that logs messages and raises exceptions 1203 | """ 1204 | 1205 | def notify(self, msg): 1206 | logging.info(msg) 1207 | 1208 | def bail(self, msg, err): 1209 | logging.error(msg) 1210 | raise err 1211 | 1212 | 1213 | class CLIProgressHandler(ProgressListener): 1214 | """A progress listener that prints messages and exits on error 1215 | """ 1216 | 1217 | def notify(self, msg): 1218 | print(msg) 1219 | 1220 | def bail(self, msg, err): 1221 | print(msg, file=sys.stderr) 1222 | sys.exit(2) 1223 | 1224 | 1225 | @contextlib.contextmanager 1226 | def synth_device(writable_hdr_size, rz_size, writable_end_size=0): 1227 | writable_sectors = bytes_to_sector(writable_hdr_size) 1228 | wrend_sectors = bytes_to_sector(writable_end_size) 1229 | rz_sectors = bytes_to_sector(rz_size) 1230 | wrend_sectors_offset = writable_sectors + rz_sectors 1231 | 1232 | with contextlib.ExitStack() as st: 1233 | imgf = st.enter_context(tempfile.NamedTemporaryFile(suffix='.img')) 1234 | imgf.truncate(writable_hdr_size + writable_end_size) 1235 | 1236 | lo_dev = subprocess.check_output( 1237 | 'losetup -f --show --'.split() + [imgf.name] 1238 | ).rstrip().decode('ascii') 1239 | assert lo_dev.startswith('/'), lo_dv 1240 | st.callback( 1241 | lambda: quiet_call('losetup -d'.split() + [lo_dev])) 1242 | rozeros_devname = 'rozeros-{}'.format(uuid.uuid1()) 1243 | synth_devname = 'synthetic-{}'.format(uuid.uuid1()) 1244 | synth_devpath = '/dev/mapper/' + synth_devname 1245 | 1246 | # The readonly flag is ignored when stacked under a linear 1247 | # target, so the use of an intermediate device does not bring 1248 | # the expected benefit. This forces us to use the 'error' 1249 | # target to catch writes that are out of bounds. 1250 | # LVM will ignore read errors in the discovery phase (we hide 1251 | # the output), and will fail on write errors appropriately. 1252 | mk_dm( 1253 | rozeros_devname, 1254 | '0 {rz_sectors} error\n' 1255 | .format( 1256 | rz_sectors=rz_sectors), 1257 | readonly=True, 1258 | exit_stack=st) 1259 | dm_table_format = ( 1260 | '0 {writable_sectors} linear {lo_dev} 0\n' 1261 | '{writable_sectors} {rz_sectors} linear {rozeros_devpath} 0\n') 1262 | if writable_end_size: 1263 | dm_table_format += ( 1264 | '{wrend_sectors_offset} {wrend_sectors} ' 1265 | 'linear {lo_dev} {writable_sectors}\n') 1266 | mk_dm( 1267 | synth_devname, 1268 | dm_table_format.format( 1269 | writable_sectors=writable_sectors, lo_dev=lo_dev, 1270 | rz_sectors=rz_sectors, wrend_sectors=wrend_sectors, 1271 | wrend_sectors_offset=wrend_sectors_offset, 1272 | rozeros_devpath='/dev/mapper/' + rozeros_devname), 1273 | readonly=False, 1274 | exit_stack=st) 1275 | 1276 | synth = SyntheticDevice(synth_devpath) 1277 | yield synth 1278 | 1279 | data = imgf.read() 1280 | assert len(data) == writable_hdr_size + writable_end_size 1281 | 1282 | # Expose the data outside of the with statement 1283 | synth.data = data 1284 | synth.rz_size = rz_size 1285 | synth.writable_hdr_size = writable_hdr_size 1286 | synth.writable_end_size = writable_end_size 1287 | 1288 | 1289 | def rotate_aug(aug, forward, size): 1290 | segment_count = aug.get_int('$lv/segment_count') 1291 | pe_sectors = aug.get_int('$vg/extent_size') 1292 | extent_total = 0 1293 | 1294 | aug.incr('$lv/segment_count') 1295 | 1296 | # checking all segments are linear 1297 | for i in range(1, segment_count + 1): 1298 | assert aug.get( 1299 | '$lv/segment{}/dict/type/str'.format(i)) == 'striped' 1300 | assert aug.get_int( 1301 | '$lv/segment{}/dict/stripe_count'.format(i)) == 1 1302 | extent_total += aug.get_int( 1303 | '$lv/segment{}/dict/extent_count'.format(i)) 1304 | 1305 | assert extent_total * pe_sectors == bytes_to_sector(size) 1306 | assert extent_total > 1 1307 | 1308 | if forward: 1309 | # Those definitions can't be factored out, 1310 | # because we move nodes and the vars would follow 1311 | aug.defvar('first', '$lv/segment1/dict') 1312 | # shifting segments 1313 | for i in range(2, segment_count + 1): 1314 | aug.decr('$lv/segment{}/dict/start_extent'.format(i)) 1315 | 1316 | # shrinking first segment by one PE 1317 | aug.decr('$first/extent_count') 1318 | 1319 | # inserting new segment at the end 1320 | aug.insert( 1321 | '$lv/segment{}'.format(segment_count), 1322 | 'segment{}'.format(segment_count + 1), 1323 | before=False) 1324 | aug.set_int( 1325 | '$lv/segment{}/dict/start_extent'.format(segment_count + 1), 1326 | extent_total - 1) 1327 | aug.defvar('last', '$lv/segment{}/dict'.format(segment_count + 1)) 1328 | aug.set_int('$last/extent_count', 1) 1329 | aug.set('$last/type/str', 'striped') 1330 | aug.set_int('$last/stripe_count', 1) 1331 | 1332 | # repossessing the first segment's first PE 1333 | aug.set( 1334 | '$last/stripes/list/1/str', 1335 | aug.get('$first/stripes/list/1/str')) 1336 | aug.set_int( 1337 | '$last/stripes/list/2', 1338 | aug.get_int('$first/stripes/list/2')) 1339 | aug.incr('$first/stripes/list/2') 1340 | 1341 | # Cleaning up an empty first PE 1342 | if aug.get_int('$first/extent_count') == 0: 1343 | aug.remove('$lv/segment1') 1344 | for i in range(2, segment_count + 2): 1345 | aug.rename('$lv/segment{}'.format(i), 'segment{}'.format(i - 1)) 1346 | aug.decr('$lv/segment_count') 1347 | else: 1348 | # shifting segments 1349 | for i in range(segment_count, 0, -1): 1350 | aug.incr('$lv/segment{}/dict/start_extent'.format(i)) 1351 | aug.rename('$lv/segment{}'.format(i), 'segment{}'.format(i + 1)) 1352 | aug.defvar('last', '$lv/segment{}/dict'.format(segment_count + 1)) 1353 | 1354 | # shrinking last segment by one PE 1355 | aug.decr('$last/extent_count') 1356 | last_count = aug.get_int('$last/extent_count') 1357 | 1358 | # inserting new segment at the beginning 1359 | aug.insert('$lv/segment2', 'segment1') 1360 | aug.set_int('$lv/segment1/dict/start_extent', 0) 1361 | aug.defvar('first', '$lv/segment1/dict') 1362 | aug.set_int('$first/extent_count', 1) 1363 | aug.set('$first/type/str', 'striped') 1364 | aug.set_int('$first/stripe_count', 1) 1365 | 1366 | # repossessing the last segment's last PE 1367 | aug.set( 1368 | '$first/stripes/list/1/str', 1369 | aug.get('$last/stripes/list/1/str')) 1370 | aug.set_int( 1371 | '$first/stripes/list/2', 1372 | aug.get_int('$last/stripes/list/2') + last_count) 1373 | 1374 | # Cleaning up an empty last PE 1375 | if last_count == 0: 1376 | aug.remove('$lv/segment{}'.format(segment_count + 1)) 1377 | aug.decr('$lv/segment_count') 1378 | 1379 | 1380 | def rotate_lv(*, device, size, debug, forward): 1381 | """Rotate a logical volume by a single PE. 1382 | 1383 | If forward: 1384 | Move the first physical extent of an LV to the end 1385 | else: 1386 | Move the last physical extent of a LV to the start 1387 | 1388 | then poke LVM to refresh the mapping. 1389 | """ 1390 | 1391 | import augeas 1392 | class Augeas(augeas.Augeas): 1393 | def get_int(self, key): 1394 | return int(self.get(key + '/int')) 1395 | 1396 | def set_int(self, key, val): 1397 | return self.set(key + '/int', '%d' % val) 1398 | 1399 | def incr(self, key, by=1): 1400 | orig = self.get_int(key) 1401 | self.set_int(key, orig + by) 1402 | 1403 | def decr(self, key): 1404 | self.incr(key, by=-1) 1405 | 1406 | lv_info = subprocess.check_output( 1407 | 'lvm lvs --noheadings --rows --units=b --nosuffix ' 1408 | '-o vg_name,vg_uuid,lv_name,lv_uuid,lv_attr --'.split() 1409 | + [device.devpath], universal_newlines=True).splitlines() 1410 | vgname, vg_uuid, lvname, lv_uuid, lv_attr = (fi.lstrip() for fi in lv_info) 1411 | active = lv_attr[4] == 'a' 1412 | 1413 | # Make sure the volume isn't in use by unmapping it 1414 | quiet_call( 1415 | ['lvm', 'lvchange', '-an', '--', '{}/{}'.format(vgname, lvname)]) 1416 | 1417 | with tempfile.TemporaryDirectory(suffix='.blocks') as tdname: 1418 | vgcfgname = tdname + '/vg.cfg' 1419 | print('Loading LVM metadata... ', end='', flush=True) 1420 | quiet_call( 1421 | ['lvm', 'vgcfgbackup', '--file', vgcfgname, '--', vgname]) 1422 | aug = Augeas( 1423 | loadpath=pkg_resources.resource_filename('blocks', 'augeas'), 1424 | root='/dev/null', 1425 | flags=augeas.Augeas.NO_MODL_AUTOLOAD | augeas.Augeas.SAVE_NEWFILE) 1426 | vgcfg = open(vgcfgname) 1427 | vgcfg_orig = vgcfg.read() 1428 | aug.set('/raw/vgcfg', vgcfg_orig) 1429 | 1430 | aug.text_store('LVM.lns', '/raw/vgcfg', '/vg') 1431 | print('ok') 1432 | 1433 | # There is no easy way to quote for XPath, so whitelist 1434 | assert all(ch in ASCII_ALNUM_WHITELIST for ch in vgname), vgname 1435 | assert all(ch in ASCII_ALNUM_WHITELIST for ch in lvname), lvname 1436 | 1437 | aug.defvar('vg', '/vg/{}/dict'.format(vgname)) 1438 | assert aug.get('$vg/id/str') == vg_uuid 1439 | aug.defvar('lv', '$vg/logical_volumes/dict/{}/dict'.format(lvname)) 1440 | assert aug.get('$lv/id/str') == lv_uuid 1441 | 1442 | rotate_aug(aug, forward, size) 1443 | aug.text_retrieve('LVM.lns', '/raw/vgcfg', '/vg', '/raw/vgcfg.new') 1444 | open(vgcfgname + '.new', 'w').write(aug.get('/raw/vgcfg.new')) 1445 | rotate_aug(aug, not forward, size) 1446 | aug.text_retrieve('LVM.lns', '/raw/vgcfg', '/vg', '/raw/vgcfg.backagain') 1447 | open(vgcfgname + '.backagain', 'w').write(aug.get('/raw/vgcfg.backagain')) 1448 | 1449 | if debug: 1450 | print('CHECK STABILITY') 1451 | subprocess.call( 1452 | ['git', '--no-pager', 'diff', '--no-index', '--patience', '--color-words', 1453 | '--', vgcfgname, vgcfgname + '.backagain']) 1454 | if forward: 1455 | print('CHECK CORRECTNESS (forward)') 1456 | else: 1457 | print('CHECK CORRECTNESS (backward)') 1458 | subprocess.call( 1459 | ['git', '--no-pager', 'diff', '--no-index', '--patience', '--color-words', 1460 | '--', vgcfgname, vgcfgname + '.new']) 1461 | 1462 | if forward: 1463 | print( 1464 | 'Rotating the second extent to be the first... ', 1465 | end='', flush=True) 1466 | else: 1467 | print( 1468 | 'Rotating the last extent to be the first... ', 1469 | end='', flush=True) 1470 | quiet_call( 1471 | ['lvm', 'vgcfgrestore', '--file', vgcfgname + '.new', '--', vgname]) 1472 | # Make sure LVM updates the mapping, this is pretty critical 1473 | quiet_call( 1474 | ['lvm', 'lvchange', '--refresh', '--', '{}/{}'.format(vgname, lvname)]) 1475 | if active: 1476 | quiet_call( 1477 | ['lvm', 'lvchange', '-ay', '--', '{}/{}'.format(vgname, lvname)]) 1478 | print('ok') 1479 | 1480 | 1481 | def make_bcache_sb(bsb_size, data_size, join): 1482 | with synth_device(bsb_size, data_size) as synth_bdev: 1483 | cmd = ['make-bcache', '--bdev', '--data_offset', 1484 | '%d' % bytes_to_sector(bsb_size), synth_bdev.devpath] 1485 | if join is not None: 1486 | cmd[1:1] = ['--cset-uuid', join] 1487 | quiet_call(cmd) 1488 | bcache_backing = BCacheBacking(synth_bdev) 1489 | bcache_backing.read_superblock() 1490 | assert bcache_backing.offset == bsb_size 1491 | return synth_bdev 1492 | 1493 | 1494 | def lv_to_bcache(device, debug, progress, join): 1495 | pe_size = int(subprocess.check_output( 1496 | 'lvm lvs --noheadings --rows --units=b --nosuffix ' 1497 | '-o vg_extent_size --'.split() 1498 | + [device.devpath], universal_newlines=True)) 1499 | 1500 | assert device.size % pe_size == 0 1501 | data_size = device.size - pe_size 1502 | 1503 | block_stack = get_block_stack(device, progress) 1504 | block_stack.read_superblocks() 1505 | block_stack.stack_reserve_end_area(data_size, progress) 1506 | block_stack.deactivate() 1507 | del block_stack 1508 | 1509 | with device.open_excl_ctx() as dev_fd: 1510 | synth_bdev = make_bcache_sb(pe_size, data_size, join) 1511 | print('Copying the bcache superblock... ', end='', flush=True) 1512 | synth_bdev.copy_to_physical(dev_fd, shift_by=-pe_size) 1513 | print('ok') 1514 | del dev_fd 1515 | 1516 | rotate_lv( 1517 | device=device, size=device.size, debug=debug, forward=False) 1518 | 1519 | 1520 | def luks_to_bcache(device, debug, progress, join): 1521 | luks = LUKS(device) 1522 | luks.deactivate() 1523 | dev_fd = device.open_excl() 1524 | luks.read_superblock() 1525 | luks.read_superblock_ll(dev_fd) 1526 | # The smallest and most compatible bcache offset 1527 | shift_by = 512*16 1528 | assert luks.sb_end + shift_by <= luks.offset 1529 | data_size = device.size - shift_by 1530 | synth_bdev = make_bcache_sb(shift_by, data_size, join) 1531 | 1532 | # XXX not atomic 1533 | print('Shifting and editing the LUKS superblock... ', end='', flush=True) 1534 | luks.shift_sb(dev_fd, shift_by=shift_by) 1535 | print('ok') 1536 | 1537 | print('Copying the bcache superblock... ', end='', flush=True) 1538 | synth_bdev.copy_to_physical(dev_fd) 1539 | os.close(dev_fd) 1540 | del dev_fd 1541 | print('ok') 1542 | 1543 | 1544 | def part_to_bcache(device, debug, progress, join): 1545 | # Detect the alignment parted would use? 1546 | # I don't think it can be greater than 1MiB, in which case 1547 | # there is no need. 1548 | bsb_size = 1024**2 1549 | data_size = device.size 1550 | import _ped 1551 | 1552 | ptable, part_start = device.ptable_context() 1553 | ptype = ptable.parted_disk.getPartitionBySector( 1554 | bytes_to_sector(part_start)).type 1555 | if ptype & _ped.PARTITION_LOGICAL: 1556 | progress.bail( 1557 | 'Converting logical partitions is not supported.' 1558 | ' Please convert this disk to GPT.', UnsupportedLayout()) 1559 | assert ptype == _ped.PARTITION_NORMAL, ptype 1560 | ptable.reserve_space_before(part_start, bsb_size, progress) 1561 | part_start1 = part_start - bsb_size 1562 | 1563 | import _ped 1564 | write_part = ptable.parted_disk.getPartitionBySector(part_start1 // 512) 1565 | 1566 | if write_part.type == _ped.PARTITION_NORMAL: 1567 | write_offset = part_start1 - (512 * write_part.geometry.start) 1568 | dev_fd = os.open(write_part.path, os.O_SYNC|os.O_RDWR|os.O_EXCL) 1569 | elif write_part.type == _ped.PARTITION_FREESPACE: 1570 | # XXX Can't open excl if one of the partitions is used by dm, apparently 1571 | dev_fd = ptable.device.open_excl() 1572 | write_offset = part_start1 1573 | else: 1574 | print( 1575 | 'Can\'t write outside of a normal partition (marked {})' 1576 | .format(_ped.partition_type_get_name(write_part.type)), 1577 | file=sys.stderr) 1578 | return 1 1579 | 1580 | synth_bdev = make_bcache_sb(bsb_size, data_size, join) 1581 | print('Copying the bcache superblock... ', end='', flush=True) 1582 | synth_bdev.copy_to_physical( 1583 | dev_fd, shift_by=write_offset, other_device=True) 1584 | os.close(dev_fd) 1585 | del dev_fd 1586 | print('ok') 1587 | 1588 | # Check the partition we're about to convert isn't in use either, 1589 | # otherwise the partition table couldn't be reloaded. 1590 | with device.open_excl_ctx(): 1591 | pass 1592 | 1593 | print( 1594 | 'Shifting partition to start on the bcache superblock... ', 1595 | end='', flush=True) 1596 | ptable.shift_left(part_start, part_start1) 1597 | print('ok') 1598 | device.reset_size() 1599 | 1600 | 1601 | SIZE_RE = re.compile(r'^(\d+)([bkmgtpe])?\Z') 1602 | 1603 | 1604 | def parse_size_arg(size): 1605 | match = SIZE_RE.match(size.lower()) 1606 | if not match: 1607 | raise argparse.ArgumentTypeError( 1608 | 'Size must be a decimal integer ' 1609 | 'and a one-character unit suffix (bkmgtpe)') 1610 | val = int(match.group(1)) 1611 | unit = match.group(2) 1612 | if unit is None: 1613 | unit = 'b' 1614 | # reserving uppercase in case decimal units are needed 1615 | return val * 1024**'bkmgtpe'.find(unit) 1616 | 1617 | 1618 | def main(): 1619 | try: 1620 | assert False 1621 | except AssertionError: 1622 | pass 1623 | else: 1624 | print('Assertions need to be enabled', file=sys.stderr) 1625 | return 2 1626 | 1627 | parser = argparse.ArgumentParser() 1628 | parser.add_argument('--debug', action='store_true') 1629 | commands = parser.add_subparsers(dest='command', metavar='command') 1630 | 1631 | sp_to_lvm = commands.add_parser( 1632 | 'to-lvm', aliases=['lvmify'], 1633 | help='Convert to LVM') 1634 | sp_to_lvm.add_argument('device') 1635 | vg_flags = sp_to_lvm.add_mutually_exclusive_group() 1636 | vg_flags.add_argument('--vg-name', dest='vgname', type=str) 1637 | vg_flags.add_argument('--join', metavar='VG-NAME-OR-UUID') 1638 | sp_to_lvm.set_defaults(action=cmd_to_lvm) 1639 | 1640 | sp_to_bcache = commands.add_parser( 1641 | 'to-bcache', 1642 | help='Convert to bcache') 1643 | sp_to_bcache.add_argument('device') 1644 | sp_to_bcache.add_argument('--join', metavar='CSET-UUID') 1645 | sp_to_bcache.add_argument('--maintboot', action='store_true') 1646 | sp_to_bcache.set_defaults(action=cmd_to_bcache) 1647 | 1648 | sp_maintboot_impl = commands.add_parser('maintboot-impl') 1649 | sp_maintboot_impl.set_defaults(action=cmd_maintboot_impl) 1650 | 1651 | # Undoes an lv to bcache conversion; useful to migrate from the GPT 1652 | # format to the bcache-offset format. 1653 | # No help, keep this undocumented for now 1654 | sp_rotate = commands.add_parser( 1655 | 'rotate') 1656 | #help='Rotate LV contents to start at the second PE') 1657 | sp_rotate.add_argument('device') 1658 | sp_rotate.set_defaults(action=cmd_rotate) 1659 | 1660 | sp_resize = commands.add_parser('resize') 1661 | sp_resize.add_argument('device') 1662 | sp_resize.add_argument( 1663 | '--resize-device', action='store_true', 1664 | help='Resize the device, not just the contents.' 1665 | ' The device must be a partition or a logical volume.') 1666 | sp_resize.add_argument( 1667 | 'newsize', type=parse_size_arg, 1668 | help='new size in byte units;' 1669 | ' bkmgtpe suffixes are accepted, in powers of 1024 units') 1670 | sp_resize.set_defaults(action=cmd_resize) 1671 | 1672 | # Give help when no subcommand is given 1673 | if not sys.argv[1:]: 1674 | parser.print_help() 1675 | return 1676 | 1677 | args = parser.parse_args() 1678 | return args.action(args) 1679 | 1680 | 1681 | def cmd_resize(args): 1682 | device = BlockDevice(args.device) 1683 | newsize = args.newsize 1684 | resize_device = args.resize_device 1685 | debug = args.debug 1686 | progress = CLIProgressHandler() 1687 | 1688 | block_stack = get_block_stack(device, progress) 1689 | 1690 | device_delta = newsize - device.size 1691 | 1692 | if device_delta > 0 and resize_device: 1693 | device.dev_resize(newsize, shrink=False) 1694 | # May have been rounded up for the sake of partition alignment 1695 | # LVM rounds up as well (and its LV metadata uses PE units) 1696 | newsize = device.size 1697 | 1698 | block_stack.read_superblocks() 1699 | assert block_stack.total_data_size <= device.size 1700 | data_delta = newsize - block_stack.total_data_size 1701 | block_stack.stack_resize(newsize, shrink=data_delta < 0, progress=progress) 1702 | 1703 | if device_delta < 0 and resize_device: 1704 | tds = block_stack.total_data_size 1705 | # LVM should be able to reload in-use devices, 1706 | # but the kernel's partition handling can't. 1707 | if device.is_partition: 1708 | block_stack.deactivate() 1709 | del block_stack 1710 | device.dev_resize(tds, shrink=True) 1711 | 1712 | 1713 | def cmd_rotate(args): 1714 | device = BlockDevice(args.device) 1715 | debug = args.debug 1716 | progress = CLIProgressHandler() 1717 | 1718 | pe_size = int(subprocess.check_output( 1719 | 'lvm lvs --noheadings --rows --units=b --nosuffix ' 1720 | '-o vg_extent_size --'.split() 1721 | + [device.devpath], universal_newlines=True)) 1722 | 1723 | if device.superblock_at(pe_size) is None: 1724 | print('No superblock on the second PE, exiting', file=sys.stderr) 1725 | return 1 1726 | 1727 | rotate_lv( 1728 | device=device, size=device.size, debug=debug, forward=True) 1729 | 1730 | 1731 | def cmd_to_bcache(args): 1732 | device = BlockDevice(args.device) 1733 | progress = CLIProgressHandler() 1734 | 1735 | if device.has_bcache_superblock: 1736 | print( 1737 | 'Device {} already has a bcache super block.' 1738 | .format(device.devpath), file=sys.stderr) 1739 | return 1 1740 | 1741 | BCacheReq.require(progress) 1742 | 1743 | if device.is_partition: 1744 | impl = part_to_bcache 1745 | elif device.is_lv: 1746 | impl = lv_to_bcache 1747 | elif device.superblock_type == 'crypto_LUKS': 1748 | impl = luks_to_bcache 1749 | else: 1750 | print( 1751 | 'Device {} is not a partition, a logical volume, or a LUKS volume' 1752 | .format(device.devpath), 1753 | file=sys.stderr) 1754 | return 1 1755 | 1756 | if args.maintboot: 1757 | return call_maintboot( 1758 | device, 'to-bcache', debug=args.debug, join=args.join) 1759 | else: 1760 | return impl( 1761 | device=device, debug=args.debug, progress=progress, join=args.join) 1762 | 1763 | 1764 | def call_maintboot(device, command, **args): 1765 | fsuuid = Filesystem(device).fsuuid 1766 | if not fsuuid: 1767 | print( 1768 | 'Device {} doesn\'t have a UUID'.format(device.devpath), 1769 | file=sys.stderr) 1770 | return 1 1771 | encoded = urllib.parse.quote(json.dumps(dict( 1772 | command=command, device=fsuuid, **args))) 1773 | subprocess.check_call( 1774 | ['maintboot', '--pkgs'] 1775 | + 'python3-blocks util-linux dash mount base-files libc-bin' 1776 | ' nilfs-tools reiserfsprogs xfsprogs e2fsprogs btrfs-tools' 1777 | ' lvm2 cryptsetup-bin bcache-tools'.split() 1778 | + ['--initscript', pkg_resources.resource_filename( 1779 | 'blocks', 'maintboot.init')] 1780 | + ['--append', 'BLOCKS_ARGS=' + encoded]) 1781 | 1782 | 1783 | def cmd_maintboot_impl(args): 1784 | kargs = json.loads( 1785 | urllib.parse.unquote(os.environ['BLOCKS_ARGS']), 1786 | object_hook=lambda args: types.SimpleNamespace(**args)) 1787 | assert kargs.command == 'to-bcache' 1788 | 1789 | # wait for devices to come up (30s max) 1790 | subprocess.check_call(['udevadm', 'settle', '--timeout=30']) 1791 | # activate lvm volumes 1792 | subprocess.check_call(['lvm', 'vgchange', '-ay']) 1793 | 1794 | kargs.device = BlockDevice.by_uuid(kargs.device).devpath 1795 | kargs.maintboot = False 1796 | return cmd_to_bcache(kargs) 1797 | 1798 | 1799 | def cmd_to_lvm(args): 1800 | device = BlockDevice(args.device) 1801 | debug = args.debug 1802 | progress = CLIProgressHandler() 1803 | 1804 | if device.superblock_type == 'LVM2_member': 1805 | print( 1806 | 'Already a physical volume', file=sys.stderr) 1807 | return 1 1808 | 1809 | LVMReq.require(progress) 1810 | 1811 | if args.join is not None: 1812 | vg_info = subprocess.check_output( 1813 | 'lvm vgs --noheadings --rows --units=b --nosuffix ' 1814 | '-o vg_name,vg_uuid,vg_extent_size --'.split() 1815 | + [args.join], universal_newlines=True).splitlines() 1816 | join_name, join_uuid, pe_size = (fi.lstrip() for fi in vg_info) 1817 | # Pick something unique, temporary until vgmerge 1818 | vgname = uuid.uuid1().hex 1819 | pe_size = int(pe_size) 1820 | elif args.vgname is not None: 1821 | # Check no VG with that name exists? 1822 | # No real need, vgrename uuid newname would fix any collision 1823 | vgname = args.vgname 1824 | pe_size = LVM_PE_SIZE 1825 | else: 1826 | vgname = 'vg.' + os.path.basename(device.devpath) 1827 | pe_size = LVM_PE_SIZE 1828 | 1829 | assert vgname 1830 | assert all(ch in ASCII_ALNUM_WHITELIST for ch in vgname) 1831 | 1832 | assert device.size % 512 == 0 1833 | 1834 | block_stack = get_block_stack(device, progress) 1835 | 1836 | if block_stack.fslabel: 1837 | lvname = block_stack.fslabel 1838 | else: 1839 | lvname = os.path.basename(device.devpath) 1840 | if not all(ch in ASCII_ALNUM_WHITELIST for ch in lvname): 1841 | lvname = 'lv1' 1842 | 1843 | pe_sectors = bytes_to_sector(pe_size) 1844 | # -1 because we reserve pe_size for the lvm label and one metadata area 1845 | pe_count = device.size // pe_size - 1 1846 | # The position of the moved pe 1847 | pe_newpos = pe_count * pe_size 1848 | 1849 | # bootloader embedding area, sector units 1850 | assert pe_size >= 4096, pe_size 1851 | # default to 1M, to match the bootloader area on a 1M-aligned ptable 1852 | ba_start = 2048 1853 | ba_size = 2048 1854 | 1855 | if debug: 1856 | print( 1857 | 'pe {} pe_newpos {} devsize {}' 1858 | .format(pe_size, pe_newpos, device.size)) 1859 | 1860 | block_stack.read_superblocks() 1861 | block_stack.stack_reserve_end_area(pe_newpos, progress) 1862 | 1863 | fsuuid = block_stack.topmost.fsuuid 1864 | block_stack.deactivate() 1865 | del block_stack 1866 | 1867 | dev_fd = device.open_excl() 1868 | print( 1869 | 'Copying {} bytes from pos 0 to pos {}... ' 1870 | .format(pe_size, pe_newpos), 1871 | end='', flush=True) 1872 | pe_data = os.pread(dev_fd, pe_size, 0) 1873 | assert len(pe_data) == pe_size 1874 | wr_len = os.pwrite(dev_fd, pe_data, pe_newpos) 1875 | assert wr_len == pe_size 1876 | print('ok') 1877 | 1878 | print('Preparing LVM metadata... ', end='', flush=True) 1879 | 1880 | # The changes so far (fs resize, possibly an fsck, and the copy) 1881 | # should have no user-visible effects. 1882 | 1883 | # Create a virtual device to do the lvm setup 1884 | with contextlib.ExitStack() as st: 1885 | synth_pv = st.enter_context( 1886 | synth_device(pe_size, device.size - pe_size)) 1887 | cfgf = st.enter_context( 1888 | tempfile.NamedTemporaryFile( 1889 | suffix='.vgcfg', mode='w', encoding='ascii', 1890 | delete=not debug)) 1891 | 1892 | pv_uuid = uuid.uuid1() 1893 | vg_uuid = uuid.uuid1() 1894 | lv_uuid = uuid.uuid1() 1895 | 1896 | cfgf.write(textwrap.dedent( 1897 | ''' 1898 | contents = "Text Format Volume Group" 1899 | version = 1 1900 | 1901 | {vgname} {{ 1902 | id = "{vg_uuid}" 1903 | seqno = 0 1904 | status = ["RESIZEABLE", "READ", "WRITE"] 1905 | extent_size = {pe_sectors} 1906 | max_lv = 0 1907 | max_pv = 0 1908 | 1909 | physical_volumes {{ 1910 | pv0 {{ 1911 | id = "{pv_uuid}" 1912 | status = ["ALLOCATABLE"] 1913 | 1914 | pe_start = {pe_sectors} 1915 | pe_count = {pe_count} 1916 | ba_start = {ba_start} 1917 | ba_size = {ba_size} 1918 | }} 1919 | }} 1920 | logical_volumes {{ 1921 | {lvname} {{ 1922 | id = "{lv_uuid}" 1923 | status = ["READ", "WRITE", "VISIBLE"] 1924 | segment_count = 2 1925 | 1926 | segment1 {{ 1927 | start_extent = 0 1928 | extent_count = 1 1929 | type = "striped" 1930 | stripe_count = 1 # linear 1931 | stripes = [ 1932 | "pv0", {pe_count_pred} 1933 | ] 1934 | }} 1935 | segment2 {{ 1936 | start_extent = 1 1937 | extent_count = {pe_count_pred} 1938 | type = "striped" 1939 | stripe_count = 1 # linear 1940 | stripes = [ 1941 | "pv0", 0 1942 | ] 1943 | }} 1944 | }} 1945 | }} 1946 | }} 1947 | '''.format( 1948 | vgname=vgname, 1949 | lvname=lvname, 1950 | pe_sectors=pe_sectors, 1951 | pv_uuid=pv_uuid, 1952 | vg_uuid=vg_uuid, 1953 | lv_uuid=lv_uuid, 1954 | pe_count=pe_count, 1955 | pe_count_pred=pe_count - 1, 1956 | ba_start=ba_start, 1957 | ba_size=ba_size, 1958 | ))) 1959 | cfgf.flush() 1960 | 1961 | # Prevent the next two commands from scanning every device (slow), 1962 | # we already know lvm should write only to the synthetic pv. 1963 | # Also work in the presence of a broken udev; udev inside uml 1964 | # appears to be fragile. 1965 | lvm_cfg = ('--config=' 1966 | 'devices {{ filter=["a/^{synth_re}$/", "r/.*/"] }}' 1967 | 'activation {{ verify_udev_operations = 1 }}' 1968 | .format(synth_re=re.escape(synth_pv.devpath))) 1969 | 1970 | quiet_call( 1971 | ['lvm', 'pvcreate', lvm_cfg, '--restorefile', cfgf.name, 1972 | '--uuid', str(pv_uuid), '--zero', 'y', '--', 1973 | synth_pv.devpath]) 1974 | quiet_call( 1975 | ['lvm', 'vgcfgrestore', lvm_cfg, '--file', cfgf.name, '--', vgname]) 1976 | print('ok') # after 'Preparing LVM metadata' 1977 | 1978 | # Recovery: copy back the PE we had moved to the end of the device. 1979 | print( 1980 | 'If the next stage is interrupted, it can be reverted with:\n' 1981 | ' dd if={devpath} of={devpath} bs={pe_size} count=1 skip={pe_count} conv=notrunc' 1982 | .format( 1983 | devpath=device.devpath, pe_size=pe_size, pe_count=pe_count)) 1984 | 1985 | print('Installing LVM metadata... ', end='', flush=True) 1986 | # This had better be atomic 1987 | # Though technically, only physical sector writes are guaranteed atomic 1988 | synth_pv.copy_to_physical(dev_fd) 1989 | print('ok') 1990 | os.close(dev_fd) 1991 | del dev_fd 1992 | 1993 | print('LVM conversion successful!') 1994 | if args.join is not None: 1995 | quiet_call( 1996 | ['lvm', 'vgmerge', '--', join_name, vgname]) 1997 | vgname = join_name 1998 | if False: 1999 | print('Enable the volume group with\n' 2000 | ' sudo lvm vgchange -ay -- {}'.format(vgname)) 2001 | elif False: 2002 | print('Enable the logical volume with\n' 2003 | ' sudo lvm lvchange -ay -- {}/{}'.format(vgname, lvname)) 2004 | else: 2005 | print('Volume group name: {}\n' 2006 | 'Logical volume name: {}\n' 2007 | 'Filesystem uuid: {}' 2008 | .format(vgname, lvname, fsuuid)) 2009 | 2010 | 2011 | def script_main(): 2012 | sys.exit(main()) 2013 | 2014 | 2015 | if __name__ == '__main__': 2016 | script_main() 2017 | 2018 | --------------------------------------------------------------------------------