├── .gitignore ├── ssh-keys ├── routes.awk ├── ips.awk ├── configuration.nix ├── kexec.nix ├── README.md ├── emergency └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | /result 2 | /emergency_ips 3 | /emergency_routes 4 | /emergency_nameservers 5 | -------------------------------------------------------------------------------- /ssh-keys: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Outputs all SSH user keys on the current system. 4 | 5 | while IFS= read -r home; do 6 | [ -f "${home}/.ssh/authorized_keys" ] && cat "${home}/.ssh/authorized_keys" 7 | done < <(getent passwd | cut -d':' -f6 | sort | uniq) 8 | 9 | [ -d /etc/ssh/authorized_keys.d ] && find /etc/ssh/authorized_keys.d -type f -exec cat '{}' \; 10 | -------------------------------------------------------------------------------- /routes.awk: -------------------------------------------------------------------------------- 1 | # Pipe `ip r` or `ip -6 r` into this script 2 | # The script returns one route per line. 3 | # Each line consists of three parts (;-delimited): 4 | # - The first part is the route target 5 | # - The second part is the `via` part of the route 6 | # - The third part is the `dev` part of the route 7 | # Both via and dev may be empty. 8 | 9 | { 10 | if ($1 == "unreachable" || $1 ~ /^fe80::/) 11 | next 12 | 13 | # Parse fields 14 | isVia = 0 15 | isDev = 0 16 | via = "" 17 | dev = "" 18 | for (i=1;i<=NF;i++) { 19 | if (isVia == 1) 20 | via = $i 21 | if (isDev == 1) 22 | dev = $i 23 | # State 24 | if ($i == "via") 25 | isVia = 1 26 | else 27 | isVia = 0 28 | if ($i == "dev") 29 | isDev = 1 30 | else 31 | isDev = 0 32 | } 33 | 34 | # No loopback routes 35 | if (dev == "lo") 36 | next 37 | 38 | print $1";"via";"dev 39 | } 40 | -------------------------------------------------------------------------------- /ips.awk: -------------------------------------------------------------------------------- 1 | # Pipe `ip a` into this script 2 | # The script returns one line per interface. 3 | # Each line begins with the interface name, and all IPs (v4 and v6, CIDR notation), separated by ;. 4 | 5 | BEGIN { 6 | first = 1; 7 | } 8 | { 9 | if ($1 ~ /[0-9]*:/) { 10 | if ($2 != "lo:") { 11 | inter = substr($2, 1, length($2)-1); 12 | printedInt = 0; 13 | } 14 | } 15 | if ($1 ~ /^inet(6)?/) { 16 | scope = 0; 17 | p = 0; 18 | for (i=1;i<=NF;i++) { 19 | if (scope == 1) { 20 | if ($i != "host" && $i != "link") 21 | p = 1; 22 | } 23 | if ($i == "scope") 24 | scope = 1; 25 | else 26 | scope = 0; 27 | } 28 | if (p == 1) { 29 | if (printedInt == 0) { 30 | if (first == 0) 31 | printf "\n"; 32 | printf inter; 33 | printedInt = 1; 34 | first = 0; 35 | } 36 | printf ";"$2; 37 | } 38 | } 39 | } 40 | END { 41 | printf "\n"; 42 | } 43 | -------------------------------------------------------------------------------- /configuration.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | with lib; { 3 | imports = [ 4 | 5 | 6 | 7 | 8 | ./kexec.nix 9 | ]; 10 | 11 | # Slim the package 12 | security.sudo.enable = false; 13 | services.udisks2.enable = false; 14 | networking.firewall.logRefusedConnections = false; 15 | 16 | # User stuff 17 | services.getty.autologinUser = "root"; 18 | users.users.root.initialHashedPassword = ""; 19 | 20 | # Packages 21 | environment.systemPackages = with pkgs; [ gnufdisk tmux vim xfsprogs.bin ]; 22 | 23 | # Kernel stuff 24 | hardware.enableRedistributableFirmware = true; 25 | boot.initrd.availableKernelModules = [ 26 | "ata_piix" 27 | "uhci_hcd" 28 | "ehci_pci" 29 | "ahci" 30 | "virtio_pci" 31 | "sd_mod" 32 | "sr_mod" 33 | "virtio_blk" 34 | ]; 35 | 36 | # Filesystems 37 | boot.supportedFilesystems = [ "vfat" "xfs" "btrfs" ]; 38 | 39 | # Enable ssh 40 | systemd.services.sshd.wantedBy = mkForce [ "multi-user.target" ]; 41 | 42 | # Hostname 43 | networking.hostName = "emergency"; 44 | networking.dhcpcd.extraConfig = '' 45 | noipv4ll 46 | ''; 47 | 48 | # Perform better with low-memory 49 | environment.variables.GC_INITIAL_HEAP_SIZE = "1M"; 50 | boot.kernel.sysctl."vm.overcommit_memory" = "1"; 51 | 52 | # nix stuff 53 | nix = { 54 | buildCores = 0; 55 | gc.automatic = false; 56 | }; 57 | nixpkgs.config.allowUnfree = true; 58 | 59 | services.openssh = { 60 | enable = true; 61 | permitRootLogin = "without-password"; 62 | hostKeys = [ ]; # We will take the keys of the old system 63 | }; 64 | } 65 | -------------------------------------------------------------------------------- /kexec.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, ... }: { 2 | system.build = rec { 3 | image = 4 | pkgs.runCommand "image" { buildInputs = [ pkgs.nukeReferences ]; } '' 5 | mkdir "$out" 6 | cp "${config.system.build.kernel}/bzImage" "$out/kernel" 7 | cp "${config.system.build.netbootRamdisk}/initrd" "$out/initrd" 8 | echo "init=${ 9 | builtins.unsafeDiscardStringContext config.system.build.toplevel 10 | }/init ${toString config.boot.kernelParams}" > "$out/cmdline" 11 | nuke-refs "$out/kernel" 12 | ''; 13 | 14 | kexecScript = pkgs.writeScript "kexec-nixos" '' 15 | #!${pkgs.stdenv.shell} 16 | 17 | export PATH="${pkgs.kexectools}/bin:${pkgs.cpio}/bin:$PATH" 18 | 19 | cd $(mktemp -d) 20 | mkdir initrd 21 | pushd initrd 22 | for i in /etc/ssh/ssh_host_*; do 23 | cat "$i" > "$(basename "$i")" 24 | done 25 | mv /tmp/ip-script . 26 | ${./ssh-keys} > ssh-keys 2>&1 27 | chmod 755 ip-script 28 | find -type f | cpio -o -H newc | gzip -9 > ../extra.gz 29 | popd 30 | cat "${image}/initrd" extra.gz > final.gz 31 | 32 | kexec -l "${image}/kernel" --initrd=final.gz --append="init=${ 33 | builtins.unsafeDiscardStringContext config.system.build.toplevel 34 | }/init ${toString config.boot.kernelParams}" 35 | systemd-run --on-active=2 --timer-property=AccuracySec=100ms $(which kexec) -e 36 | ''; 37 | }; 38 | 39 | boot.initrd.postMountCommands = '' 40 | mkdir -p /mnt-root/etc/ssh /mnt-root/root/.ssh 41 | umask 077 42 | for i in /ssh_host_*; do 43 | cat "$i" > /mnt-root/etc/ssh/"$i" 44 | done 45 | cat /ssh-keys > /mnt-root/root/.ssh/authorized_keys 46 | cat /ip-script > /mnt-root/kexec-ips 47 | chmod +x /mnt-root/kexec-ips 48 | ''; 49 | 50 | networking.localCommands = '' 51 | export PATH="${pkgs.iproute}/bin:$PATH" 52 | /kexec-ips 53 | ''; 54 | 55 | system.build.kexec_tarball = 56 | pkgs.callPackage { 57 | storeContents = [{ 58 | object = config.system.build.kexecScript; 59 | symlink = "/kexec"; 60 | }]; 61 | contents = [ ]; 62 | }; 63 | } 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # emergency-kexec 2 | 3 | Okay, your system is completely broken, and you need to umount `/` or something like that. 4 | What do you do? 5 | 6 | ## Motivation 7 | 8 | One of our servers had a broken root filesystem (btrfs, don't judge me). 9 | Online recovery was not possible, so the filesystem needed to be unmounted which is not possible for the root fs. 10 | Additionally, as errors were detected, the kernel decided to mount it read only and didn't let me remount it as `rw`. 11 | IPMI? Yes, I had the password in my password store but not the username. 12 | So the only logical solution was to kexec into an emergency system. 13 | This code is what I used. 14 | It recovers all IP addresses as well as SSH host and user keys from the old system and kexecs into a new one - entirely in-memory. 15 | 16 | ## What it does 17 | 18 | The `emergency` script (found in the repository root) will SSH over and execute the following things: 19 | 20 | 1. Build the recovery image (a `.tar.xz` with a small nix store and a `kexec` script) from the files in this repository locally on the machine you're executing this code on 21 | 1. The system configuration is found in `configuration.nix` 22 | 2. Some `kexec`-related features are imported from `kexec.nix` 23 | 3. The scripts will be included to be used in the `kexec` script (see below) 24 | 2. Try to `mkdir` `/nix` and `/tmp`. If the don't already exist and your root fs is read-only, you have a problem this project can't fix 25 | 3. Mount a fresh `tmpfs` on `/tmp` because there might not be one already 26 | 4. `scp` the emergency image over and extract it 27 | 5. Mount the nix store from the emergency image over `/nix` using `overlayfs` 28 | 6. Run the kexec script 29 | 30 | The `kexec` script (found in `kexec.nix`) will do the following: 31 | 32 | 1. Prepare a second initrd 33 | 2. Put your SSH host keys into the initrd 34 | 3. Put all of your SSH user keys into the initrd 35 | 4. Fetch all your IP addresses and routes and put them into the initrd 36 | 5. Pack the second initrd and append it to the default NixOS initrd from the emergency image 37 | 6. `kexec` into the kernel from the emergency image while using the new initrd 38 | 7. In case you didn't already notice: **This will crash your currently running system, so maybe it's a good idea to gracefully shut down remaining daemons if that's still possible** 39 | 40 | The script that is packed into the initrd of the new system will do the following: 41 | 42 | 1. Place the SSH host key 43 | 2. Place the SSH user keys 44 | 3. Place a script for the IP addresses which will be executed using `networking.localCommands` so the interfaces are available 45 | 46 | If you set the environment variable `EMERGENCY_DUMP_NETWORK` to `1`, all IPs, routes, and nameservers will be placed in the `emergency_ips`, `emergency_routes`, and `emergency_nameservers` files, respectively. 47 | 48 | ## How to use 49 | 50 | ``` 51 | $ ./emergency root@somehost 52 | # or 53 | $ ./emergency somebody@somehost 54 | ``` 55 | 56 | ## Disclaimer and license 57 | 58 | If it doesn't work for you, I'm sorry. 59 | I can probably not help you, but if you're able to fix something, feel free to create a PR. 60 | 61 | The code is based on [clever's](https://github.com/cleverca22) kexec nix-test (found [here](https://github.com/cleverca22/nix-tests/tree/master/kexec)). 62 | 63 | The code is licensed under the [LGPL3](LICENSE). 64 | -------------------------------------------------------------------------------- /emergency: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | if [ "${#}" != 1 ]; then 6 | ( 7 | echo "Usage: ${0} " 8 | echo 9 | echo 'connectTo is an SSH connection string' 10 | ) >&2 11 | exit 1 12 | fi 13 | 14 | ##### 15 | # Variables 16 | ##### 17 | 18 | connectTo="${1}" 19 | 20 | sshDir="/run/user/$(id -u)/helsinki-ssh" 21 | # shellcheck disable=SC2174 22 | mkdir -m 0700 -p "${sshDir}" 23 | sshFlags=(-o 'ControlMaster=auto' -o "ControlPath=${sshDir}/%r@%h:%p" -o 'ControlPersist=1h') 24 | ssh="ssh ${connectTo} ${sshFlags[*]}" 25 | myDir="$(dirname "${0}")" 26 | 27 | ##### 28 | # kexec image 29 | ##### 30 | 31 | nix build -f '' -I nixos-config="${myDir}/configuration.nix" config.system.build.kexec_tarball 32 | 33 | ##### 34 | # Test connection and prepare 35 | ##### 36 | 37 | if ! ${ssh} echo Connection successful; then 38 | echo "Connection cannot be established" 39 | exit 1 40 | fi 41 | if ! ${ssh} zgrep CONFIG_KEXEC=y /proc/config.gz; then 42 | echo "Target system lacks kexec capabilities" 43 | exit 1 44 | fi 45 | 46 | ${ssh} apt install rsync || : 47 | ${ssh} mkdir -p /nix /tmp 48 | ${ssh} mount -t tmpfs tmpfs /tmp 49 | ${ssh} mkdir -p /tmp/work /tmp/nix 50 | 51 | ##### 52 | # Networking 53 | ##### 54 | 55 | # Serialize the current network state 56 | ips="$(${ssh} ip a | awk -f "${myDir}/ips.awk")" 57 | routes="$(${ssh} ip r | awk -f "${myDir}/routes.awk" 58 | ${ssh} ip -6 r | awk -f "${myDir}/routes.awk")" 59 | nameservers="$(${ssh} grep ^nameserver /etc/resolv.conf | cut -d' ' -f2)" 60 | # Handle systemd-resolved 61 | nameservers="${nameservers/127.0.0.53/1.1.1.1}" 62 | 63 | # Dump the network state if desired 64 | if [ "${EMERGENCY_DUMP_NETWORK:-}" = 1 ]; then 65 | echo "$ips" > emergency_ips 66 | echo "$routes" > emergency_routes 67 | echo "$nameservers" > emergency_nameservers 68 | fi 69 | 70 | # Deserialize the network state 71 | { 72 | # Do not set -e here which would fail if interfaces (e.g. VPN) are missing. 73 | echo "set -x" 74 | 75 | # IPs 76 | while IFS=";" read -r interface interfaceIps; do 77 | echo "ip l set ${interface} up" 78 | for ip in $(echo "${interfaceIps}" | tr ';' '\n'); do 79 | echo "ip a a ${ip} dev ${interface}" 80 | done 81 | done < <(echo "${ips}") 82 | 83 | # Routes 84 | allRoutes="$(while IFS=';' read -r net via dev; do 85 | echo -n "ip r a $net" 86 | [ -n "${via:-}" ] && echo -n " via ${via}" 87 | [ -n "${dev:-}" ] && echo -n " dev ${dev}" 88 | echo 89 | done < <(echo "${routes}"))" 90 | # Output per-link routes first 91 | echo "${allRoutes}" | grep -v via 92 | echo "${allRoutes}" | grep via 93 | 94 | # Nameservers 95 | echo '(' 96 | while IFS= read -r ns; do 97 | echo "echo 'nameserver ${ns}'" 98 | done < <(echo "${nameservers}") 99 | echo ') > /etc/resolv.conf' 100 | } | ${ssh} tee /tmp/ip-script > /dev/null 101 | 102 | ##### 103 | # kexec 104 | ##### 105 | 106 | # Copy and extract 107 | rsync -P result/tarball/*.xz "${connectTo}:/tmp/emergency.tar.xz" 108 | ${ssh} tar xf /tmp/emergency.tar.xz -C /tmp 109 | # Overlay the second /nix 110 | ${ssh} mount -t overlay overlay -o lowerdir=/nix,upperdir=/tmp/nix,workdir=/tmp/work /nix 111 | # Here goes nothing 112 | ${ssh} /tmp/kexec 113 | ${ssh} -O exit 114 | 115 | ##### 116 | # Wait for SSH to be available again 117 | ##### 118 | 119 | sleep 10 # Ensure the old system is already shut down 120 | echo -n "Attempting to connect..." 121 | while ! ${ssh} echo Connection successful; do 122 | sleep 5 123 | echo -n "." 124 | done 125 | echo 126 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | --------------------------------------------------------------------------------