├── .gitignore ├── DEBUG.md ├── LICENSE ├── README.md ├── constants.nix ├── flake.lock ├── flake.nix ├── module.nix └── package.nix /.gitignore: -------------------------------------------------------------------------------- 1 | /result 2 | -------------------------------------------------------------------------------- /DEBUG.md: -------------------------------------------------------------------------------- 1 | # Debugging nix-rosetta-builder 2 | 3 | Rough debugging notes and suggestions. 4 | If you think of something that's missing or have something to add please 5 | [open an issue](https://github.com/cpick/nix-rosetta-builder/issues/new). 6 | 7 | ## Gather information 8 | 9 | ### General 10 | 11 | Test whether VM is running: 12 | ```sh 13 | sudo ssh rosetta-builder 14 | ``` 15 | 16 | ### Launchd 17 | 18 | See the VM daemon's status in `launchd`: 19 | ```sh 20 | launchctl print system/org.nixos.rosetta-builderd 21 | ``` 22 | Some fields in that output that deserve attention (see also "launchd.log" below): 23 | * state 24 | * pid 25 | * sockets."Listener".error 26 | 27 | Unload/stop the VM daemon: 28 | ```sh 29 | sudo launchctl bootout system/org.nixos.rosetta-builderd 30 | ``` 31 | (Possibly followed by `sudo rm /Library/LaunchDaemons/org.nixos.rosetta-builderd.plist` to force a 32 | subsequent `darwin-rebuild` to reload it, or else...) 33 | 34 | (Re)load the VM daemon: 35 | ```sh 36 | sudo launchctl bootstrap system /Library/LaunchDaemons/org.nixos.rosetta-builderd.plist 37 | ``` 38 | 39 | ## Logs 40 | 41 | * /private/var/log/com.apple.xpc.launchd/launchd.log 42 | * /var/lib/rosetta-builder/.lima/rosetta-builder-vm/ha.stderr.log 43 | * /var/lib/rosetta-builder/.lima/rosetta-builder-vm/ha.stdout.log 44 | 45 | When module.nix's `debugInsecurely = true`: 46 | * /tmp/rosetta-builderd.err.log 47 | * /tmp/rosetta-builderd.out.log 48 | 49 | ## Uninstall 50 | 51 | See [README.md#Uninstall]. 52 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2024 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nix-rosetta-builder 2 | 3 | A [Rosetta 2](https://developer.apple.com/documentation/virtualization/running_intel_binaries_in_linux_vms_with_rosetta)-enabled, 4 | Apple silicon (macOS/Darwin)-hosted Linux 5 | [Nix builder](https://nix.dev/manual/nix/2.18/advanced-topics/distributed-builds). 6 | 7 | Runs on aarch64-darwin and builds aarch64-linux (natively) and x86_64-linux (quickly using Rosetta 8 | 2). 9 | 10 | ## Features 11 | 12 | Advantages over nix-darwin's built in 13 | [`nix.linux-builder`](https://daiderd.com/nix-darwin/manual/index.html#opt-nix.linux-builder.enable) 14 | (which is based on 15 | [`pkgs.darwin.linux-builder`](https://nixos.org/manual/nixpkgs/stable/#sec-darwin-builder)): 16 | 17 | * x86_64-linux support enabled by default and much faster (using Rosetta 2) 18 | * Multi-core by default 19 | * Optionally runs VM on-demand, powering off when idle (see: `nix-rosetta-builder.onDemand` option) 20 | * More secure: 21 | * VM runs with minimum permissions (runs as a non-root/admin/wheel user/service account) 22 | * VM doesn't accept remote connections (it binds to the loopback interface (127.0.0.1)) 23 | * VM cannot be impersonated (its private SSH host key is not publicly-known) 24 | 25 | ## nix-darwin flake setup 26 | 27 | flake.nix: 28 | ```nix 29 | { 30 | description = "Configure macOS using nix-darwin with rosetta-builder"; 31 | 32 | inputs = { 33 | nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; 34 | nix-darwin = { 35 | url = "github:lnl7/nix-darwin"; 36 | inputs.nixpkgs.follows = "nixpkgs"; 37 | }; 38 | nix-rosetta-builder = { 39 | url = "github:cpick/nix-rosetta-builder"; 40 | inputs.nixpkgs.follows = "nixpkgs"; 41 | }; 42 | }; 43 | 44 | outputs = inputs@{ self, nix-darwin, nix-rosetta-builder, nixpkgs }: { 45 | darwinConfigurations."${hostname}" = nix-darwin.lib.darwinSystem { 46 | modules = [ 47 | # An existing Linux builder is needed to initially bootstrap `nix-rosetta-builder`. 48 | # If one isn't already available: comment out the `nix-rosetta-builder` module below, 49 | # uncomment this `linux-builder` module, and run `darwin-rebuild switch`: 50 | # { nix.linux-builder.enable = true; } 51 | # Then: uncomment `nix-rosetta-builder`, remove `linux-builder`, and `darwin-rebuild switch` 52 | # a second time. Subsequently, `nix-rosetta-builder` can rebuild itself. 53 | nix-rosetta-builder.darwinModules.default 54 | { 55 | # see available options in module.nix's `options.nix-rosetta-builder` 56 | nix-rosetta-builder.onDemand = true; 57 | } 58 | ]; 59 | }; 60 | }; 61 | } 62 | ``` 63 | 64 | ## Uninstall 65 | 66 | 1. Set `nix-rosetta-builder.enable = false` in the nix-darwin configuration and run 67 | `darwin-rebuild switch` to clean up resources (including VM, user, group, storage, etc) 68 | 2. Optionally remove `nix-rosetta-builder` from the nix-darwin configuration 69 | 70 | ## Contributing 71 | 72 | Feature requests, bug reports, and pull requests are all welcome. 73 | -------------------------------------------------------------------------------- /constants.nix: -------------------------------------------------------------------------------- 1 | rec { 2 | name = "rosetta-builder"; # update `darwinGroup` if adding or removing special characters 3 | linuxHostName = name; # no prefix because it's user visible (on prompt when `ssh`d in) 4 | linuxUser = "builder"; # follow linux-builder/darwin-builder precedent 5 | 6 | sshKeyType = "ed25519"; 7 | sshHostPrivateKeyFileName = "ssh_host_${sshKeyType}_key"; 8 | sshHostPublicKeyFileName = "${sshHostPrivateKeyFileName}.pub"; 9 | sshUserPrivateKeyFileName = "ssh_user_${sshKeyType}_key"; 10 | sshUserPublicKeyFileName = "${sshUserPrivateKeyFileName}.pub"; 11 | } 12 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "nixlib": { 4 | "locked": { 5 | "lastModified": 1736643958, 6 | "narHash": "sha256-tmpqTSWVRJVhpvfSN9KXBvKEXplrwKnSZNAoNPf/S/s=", 7 | "owner": "nix-community", 8 | "repo": "nixpkgs.lib", 9 | "rev": "1418bc28a52126761c02dd3d89b2d8ca0f521181", 10 | "type": "github" 11 | }, 12 | "original": { 13 | "owner": "nix-community", 14 | "repo": "nixpkgs.lib", 15 | "type": "github" 16 | } 17 | }, 18 | "nixos-generators": { 19 | "inputs": { 20 | "nixlib": "nixlib", 21 | "nixpkgs": [ 22 | "nixpkgs" 23 | ] 24 | }, 25 | "locked": { 26 | "lastModified": 1737057290, 27 | "narHash": "sha256-3Pe0yKlCc7EOeq1X/aJVDH0CtNL+tIBm49vpepwL1MQ=", 28 | "owner": "nix-community", 29 | "repo": "nixos-generators", 30 | "rev": "d002ce9b6e7eb467cd1c6bb9aef9c35d191b5453", 31 | "type": "github" 32 | }, 33 | "original": { 34 | "owner": "nix-community", 35 | "repo": "nixos-generators", 36 | "type": "github" 37 | } 38 | }, 39 | "nixpkgs": { 40 | "locked": { 41 | "lastModified": 1737885589, 42 | "narHash": "sha256-Zf0hSrtzaM1DEz8//+Xs51k/wdSajticVrATqDrfQjg=", 43 | "owner": "nixos", 44 | "repo": "nixpkgs", 45 | "rev": "852ff1d9e153d8875a83602e03fdef8a63f0ecf8", 46 | "type": "github" 47 | }, 48 | "original": { 49 | "owner": "nixos", 50 | "ref": "nixos-unstable", 51 | "repo": "nixpkgs", 52 | "type": "github" 53 | } 54 | }, 55 | "root": { 56 | "inputs": { 57 | "nixos-generators": "nixos-generators", 58 | "nixpkgs": "nixpkgs" 59 | } 60 | } 61 | }, 62 | "root": "root", 63 | "version": 7 64 | } 65 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "Lima-based, Rosetta 2-enabled, Apple silicon (macOS/Darwin)-hosted Linux builder"; 3 | 4 | inputs = { 5 | nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; 6 | nixos-generators = { 7 | url = "github:nix-community/nixos-generators"; 8 | inputs.nixpkgs.follows = "nixpkgs"; 9 | }; 10 | }; 11 | 12 | outputs = 13 | { 14 | self, 15 | nixos-generators, 16 | nixpkgs, 17 | }: 18 | let 19 | darwinSystem = "aarch64-darwin"; 20 | linuxSystem = builtins.replaceStrings [ "darwin" ] [ "linux" ] darwinSystem; 21 | in 22 | { 23 | packages."${linuxSystem}" = 24 | let 25 | pkgs = nixpkgs.legacyPackages."${linuxSystem}"; 26 | in 27 | rec { 28 | default = image; 29 | 30 | image = pkgs.callPackage ./package.nix { 31 | inherit linuxSystem nixos-generators nixpkgs; 32 | # Optional: override default argument values passed to the derivation. 33 | # Many can also be accessed through the module. 34 | }; 35 | }; 36 | 37 | devShells."${darwinSystem}".default = 38 | let 39 | pkgs = nixpkgs.legacyPackages."${darwinSystem}"; 40 | in 41 | pkgs.mkShell { 42 | packages = [ pkgs.lima ]; 43 | }; 44 | 45 | darwinModules.default = import ./module.nix { 46 | inherit linuxSystem; 47 | image = self.packages."${linuxSystem}".image; 48 | }; 49 | 50 | formatter."${darwinSystem}" = nixpkgs.legacyPackages."${darwinSystem}".nixfmt-rfc-style; 51 | }; 52 | } 53 | -------------------------------------------------------------------------------- /module.nix: -------------------------------------------------------------------------------- 1 | { 2 | # configuration 3 | image, 4 | linuxSystem, 5 | }: 6 | { 7 | config, 8 | lib, 9 | pkgs, 10 | ... 11 | }: 12 | let 13 | inherit (lib) 14 | boolToString 15 | escapeShellArg 16 | mkAfter 17 | mkBefore 18 | mkDefault 19 | mkEnableOption 20 | mkForce 21 | mkIf 22 | mkMerge 23 | mkOption 24 | optionalAttrs 25 | optionalString 26 | types 27 | ; 28 | in 29 | { 30 | options.nix-rosetta-builder = { 31 | enable = (mkEnableOption "Nix Rosetta Linux builder") // { 32 | default = true; 33 | }; 34 | 35 | potentiallyInsecureExtraNixosModule = mkOption { 36 | type = types.attrs; 37 | default = { }; 38 | description = '' 39 | Extra NixOS configuration module to pass to the VM. 40 | The VM's default configuration allows it to be securely used as a builder. Some extra 41 | configuration changes may endager this security and allow compromised deriviations into the 42 | host's Nix store. Care should be taken to think through the implications of any extra 43 | configuration changes using this option. When in doubt, please open a GitHub issue to 44 | discuss (additional, restricted options can be added to support safe configurations). 45 | ''; 46 | }; 47 | 48 | cores = mkOption { 49 | type = types.int; 50 | default = 8; 51 | description = '' 52 | The number of CPU cores allocated to the VM. 53 | This also sets the maximum number of jobs allowed for the 54 | builder in the `nix.buildMachines` specification. 55 | ''; 56 | }; 57 | 58 | diskSize = mkOption { 59 | type = types.str; 60 | default = "100GiB"; 61 | description = '' 62 | The size of the disk image for the VM. 63 | ''; 64 | }; 65 | 66 | memory = mkOption { 67 | type = types.str; 68 | default = "6GiB"; 69 | description = '' 70 | The amount of memory to allocate to the VM. 71 | ''; 72 | example = "8GiB"; 73 | }; 74 | 75 | onDemand = mkOption { 76 | type = types.bool; 77 | default = false; 78 | description = '' 79 | By default, the VM will run all the time as a daemon in the background. This allows Linux 80 | builds to start right away, but means the VM is always consuming RAM (and a bit of CPU). 81 | 82 | Alternatively, this option will cause the VM to run only "on-demand": when not in use the VM 83 | will not be running. Any Linux build will cause it to automatically start up 84 | (blocking/pausing the build for several seconds until the VM boots) and after a period of 85 | time/hours without any active Linux builds, the VM will power itself off. 86 | ''; 87 | }; 88 | 89 | onDemandLingerMinutes = mkOption { 90 | type = types.ints.positive; 91 | default = 180; 92 | description = '' 93 | If onDemand=true, this specifies the number of minutes of inactivity before the VM will 94 | power itself off. 95 | ''; 96 | }; 97 | 98 | permitNonRootSshAccess = mkOption { 99 | type = types.bool; 100 | default = false; 101 | description = '' 102 | Allow regular, non-root users to SSH into the VM with `ssh rosetta-builder`. 103 | 104 | By default, regular users can `nix build` using the VM without any extra permissions (since 105 | it's configured as a remote builder), but they can only SSH directly into it with 106 | `sudo ssh rosetta-builder`. 107 | ''; 108 | }; 109 | 110 | port = mkOption { 111 | type = types.int; 112 | 113 | # `nix.linux-builder` uses 31022: 114 | # https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/nix/linux-builder.nix#L199 115 | # Use a similar, but different one: 116 | default = 31122; 117 | 118 | description = '' 119 | The SSH port used by the VM. 120 | ''; 121 | }; 122 | }; 123 | 124 | config = 125 | let 126 | inherit (import ./constants.nix) 127 | name 128 | linuxHostName 129 | linuxUser 130 | sshKeyType 131 | sshHostPrivateKeyFileName 132 | sshHostPublicKeyFileName 133 | sshUserPrivateKeyFileName 134 | sshUserPublicKeyFileName 135 | ; 136 | 137 | debugInsecurely = false; # enable root access in VM and debug logging 138 | 139 | imageWithFinalConfig = image.override { 140 | inherit debugInsecurely; 141 | onDemand = cfg.onDemand; 142 | onDemandLingerMinutes = cfg.onDemandLingerMinutes; 143 | potentiallyInsecureExtraNixosModule = cfg.potentiallyInsecureExtraNixosModule; 144 | }; 145 | 146 | cfg = config.nix-rosetta-builder; 147 | daemonName = "${name}d"; 148 | daemonSocketName = "Listener"; 149 | 150 | # `sysadminctl -h` says role account UIDs (no mention of service accounts or GIDs) should be 151 | # in the 200-400 range `mkuser`s README.md mentions the same: 152 | # https://github.com/freegeek-pdx/mkuser/blob/b7a7900d2e6ef01dfafad1ba085c94f7302677d9/README.md?plain=1#L413-L437 153 | # Determinate's `nix-installer` (and, I believe, current versions of the official one) uses a 154 | # variable number starting at 350 and up: 155 | # https://github.com/DeterminateSystems/nix-installer/blob/6beefac4d23bd9a0b74b6758f148aa24d6df3ca9/README.md?plain=1#L511-L514 156 | # Meanwhile, new macOS versions are installing accounts that encroach from below. 157 | # Try to fit in between: 158 | darwinGid = 349; 159 | darwinUid = darwinGid; 160 | 161 | darwinGroup = builtins.replaceStrings [ "-" ] [ "" ] name; # keep in sync with `name`s format 162 | darwinUser = "_${darwinGroup}"; 163 | linuxSshdKeysDirName = "linux-sshd-keys"; 164 | 165 | sshGlobalKnownHostsFileName = "ssh_known_hosts"; 166 | sshHost = name; # no prefix because it's user visible (in `sudo ssh '${sshHost}'`) 167 | sshHostKeyAlias = "${sshHost}-key"; 168 | workingDirPath = "/var/lib/${name}"; 169 | 170 | gidSh = escapeShellArg (toString darwinGid); 171 | groupSh = escapeShellArg darwinGroup; 172 | groupPathSh = escapeShellArg "/Groups/${darwinGroup}"; 173 | 174 | uidSh = escapeShellArg (toString darwinUid); 175 | userSh = escapeShellArg darwinUser; 176 | userPathSh = escapeShellArg "/Users/${darwinUser}"; 177 | 178 | workingDirPathSh = escapeShellArg workingDirPath; 179 | 180 | vmYaml = (pkgs.formats.yaml { }).generate "${name}.yaml" { 181 | # Prevent ~200MiB unused nerdctl-full*.tar.gz download 182 | # https://github.com/lima-vm/lima/blob/0e931107cadbcb6dbc7bbb25626f66cdbca1f040/pkg/instance/start.go#L43 183 | containerd.user = false; 184 | 185 | cpus = cfg.cores; 186 | 187 | disk = cfg.diskSize; 188 | 189 | images = [ 190 | { 191 | # extension must match `imageFormat` 192 | location = "${imageWithFinalConfig}/nixos.qcow2"; 193 | } 194 | ]; 195 | 196 | memory = cfg.memory; 197 | 198 | mounts = [ 199 | { 200 | # order must match `sshdKeysVirtiofsTag`s suffix 201 | location = "${workingDirPath}/${linuxSshdKeysDirName}"; 202 | } 203 | ]; 204 | 205 | rosetta.enabled = true; 206 | 207 | ssh = { 208 | launchdSocketName = optionalString cfg.onDemand daemonSocketName; 209 | localPort = cfg.port; 210 | }; 211 | }; 212 | in 213 | mkMerge [ 214 | (mkIf (!cfg.enable) { 215 | # This `postActivation` was chosen in particiular because it's one of the system level (as 216 | # opposed to user level) ones that's been set aside for customization: 217 | # https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/system/activation-scripts.nix#L121-L125 218 | # And of those, it's the one that's executed after `activationScripts.launchd` which stops 219 | # the VM: 220 | # https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/system/activation-scripts.nix#L58-L66 221 | # https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/system/activation-scripts.nix#L66-L75 222 | system.activationScripts.postActivation.text = 223 | # apply "before" to work cooperatively with any other modules using this activation script 224 | mkBefore '' 225 | if [ -d ${workingDirPathSh} ] ; then 226 | printf >&2 'removing working directory %s...\n' ${workingDirPathSh} 227 | rm -rf ${workingDirPathSh} 228 | fi 229 | 230 | if uid="$(id -u ${userSh} 2>'/dev/null')" ; then 231 | if [ "$uid" -ne ${uidSh} ] ; then 232 | printf >&2 \ 233 | '\e[1;31merror: existing user: %s has unexpected UID: %s\e[0m\n' \ 234 | ${userSh} \ 235 | "$uid" 236 | exit 1 237 | fi 238 | printf >&2 'deleting user %s...\n' ${userSh} 239 | dscl . -delete ${userPathSh} 240 | fi 241 | unset 'uid' 242 | 243 | if primaryGroupId="$(dscl . -read ${groupPathSh} 'PrimaryGroupID' 2>'/dev/null')" ; then 244 | if [[ "$primaryGroupId" != *\ ${gidSh} ]] ; then 245 | printf >&2 \ 246 | '\e[1;31merror: existing group: %s has unexpected %s\e[0m\n' \ 247 | ${groupSh} \ 248 | "$primaryGroupId" 249 | exit 1 250 | fi 251 | printf >&2 'deleting group %s...\n' ${groupSh} 252 | dscl . -delete ${groupPathSh} 253 | fi 254 | unset 'primaryGroupId' 255 | ''; 256 | }) 257 | (mkIf cfg.enable { 258 | environment.etc."ssh/ssh_config.d/100-${sshHost}.conf".text = '' 259 | Host "${sshHost}" 260 | GlobalKnownHostsFile "${workingDirPath}/${sshGlobalKnownHostsFileName}" 261 | Hostname localhost 262 | HostKeyAlias "${sshHostKeyAlias}" 263 | Port "${toString cfg.port}" 264 | StrictHostKeyChecking yes 265 | User "${linuxUser}" 266 | IdentityFile "${workingDirPath}/${sshUserPrivateKeyFileName}" 267 | ''; 268 | 269 | launchd.daemons."${daemonName}" = { 270 | path = [ 271 | pkgs.coreutils 272 | pkgs.diffutils 273 | pkgs.findutils 274 | pkgs.gnugrep 275 | (pkgs.lima.overrideAttrs (old: { 276 | src = pkgs.fetchFromGitHub { 277 | owner = "cpick"; 278 | repo = "lima"; 279 | rev = "afbfdfb8dd5fa370547b7fc64a16ce2a354b1ff0"; 280 | hash = "sha256-tCildZJp6ls+WxRAbkoeLRb4WdroBYn/gvE5Vb8Hm5A="; 281 | }; 282 | 283 | vendorHash = "sha256-I84971WovhJL/VO/Ycu12qa9lDL3F9USxlt9rXcsnTU="; 284 | })) 285 | pkgs.openssh 286 | 287 | # Lima calls `sw_vers` which is not packaged in Nix: 288 | # https://github.com/lima-vm/lima/blob/0e931107cadbcb6dbc7bbb25626f66cdbca1f040/pkg/osutil/osversion_darwin.go#L13 289 | # If the call fails it will not use the Virtualization framework bakend (by default? among 290 | # other things?). 291 | "/usr/bin" 292 | ]; 293 | 294 | script = 295 | let 296 | darwinUserSh = escapeShellArg darwinUser; 297 | linuxHostNameSh = escapeShellArg linuxHostName; 298 | linuxSshdKeysDirNameSh = escapeShellArg linuxSshdKeysDirName; 299 | sshGlobalKnownHostsFileNameSh = escapeShellArg sshGlobalKnownHostsFileName; 300 | sshHostKeyAliasSh = escapeShellArg sshHostKeyAlias; 301 | sshHostPrivateKeyFileNameSh = escapeShellArg sshHostPrivateKeyFileName; 302 | sshHostPublicKeyFileNameSh = escapeShellArg sshHostPublicKeyFileName; 303 | sshKeyTypeSh = escapeShellArg sshKeyType; 304 | sshUserPrivateKeyFileNameSh = escapeShellArg sshUserPrivateKeyFileName; 305 | sshUserPublicKeyFileNameSh = escapeShellArg sshUserPublicKeyFileName; 306 | vmNameSh = escapeShellArg "${name}-vm"; 307 | vmYamlSh = escapeShellArg vmYaml; 308 | in 309 | '' 310 | set -e 311 | set -u 312 | 313 | umask 'g-w,o=' 314 | chmod 'g-w,o=x' . 315 | 316 | # must be idempotent in the face of partial failues 317 | # the `find` test must fail if the user private key was readable but should no longer be 318 | cmp -s ${vmYamlSh} .lima/${vmNameSh}/lima.yaml && \ 319 | limactl list -q 2>'/dev/null' | grep -q ${vmNameSh} && \ 320 | find ${sshUserPrivateKeyFileNameSh} \ 321 | -perm '-go=r' -exec ${boolToString cfg.permitNonRootSshAccess} '{}' '+' \ 322 | 2>'/dev/null' && \ 323 | true || { 324 | rm -f ${sshUserPrivateKeyFileNameSh} ${sshUserPublicKeyFileNameSh} 325 | ssh-keygen \ 326 | -C ${darwinUserSh}@darwin -f ${sshUserPrivateKeyFileNameSh} -N "" -t ${sshKeyTypeSh} 327 | 328 | rm -f ${sshHostPrivateKeyFileNameSh} ${sshHostPublicKeyFileNameSh} 329 | ssh-keygen \ 330 | -C root@${linuxHostNameSh} -f ${sshHostPrivateKeyFileNameSh} -N "" -t ${sshKeyTypeSh} 331 | 332 | mkdir -p ${linuxSshdKeysDirNameSh} 333 | mv \ 334 | ${sshUserPublicKeyFileNameSh} ${sshHostPrivateKeyFileNameSh} \ 335 | ${linuxSshdKeysDirNameSh} 336 | 337 | echo ${sshHostKeyAliasSh} "$(cat ${sshHostPublicKeyFileNameSh})" \ 338 | >${sshGlobalKnownHostsFileNameSh} 339 | 340 | limactl delete --force ${vmNameSh} 341 | 342 | # must be last so `limactl list` only now succeeds 343 | limactl create --name=${vmNameSh} ${vmYamlSh} 344 | } 345 | 346 | # outside the block so both new and old installations end up with the same permissions 347 | chmod 'go+r' ${sshGlobalKnownHostsFileNameSh} 348 | 349 | # outside the block so non-root access may be enabled without recreating VM 350 | ${optionalString cfg.permitNonRootSshAccess '' 351 | chmod 'go+r' ${sshUserPrivateKeyFileNameSh} 352 | ''} 353 | 354 | exec limactl start ${optionalString debugInsecurely "--debug"} --foreground ${vmNameSh} 355 | ''; 356 | 357 | serviceConfig = 358 | { 359 | KeepAlive = !cfg.onDemand; 360 | 361 | Sockets."${daemonSocketName}" = optionalAttrs cfg.onDemand { 362 | SockFamily = "IPv4"; 363 | SockNodeName = "localhost"; 364 | SockServiceName = toString cfg.port; 365 | }; 366 | 367 | UserName = darwinUser; 368 | WorkingDirectory = workingDirPath; 369 | } 370 | // optionalAttrs debugInsecurely { 371 | StandardErrorPath = "/tmp/${daemonName}.err.log"; 372 | StandardOutPath = "/tmp/${daemonName}.out.log"; 373 | }; 374 | }; 375 | 376 | nix = { 377 | buildMachines = [ 378 | { 379 | hostName = sshHost; 380 | maxJobs = cfg.cores; 381 | protocol = "ssh-ng"; 382 | supportedFeatures = [ 383 | "benchmark" 384 | "big-parallel" 385 | "kvm" 386 | "nixos-test" 387 | ]; 388 | systems = [ 389 | linuxSystem 390 | "x86_64-linux" 391 | ]; 392 | } 393 | ]; 394 | 395 | distributedBuilds = mkForce true; 396 | settings.builders-use-substitutes = mkDefault true; 397 | }; 398 | 399 | # `users.users` cannot create a service account and cannot create an empty home directory so do 400 | # it manually in an activation script. This `extraActivation` was chosen in particiular because 401 | # it's one of the system level (as opposed to user level) ones that's been set aside for 402 | # customization: 403 | # https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/system/activation-scripts.nix#L121-L125 404 | # And of those, it's the one that's executed latest but still before 405 | # `activationScripts.launchd` which needs the group, user, and directory in place: 406 | # https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/system/activation-scripts.nix#L58-L66 407 | system.activationScripts.extraActivation.text = 408 | # apply "after" to work cooperatively with any other modules using this activation script 409 | mkAfter '' 410 | printf >&2 'setting up group %s...\n' ${groupSh} 411 | 412 | if ! primaryGroupId="$(dscl . -read ${groupPathSh} 'PrimaryGroupID' 2>'/dev/null')" ; then 413 | printf >&2 'creating group %s...\n' ${groupSh} 414 | dscl . -create ${groupPathSh} 'PrimaryGroupID' ${gidSh} 415 | elif [[ "$primaryGroupId" != *\ ${gidSh} ]] ; then 416 | printf >&2 \ 417 | '\e[1;31merror: existing group: %s has unexpected %s\e[0m\n' \ 418 | ${groupSh} \ 419 | "$primaryGroupId" 420 | exit 1 421 | fi 422 | unset 'primaryGroupId' 423 | 424 | 425 | printf >&2 'setting up user %s...\n' ${userSh} 426 | 427 | if ! uid="$(id -u ${userSh} 2>'/dev/null')" ; then 428 | printf >&2 'creating user %s...\n' ${userSh} 429 | dscl . -create ${userPathSh} 430 | dscl . -create ${userPathSh} 'PrimaryGroupID' ${gidSh} 431 | dscl . -create ${userPathSh} 'NFSHomeDirectory' ${workingDirPathSh} 432 | dscl . -create ${userPathSh} 'UserShell' '/usr/bin/false' 433 | dscl . -create ${userPathSh} 'IsHidden' 1 434 | dscl . -create ${userPathSh} 'UniqueID' ${uidSh} # must be last so `id` only now succeeds 435 | elif [ "$uid" -ne ${uidSh} ] ; then 436 | printf >&2 \ 437 | '\e[1;31merror: existing user: %s has unexpected UID: %s\e[0m\n' \ 438 | ${userSh} \ 439 | "$uid" 440 | exit 1 441 | fi 442 | unset 'uid' 443 | 444 | 445 | printf >&2 'setting up working directory %s...\n' ${workingDirPathSh} 446 | mkdir -p ${workingDirPathSh} 447 | chown ${userSh}:${groupSh} ${workingDirPathSh} 448 | ''; 449 | }) 450 | ]; 451 | } 452 | -------------------------------------------------------------------------------- /package.nix: -------------------------------------------------------------------------------- 1 | { 2 | # dependencies 3 | nixos-generators, 4 | nixpkgs, 5 | # pkgs 6 | lib, 7 | mount, 8 | umount, 9 | # configuration 10 | linuxSystem, 11 | debugInsecurely ? false, # enable auto-login and passwordless sudo to root 12 | potentiallyInsecureExtraNixosModule ? { }, 13 | onDemand ? false, # enable launchd socket activation 14 | onDemandLingerMinutes ? 180, # poweroff after 3 hours of inactivity 15 | withRosetta ? true, 16 | }: 17 | nixos-generators.nixosGenerate ( 18 | let 19 | inherit (lib) escapeShellArg optionalAttrs optionals; 20 | inherit (import ./constants.nix) 21 | linuxHostName 22 | linuxUser 23 | sshHostPrivateKeyFileName 24 | sshUserPublicKeyFileName 25 | ; 26 | imageFormat = "qcow-efi"; # must match `vmYaml.images.location`s extension 27 | 28 | sshdKeys = "sshd-keys"; 29 | sshDirPath = "/etc/ssh"; 30 | sshHostPrivateKeyFilePath = "${sshDirPath}/${sshHostPrivateKeyFileName}"; 31 | in 32 | { 33 | format = imageFormat; 34 | modules = [ 35 | { 36 | boot = { 37 | kernelParams = [ "console=tty0" ]; 38 | 39 | loader = { 40 | efi.canTouchEfiVariables = true; 41 | systemd-boot.enable = true; 42 | }; 43 | }; 44 | 45 | documentation.enable = false; 46 | 47 | fileSystems = { 48 | "/".options = [ 49 | "discard" 50 | "noatime" 51 | ]; 52 | "/boot".options = [ 53 | "discard" 54 | "noatime" 55 | "umask=0077" 56 | ]; 57 | }; 58 | 59 | networking.hostName = linuxHostName; 60 | 61 | nix = { 62 | channel.enable = false; 63 | registry.nixpkgs.flake = nixpkgs; 64 | 65 | settings = { 66 | auto-optimise-store = true; 67 | experimental-features = [ 68 | "flakes" 69 | "nix-command" 70 | ]; 71 | min-free = "5G"; 72 | max-free = "7G"; 73 | trusted-users = [ linuxUser ]; 74 | }; 75 | }; 76 | 77 | security = { 78 | sudo = { 79 | enable = debugInsecurely; 80 | wheelNeedsPassword = !debugInsecurely; 81 | }; 82 | }; 83 | 84 | services = { 85 | getty = optionalAttrs debugInsecurely { autologinUser = linuxUser; }; 86 | 87 | logind = optionalAttrs onDemand { 88 | extraConfig = '' 89 | IdleAction=poweroff 90 | IdleActionSec=${toString onDemandLingerMinutes}minutes 91 | ''; 92 | }; 93 | 94 | openssh = { 95 | enable = true; 96 | hostKeys = [ ]; # disable automatic host key generation 97 | 98 | settings = { 99 | HostKey = sshHostPrivateKeyFilePath; 100 | PasswordAuthentication = false; 101 | }; 102 | }; 103 | }; 104 | 105 | system = { 106 | disableInstallerTools = true; 107 | stateVersion = "24.05"; 108 | }; 109 | 110 | # macOS' Virtualization framework's virtiofs implementation will grant any guest user access 111 | # to mounted files; they always appear to be owned by the effective UID and so access cannot 112 | # be restricted. 113 | # To protect the guest's SSH host key, the VM is configured to prevent any logins (via 114 | # console, SSH, etc) by default. This service then runs before sshd, mounts virtiofs, 115 | # copies the keys to local files (with appropriate ownership and permissions), and unmounts 116 | # the filesystem before allowing SSH to start. 117 | # Once SSH has been allowed to start (and given the guest user a chance to log in), the 118 | # virtiofs must never be mounted again (as the user could have left some process active to 119 | # read its secrets). This is prevented by `unitconfig.ConditionPathExists` below. 120 | systemd.services."${sshdKeys}" = 121 | let 122 | # Lima labels its virtiofs folder mounts counting up: 123 | # https://github.com/lima-vm/lima/blob/0e931107cadbcb6dbc7bbb25626f66cdbca1f040/pkg/vz/vm_darwin.go#L568 124 | # So this suffix must match `vmYaml.mounts.location`s order: 125 | sshdKeysVirtiofsTag = "mount0"; 126 | 127 | sshdKeysDirPath = "/var/${sshdKeys}"; 128 | sshAuthorizedKeysUserFilePath = "${sshDirPath}/authorized_keys.d/${linuxUser}"; 129 | sshdService = "sshd.service"; 130 | in 131 | { 132 | before = [ sshdService ]; 133 | description = "Install sshd's host and authorized keys"; 134 | enableStrictShellChecks = true; 135 | path = [ 136 | mount 137 | umount 138 | ]; 139 | requiredBy = [ sshdService ]; 140 | 141 | script = 142 | let 143 | sshAuthorizedKeysUserFilePathSh = escapeShellArg sshAuthorizedKeysUserFilePath; 144 | sshAuthorizedKeysUserTmpFilePathSh = escapeShellArg "${sshAuthorizedKeysUserFilePath}.tmp"; 145 | sshHostPrivateKeyFileNameSh = escapeShellArg sshHostPrivateKeyFileName; 146 | sshHostPrivateKeyFilePathSh = escapeShellArg sshHostPrivateKeyFilePath; 147 | sshUserPublicKeyFileNameSh = escapeShellArg sshUserPublicKeyFileName; 148 | sshdKeysDirPathSh = escapeShellArg sshdKeysDirPath; 149 | sshdKeysVirtiofsTagSh = escapeShellArg sshdKeysVirtiofsTag; 150 | in 151 | '' 152 | # must be idempotent in the face of partial failues 153 | 154 | mkdir -p ${sshdKeysDirPathSh} 155 | mount \ 156 | -t 'virtiofs' \ 157 | -o 'nodev,noexec,nosuid,ro' \ 158 | ${sshdKeysVirtiofsTagSh} \ 159 | ${sshdKeysDirPathSh} 160 | 161 | mkdir -p "$(dirname ${sshHostPrivateKeyFilePathSh})" 162 | ( 163 | umask 'go=' 164 | cp ${sshdKeysDirPathSh}/${sshHostPrivateKeyFileNameSh} ${sshHostPrivateKeyFilePathSh} 165 | ) 166 | 167 | mkdir -p "$(dirname ${sshAuthorizedKeysUserTmpFilePathSh})" 168 | cp \ 169 | ${sshdKeysDirPathSh}/${sshUserPublicKeyFileNameSh} \ 170 | ${sshAuthorizedKeysUserTmpFilePathSh} 171 | chmod 'a+r' ${sshAuthorizedKeysUserTmpFilePathSh} 172 | 173 | umount ${sshdKeysDirPathSh} 174 | rmdir ${sshdKeysDirPathSh} 175 | 176 | # must be last so only now `unitConfig.ConditionPathExists` triggers 177 | mv ${sshAuthorizedKeysUserTmpFilePathSh} ${sshAuthorizedKeysUserFilePathSh} 178 | ''; 179 | 180 | serviceConfig.Type = "oneshot"; 181 | 182 | # see comments on this service and in its `script` 183 | unitConfig.ConditionPathExists = "!${sshAuthorizedKeysUserFilePath}"; 184 | }; 185 | 186 | users = { 187 | # console and (initial) SSH logins are purposely disabled 188 | # see: `systemd.services."${sshdKeys}"` 189 | allowNoPasswordLogin = true; 190 | 191 | mutableUsers = false; 192 | 193 | users."${linuxUser}" = { 194 | isNormalUser = true; 195 | extraGroups = optionals debugInsecurely [ "wheel" ]; 196 | }; 197 | }; 198 | 199 | virtualisation.rosetta = optionalAttrs withRosetta { 200 | enable = true; 201 | 202 | # Lima's virtiofs label for rosetta: 203 | # https://github.com/lima-vm/lima/blob/0e931107cadbcb6dbc7bbb25626f66cdbca1f040/pkg/vz/rosetta_directory_share_arm64.go#L15 204 | mountTag = "vz-rosetta"; 205 | }; 206 | } 207 | ] ++ [ potentiallyInsecureExtraNixosModule ]; 208 | system = linuxSystem; 209 | } 210 | ) 211 | --------------------------------------------------------------------------------