├── .gitignore ├── README.md ├── image.nix ├── master-config.nix ├── netboot.nix ├── qemu.nix ├── run.sh └── worker-config.nix /.gitignore: -------------------------------------------------------------------------------- 1 | result 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | A nix expression to create a netboot k8s cluster of qemu instances. 2 | 3 | If you have a bunch of underutilized workstations, run a master VM using this image. Then run worker VMs on the other workstations. 4 | The worker will simply netboot from the master, adding another node to the k8s cluster. 5 | 6 | The `run.sh` script will help you with the correct qemu commands to run master and worker. 7 | 8 | The main expression is `image.nix`, which creates the master image. 9 | It takes its config from `master-config.nix`, which incudes a netboot server serving the image defined in `worker-config.nix`. 10 | 11 | 12 | 13 | KNOWN BUGS: 14 | * k8s nodes are not yet correctly connected 15 | * nodes probably need more direct networking connection 16 | * working completely in memory will probably not be enought 17 | -------------------------------------------------------------------------------- /image.nix: -------------------------------------------------------------------------------- 1 | import { 2 | inherit (import {}) pkgs lib; 3 | inherit (import { configuration = import ./master-config.nix;}) config; 4 | diskSize = 8192; 5 | format = "qcow2"; 6 | } 7 | -------------------------------------------------------------------------------- /master-config.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | { 4 | imports = [ 5 | ./qemu.nix 6 | ./netboot.nix 7 | ]; 8 | 9 | netboot = { 10 | network.wan="ens6"; 11 | network.lan="ens5"; 12 | image=./worker-config.nix; 13 | }; 14 | 15 | users.users.root.password="root"; 16 | i18n.consoleKeyMap = "dvorak"; 17 | environment.systemPackages = [ pkgs.tcpdump ]; 18 | 19 | services.kubernetes = { 20 | roles = ["master"]; 21 | }; 22 | } 23 | -------------------------------------------------------------------------------- /netboot.nix: -------------------------------------------------------------------------------- 1 | # mostly taken from https://github.com/cleverca22/nixos-configs/blob/master/netboot_server.nix 2 | { lib, config, pkgs, ... }: 3 | 4 | 5 | let 6 | 7 | nixos_release = import (pkgs.path + "/nixos/release.nix") {}; 8 | 9 | 10 | netboot = let 11 | build = (import (pkgs.path + "/nixos/lib/eval-config.nix") { 12 | modules = [ 13 | (pkgs.path + "/nixos/modules/installer/netboot/netboot-minimal.nix") 14 | cfg.image 15 | ]; 16 | }).config.system.build; 17 | in pkgs.symlinkJoin { 18 | name = "netboot"; 19 | paths = with build; [ netbootRamdisk kernel netbootIpxeScript ]; 20 | }; 21 | 22 | 23 | ipxe' = pkgs.ipxe.overrideDerivation (drv: { 24 | installPhase = '' 25 | ${drv.installPhase} 26 | make $makeFlags bin-x86_64-efi/ipxe.efi bin-i386-efi/ipxe.efi 27 | cp -v bin-x86_64-efi/ipxe.efi $out/x86_64-ipxe.efi 28 | cp -v bin-i386-efi/ipxe.efi $out/i386-ipxe.efi 29 | ''; 30 | }); 31 | 32 | 33 | tftp_root = pkgs.runCommand "tftproot" {} '' 34 | mkdir -pv $out 35 | cp -vi ${ipxe'}/undionly.kpxe $out/undionly.kpxe 36 | cp -vi ${ipxe'}/x86_64-ipxe.efi $out/x86_64-ipxe.efi 37 | cp -vi ${ipxe'}/i386-ipxe.efi $out/i386-ipxe.efi 38 | ''; 39 | 40 | 41 | nginx_root = pkgs.runCommand "nginxroot" {} '' 42 | mkdir -pv $out 43 | cat < $out/boot.php 44 | #!ipxe 45 | chain netboot/netboot.ipxe 46 | EOF 47 | ln -sv ${netboot} $out/netboot 48 | ''; 49 | 50 | 51 | cfg = config.netboot; 52 | 53 | 54 | in { 55 | options = { 56 | netboot = { 57 | network.wan = lib.mkOption { 58 | type = lib.types.str; 59 | description = "the internet facing IF"; 60 | }; 61 | network.lan = lib.mkOption { 62 | type = lib.types.str; 63 | description = "the netboot client facing IF"; 64 | }; 65 | image = lib.mkOption { 66 | type = lib.types.path; 67 | description = "the image to serve"; 68 | }; 69 | }; 70 | }; 71 | config = { 72 | services = { 73 | nginx = { 74 | enable = true; 75 | virtualHosts = { 76 | "192.168.3.1" = { 77 | root = nginx_root; 78 | }; 79 | }; 80 | }; 81 | dhcpd4 = { 82 | interfaces = [ cfg.network.lan ]; 83 | enable = true; 84 | extraConfig = '' 85 | option arch code 93 = unsigned integer 16; 86 | subnet 192.168.3.0 netmask 255.255.255.0 { 87 | option domain-search "localnetboot"; 88 | option subnet-mask 255.255.255.0; 89 | option broadcast-address 192.168.3.255; 90 | option routers 192.168.3.1; 91 | option domain-name-servers 192.168.3.1, 8.8.8.8, 8.8.4.4; 92 | range 192.168.3.100 192.168.3.200; 93 | next-server 192.168.3.1; 94 | if exists user-class and option user-class = "iPXE" { 95 | filename "http://192.168.3.1/boot.php?mac=''${net0/mac}&asset=''${asset:uristring}&version=''${builtin/version}"; 96 | } else { 97 | if option arch = 00:07 or option arch = 00:09 { 98 | filename = "x86_64-ipxe.efi"; 99 | } else { 100 | filename = "undionly.kpxe"; 101 | } 102 | } 103 | } 104 | ''; 105 | }; 106 | tftpd = { 107 | enable = true; 108 | path = tftp_root; 109 | }; 110 | bind = { 111 | enable = true; 112 | cacheNetworks = [ "192.168.3.0/24" "127.0.0.0/8" ]; 113 | }; 114 | }; 115 | networking = { 116 | interfaces = { 117 | ${cfg.network.lan} = { 118 | ipv4.addresses = [ { address = "192.168.3.1"; prefixLength = 24; } ]; 119 | }; 120 | }; 121 | nat = { 122 | enable = true; 123 | externalInterface = cfg.network.wan; 124 | internalIPs = [ "192.168.3.0/24" ]; 125 | internalInterfaces = [ cfg.network.lan ]; 126 | }; 127 | }; 128 | }; 129 | } 130 | -------------------------------------------------------------------------------- /qemu.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | { 4 | imports = [ 5 | 6 | ]; 7 | 8 | fileSystems."/" = { 9 | device = "/dev/disk/by-label/nixos"; 10 | autoResize = true; 11 | }; 12 | 13 | boot.growPartition = true; 14 | boot.loader.grub.device = "/dev/vda"; 15 | boot.loader.timeout = 0; 16 | 17 | networking.firewall.enable = false; 18 | } 19 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ############################################################## 4 | # change to directory of this script to allow relative paths 5 | ############################################################## 6 | cd "$( dirname "${BASH_SOURCE[0]}" )" 7 | 8 | ############################################################## 9 | # stop on error 10 | ############################################################## 11 | set -eu 12 | 13 | master(){ 14 | ############################################################## 15 | # build image 16 | ############################################################## 17 | nix-build image.nix 18 | 19 | ############################################################## 20 | # make a writable copy of the image 21 | ############################################################## 22 | file=/tmp/nixos.qcow2 23 | cp result/nixos.qcow2 $file 24 | chmod +w $file 25 | 26 | ############################################################## 27 | # generate a random MAC address 28 | ############################################################## 29 | printf -v MAC "52:54:%02x:%02x:%02x:%02x" $(( $RANDOM & 0xff)) $(( $RANDOM & 0xff )) $(( $RANDOM & 0xff)) $(( $RANDOM & 0xff )) 30 | 31 | ############################################################## 32 | # run the image 33 | ############################################################## 34 | qemu-system-x86_64 -enable-kvm -m 4G -smp 2 \ 35 | -drive format=qcow2,file=$file \ 36 | -device virtio-net,netdev=mesh,addr=5,mac=$MAC -netdev socket,mcast=230.0.0.1:1234,id=mesh \ 37 | -device virtio-net,netdev=user,addr=6 -netdev user,id=user 38 | } 39 | 40 | worker(){ 41 | ############################################################## 42 | # run a netboot client 43 | ############################################################## 44 | qemu-system-x86_64 -enable-kvm -m 4G -smp 2 -boot n \ 45 | -device virtio-net,netdev=mesh,addr=5 -netdev socket,mcast=230.0.0.1:1234,id=mesh 46 | } 47 | 48 | 49 | ############################################################## 50 | # add "worker" to start a worker instead of the netboot master 51 | ############################################################## 52 | if [ "${1:-x}" = "worker" ]; then 53 | worker 54 | else 55 | master 56 | fi 57 | -------------------------------------------------------------------------------- /worker-config.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | { 4 | imports = [ 5 | ./qemu.nix 6 | ]; 7 | 8 | users.users.root.password="root"; 9 | i18n.consoleKeyMap = "dvorak"; 10 | 11 | services.kubernetes = { 12 | roles = ["node"]; 13 | }; 14 | } 15 | --------------------------------------------------------------------------------