├── consts.nix ├── modules ├── base.nix ├── controlplane │ ├── default.nix │ ├── scheduler.nix │ ├── controller-manager.nix │ └── apiserver.nix ├── kubernetes.nix ├── boot.nix ├── login.nix ├── autoresources.nix ├── worker │ ├── coredns.nix │ ├── default.nix │ └── flannel.nix ├── loadbalancer │ └── default.nix └── etcd.nix ├── terraform.tfvars ├── utils.nix ├── nixpkgs.nix ├── .gitignore ├── certs ├── coredns.nix ├── flannel.nix ├── etcd.nix ├── utils.nix ├── default.nix └── kubernetes.nix ├── resources.nix ├── boot └── image.nix ├── replicas ├── variables.tf └── main.tf ├── check.sh ├── LICENSE ├── shell.nix ├── hive.nix ├── main.tf └── README.md /consts.nix: -------------------------------------------------------------------------------- 1 | { 2 | virtualIP = "10.240.0.10"; 3 | } 4 | -------------------------------------------------------------------------------- /modules/base.nix: -------------------------------------------------------------------------------- 1 | { 2 | imports = [ ./boot.nix ./login.nix ]; 3 | system.stateVersion = "23.11"; 4 | } 5 | -------------------------------------------------------------------------------- /terraform.tfvars: -------------------------------------------------------------------------------- 1 | etcd_instances = 3 2 | control_plane_instances = 3 3 | worker_instances = 2 4 | load_balancer_instances = 2 5 | -------------------------------------------------------------------------------- /modules/controlplane/default.nix: -------------------------------------------------------------------------------- 1 | { ... }: { 2 | imports = [ ../kubernetes.nix ./apiserver.nix ./controller-manager.nix ./scheduler.nix ]; 3 | } 4 | -------------------------------------------------------------------------------- /utils.nix: -------------------------------------------------------------------------------- 1 | { 2 | nodeIP = r: 3 | let interface = (builtins.head r.values.network_interface); 4 | in (builtins.head interface.addresses); 5 | } 6 | -------------------------------------------------------------------------------- /nixpkgs.nix: -------------------------------------------------------------------------------- 1 | builtins.fetchGit { 2 | name = "nixos-23.11-2024-01-29"; 3 | url = "https://github.com/NixOS/nixpkgs"; 4 | ref = "refs/heads/nixos-23.11"; 5 | rev = "f4a8d6d5324c327dcc2d863eb7f3cc06ad630df4"; 6 | } 7 | -------------------------------------------------------------------------------- /modules/kubernetes.nix: -------------------------------------------------------------------------------- 1 | { 2 | deployment.keys."ca.pem" = { 3 | keyFile = ../certs/generated/kubernetes/ca.pem; 4 | destDir = "/var/lib/secrets/kubernetes"; 5 | user = "kubernetes"; 6 | }; 7 | 8 | services.kubernetes = { 9 | clusterCidr = "10.200.0.0/16"; 10 | caFile = "/var/lib/secrets/kubernetes/ca.pem"; 11 | }; 12 | } 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### Terraform ### 2 | 3 | # Local .terraform directories 4 | **/.terraform/* 5 | 6 | # .tfstate files 7 | *.tfstate 8 | *.tfstate.* 9 | 10 | # Terraform modules managed by the shell, lock just gets in the way 11 | .terraform.lock.hcl 12 | 13 | # created by the "ter" wrapper 14 | show.json 15 | 16 | ### Nix ### 17 | boot/image 18 | certs/generated 19 | -------------------------------------------------------------------------------- /certs/coredns.nix: -------------------------------------------------------------------------------- 1 | { pkgs, cfssl, kubectl }: 2 | let 3 | inherit (pkgs.callPackage ./utils.nix { }) mkCsr; 4 | 5 | corednsKubeCsr = mkCsr "coredns" { 6 | cn = "system:coredns"; 7 | }; 8 | 9 | in 10 | '' 11 | mkdir -p $out/coredns 12 | 13 | pushd $out/kubernetes > /dev/null 14 | genCert client ../coredns/coredns-kube ${corednsKubeCsr} 15 | popd > /dev/null 16 | '' 17 | -------------------------------------------------------------------------------- /certs/flannel.nix: -------------------------------------------------------------------------------- 1 | { pkgs, cfssl, kubectl }: 2 | let 3 | inherit (pkgs.callPackage ./utils.nix { }) getAltNames mkCsr; 4 | 5 | etcdClientCsr = mkCsr "etcd-client" { 6 | cn = "flannel"; 7 | altNames = getAltNames "worker"; 8 | }; 9 | 10 | in 11 | '' 12 | mkdir -p $out/flannel 13 | 14 | pushd $out/etcd > /dev/null 15 | genCert client ../flannel/etcd-client ${etcdClientCsr} 16 | popd > /dev/null 17 | '' 18 | -------------------------------------------------------------------------------- /modules/boot.nix: -------------------------------------------------------------------------------- 1 | { modulesPath, ... }: { 2 | imports = [ 3 | "${toString modulesPath}/profiles/qemu-guest.nix" 4 | ]; 5 | 6 | fileSystems."/" = { 7 | device = "/dev/disk/by-label/nixos"; 8 | fsType = "ext4"; 9 | autoResize = true; 10 | }; 11 | 12 | boot.growPartition = true; 13 | boot.kernelParams = [ "console=ttyS0" ]; 14 | boot.loader.grub.device = "/dev/vda"; 15 | boot.loader.timeout = 1; 16 | } 17 | -------------------------------------------------------------------------------- /resources.nix: -------------------------------------------------------------------------------- 1 | { lib, ... }: 2 | let 3 | payload = builtins.fromJSON (builtins.readFile ./show.json); 4 | resourcesInModule = type: module: 5 | builtins.filter (r: r.type == type) module.resources ++ 6 | lib.flatten (map (resourcesInModule type) (module.child_modules or [ ])); 7 | resourcesByType = type: resourcesInModule type payload.values.root_module; 8 | in 9 | rec { 10 | resources = resourcesByType "libvirt_domain"; 11 | resourcesByRole = role: 12 | (builtins.filter (r: lib.strings.hasPrefix role r.values.name) resources); 13 | } 14 | -------------------------------------------------------------------------------- /modules/login.nix: -------------------------------------------------------------------------------- 1 | # Allow yourself to SSH to the machines using your public key 2 | let 3 | # read the first file that exists 4 | # filenames: list of paths 5 | readFirst = filenames: builtins.readFile 6 | (builtins.head (builtins.filter builtins.pathExists filenames)); 7 | 8 | sshKey = readFirst [ ~/.ssh/id_ed25519.pub ~/.ssh/id_rsa.pub ]; 9 | in 10 | { config, ... }: 11 | { 12 | networking.firewall.allowedTCPPorts = config.services.openssh.ports; 13 | services.openssh.enable = true; 14 | users.users.root.openssh.authorizedKeys.keys = [ sshKey ]; 15 | } 16 | -------------------------------------------------------------------------------- /certs/etcd.nix: -------------------------------------------------------------------------------- 1 | { pkgs, cfssl }: 2 | let 3 | inherit (pkgs.callPackage ./utils.nix { }) getAltNames mkCsr; 4 | 5 | caCsr = mkCsr "etcd-ca" { cn = "etcd-ca"; }; 6 | serverCsr = mkCsr "etcd-server" { 7 | cn = "etcd"; 8 | altNames = getAltNames "etcd"; 9 | }; 10 | peerCsr = mkCsr "etcd-peer" { 11 | cn = "etcd-peer"; 12 | altNames = getAltNames "etcd"; 13 | }; 14 | in 15 | '' 16 | mkdir -p $out/etcd 17 | 18 | pushd $out/etcd > /dev/null 19 | 20 | genCa ${caCsr} 21 | genCert server server ${serverCsr} 22 | genCert peer peer ${peerCsr} 23 | 24 | popd > /dev/null 25 | '' 26 | -------------------------------------------------------------------------------- /boot/image.nix: -------------------------------------------------------------------------------- 1 | # Build a basic boot image to provision a NixOS virtual machine 2 | # Based on: https://gist.github.com/tarnacious/f9674436fff0efeb4bb6585c79a3b9ff 3 | { pkgs ? import (import ../nixpkgs.nix) { } }: 4 | let config = { config, lib, modulesPath, pkgs, ... }: 5 | { 6 | imports = [ 7 | ../modules/base.nix 8 | ]; 9 | 10 | config = { 11 | system.build.qcow = import "${toString modulesPath}/../lib/make-disk-image.nix" { 12 | inherit config lib pkgs; 13 | diskSize = 8192; 14 | format = "qcow2"; 15 | }; 16 | }; 17 | }; 18 | in 19 | (pkgs.nixos config).qcow 20 | -------------------------------------------------------------------------------- /replicas/variables.tf: -------------------------------------------------------------------------------- 1 | variable "memory" { 2 | type = number 3 | description = "Amount of megabytes of RAM to give to each machine" 4 | } 5 | 6 | variable "name" { 7 | type = string 8 | description = "Base name for the machine and boot volume" 9 | } 10 | 11 | variable "num_replicas" { 12 | type = number 13 | description = "Amount of machines to spawn" 14 | } 15 | 16 | variable "network_id" { 17 | type = string 18 | description = "Libvirt network to attach the machines to" 19 | } 20 | 21 | variable "volume_id" { 22 | type = string 23 | description = "ID of the volume to base the boot drive on" 24 | } 25 | -------------------------------------------------------------------------------- /modules/autoresources.nix: -------------------------------------------------------------------------------- 1 | # Automatically provide these arguments to modules: 2 | # * `self` - terraform resource for this node 3 | # i.e. libvirt_domain where resource.name == name (as defined in the hive) 4 | # * `resources` - a list of all libvirt_domain resources 5 | # * `resourcesByRole` - function that returns resources with the given prefix 6 | # 7 | # See: https://github.com/NixOS/nixpkgs/blob/f9c6dd42d98a5a55e9894d82dc6338ab717cda23/lib/modules.nix#L75-L95 8 | { lib, pkgs, name, ... }: 9 | { 10 | _module.args = rec { 11 | inherit (pkgs.callPackage ../resources.nix { }) resources resourcesByRole; 12 | self = builtins.head (builtins.filter (r: r.values.name == name) resources); 13 | }; 14 | } 15 | -------------------------------------------------------------------------------- /replicas/main.tf: -------------------------------------------------------------------------------- 1 | # Spawns the given amount of machines, 2 | # using the given base image as their root disk, 3 | # attached to the same network. 4 | 5 | terraform { 6 | required_providers { 7 | libvirt = { 8 | source = "dmacvicar/libvirt" 9 | } 10 | } 11 | } 12 | 13 | resource "libvirt_volume" "boot" { 14 | count = var.num_replicas 15 | 16 | name = "${var.name}${count.index + 1}_boot" 17 | base_volume_id = var.volume_id 18 | } 19 | 20 | resource "libvirt_domain" "node" { 21 | count = var.num_replicas 22 | 23 | name = "${var.name}${count.index + 1}" 24 | 25 | memory = var.memory 26 | 27 | disk { 28 | volume_id = libvirt_volume.boot[count.index].id 29 | } 30 | 31 | network_interface { 32 | network_id = var.network_id 33 | wait_for_lease = true 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /modules/controlplane/scheduler.nix: -------------------------------------------------------------------------------- 1 | { resourcesByRole, ... }: 2 | let 3 | inherit (import ../../consts.nix) virtualIP; 4 | in 5 | { 6 | deployment.keys = { 7 | "scheduler.pem" = { 8 | keyFile = ../../certs/generated/kubernetes/scheduler.pem; 9 | destDir = "/var/lib/secrets/kubernetes"; 10 | user = "kubernetes"; 11 | }; 12 | "scheduler-key.pem" = { 13 | keyFile = ../../certs/generated/kubernetes/scheduler-key.pem; 14 | destDir = "/var/lib/secrets/kubernetes"; 15 | user = "kubernetes"; 16 | }; 17 | }; 18 | 19 | services.kubernetes.scheduler = { 20 | enable = true; 21 | kubeconfig = { 22 | certFile = "/var/lib/secrets/kubernetes/scheduler.pem"; 23 | keyFile = "/var/lib/secrets/kubernetes/scheduler-key.pem"; 24 | server = "https://${virtualIP}"; 25 | }; 26 | }; 27 | } 28 | -------------------------------------------------------------------------------- /check.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | etcd1_ip=$(jq -r '.values.root_module.child_modules[] | .resources[] | select(.values.name == "etcd1").values.network_interface[0].addresses[0]' show.json) 5 | 6 | etcdctl --endpoints "https://$etcd1_ip:2379" \ 7 | --cacert ./certs/generated/etcd/ca.pem \ 8 | --cert ./certs/generated/etcd/peer.pem \ 9 | --key ./certs/generated/etcd/peer-key.pem \ 10 | endpoint status --cluster \ 11 | | tee /dev/stderr | grep -q 'true, false' # isLeader, isLearner 12 | 13 | k --request-timeout 1 cluster-info 14 | 15 | k run --rm --attach --restart Never \ 16 | --image busybox \ 17 | busybox \ 18 | --command id \ 19 | | tee /dev/stderr | grep -q "uid=0(root) gid=0(root) groups=0(root),10(wheel)" 20 | 21 | k run --rm --attach --restart Never \ 22 | --image busybox \ 23 | busybox \ 24 | --command nslookup kubernetes \ 25 | | tee /dev/stderr | grep -q "Address: 10.32.0.1" 26 | 27 | echo "Success." 28 | -------------------------------------------------------------------------------- /modules/controlplane/controller-manager.nix: -------------------------------------------------------------------------------- 1 | { resourcesByRole, ... }: 2 | let 3 | inherit (import ../../consts.nix) virtualIP; 4 | in 5 | { 6 | deployment.keys = { 7 | "controller-manager.pem" = { 8 | keyFile = ../../certs/generated/kubernetes/controller-manager.pem; 9 | destDir = "/var/lib/secrets/kubernetes"; 10 | user = "kubernetes"; 11 | }; 12 | "controller-manager-key.pem" = { 13 | keyFile = ../../certs/generated/kubernetes/controller-manager-key.pem; 14 | destDir = "/var/lib/secrets/kubernetes"; 15 | user = "kubernetes"; 16 | }; 17 | }; 18 | 19 | services.kubernetes.controllerManager = { 20 | enable = true; 21 | kubeconfig = { 22 | certFile = "/var/lib/secrets/kubernetes/controller-manager.pem"; 23 | keyFile = "/var/lib/secrets/kubernetes/controller-manager-key.pem"; 24 | server = "https://${virtualIP}"; 25 | }; 26 | 27 | # TODO: separate from server keys? 28 | serviceAccountKeyFile = "/var/lib/secrets/kubernetes/apiserver/server-key.pem"; 29 | }; 30 | } 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2021 Justinas Stankevicius 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /certs/utils.nix: -------------------------------------------------------------------------------- 1 | { lib, pkgs, ... }: 2 | let 3 | domain = "k8s.local"; 4 | 5 | inherit (pkgs.callPackage ../resources.nix { }) resourcesByRole; 6 | inherit (import ../utils.nix) nodeIP; 7 | 8 | writeJSONText = name: obj: pkgs.writeText "${name}.json" (builtins.toJSON obj); 9 | 10 | csrDefaults = { 11 | key = { 12 | algo = "rsa"; 13 | size = 2048; 14 | }; 15 | }; 16 | in 17 | { 18 | # Get IP/DNS alternative names for all servers of this role. 19 | # We currently use the same certificates for all replicas of a role (where possible), 20 | # so, for example, etcd certificate will have alt names: 21 | # etcd1, etcd2, etcd3, 10.240.0.xx1, 10.240.0.xx2, 10.240.0.xx3 22 | getAltNames = role: 23 | let 24 | hosts = map (r: r.values.name) (resourcesByRole role); 25 | ips = map nodeIP (resourcesByRole role); 26 | in 27 | hosts ++ (map (h: "${h}.${domain}") hosts) ++ ips; 28 | 29 | # Form a CSR request, as expected by cfssl 30 | mkCsr = name: { cn, altNames ? [ ], organization ? null }: 31 | writeJSONText name (lib.attrsets.recursiveUpdate csrDefaults { 32 | CN = cn; 33 | hosts = [ cn ] ++ altNames; 34 | names = if organization == null then null else [ 35 | { "O" = organization; } 36 | ]; 37 | }); 38 | } 39 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import (import ./nixpkgs.nix) { config.allowUnfree = true; } }: 2 | let 3 | myTerraform = pkgs.terraform.withPlugins (tp: [ tp.libvirt ]); 4 | ter = pkgs.writeShellScriptBin "ter" '' 5 | ${myTerraform}/bin/terraform $@ && \ 6 | ${myTerraform}/bin/terraform show -json > show.json 7 | ''; 8 | 9 | ci-lint = pkgs.writeShellScriptBin "ci-lint" '' 10 | echo Checking the formatting of Nix files 11 | ${pkgs.nixpkgs-fmt}/bin/nixpkgs-fmt --check **/*.nix 12 | 13 | echo 14 | 15 | echo Checking the formatting of Terraform files 16 | ${myTerraform}/bin/terraform fmt -check -recursive 17 | ''; 18 | 19 | k = pkgs.writeShellScriptBin "k" '' 20 | kubectl --kubeconfig certs/generated/kubernetes/admin.kubeconfig $@ 21 | ''; 22 | 23 | make-boot-image = pkgs.writeShellScriptBin "make-boot-image" '' 24 | nix-build -o boot/image boot/image.nix 25 | ''; 26 | 27 | make-certs = pkgs.writeShellScriptBin "make-certs" '' 28 | $(nix-build --no-out-link certs)/bin/generate-certs 29 | ''; 30 | in 31 | pkgs.mkShell { 32 | buildInputs = with pkgs; [ 33 | # software for deployment 34 | colmena 35 | jq 36 | libxslt 37 | myTerraform 38 | 39 | # software for testing 40 | etcd 41 | kubectl 42 | 43 | # scripts 44 | ci-lint 45 | k 46 | make-boot-image 47 | make-certs 48 | ter 49 | ]; 50 | } 51 | -------------------------------------------------------------------------------- /modules/worker/coredns.nix: -------------------------------------------------------------------------------- 1 | { pkgs, resourcesByRole, self, ... }: 2 | let 3 | inherit (import ../../consts.nix) virtualIP; 4 | inherit (import ../../utils.nix) nodeIP; 5 | in 6 | { 7 | deployment.keys = { 8 | "coredns-kube.pem" = { 9 | keyFile = ../../certs/generated/coredns/coredns-kube.pem; 10 | destDir = "/var/lib/secrets/coredns"; 11 | user = "coredns"; 12 | }; 13 | "coredns-kube-key.pem" = { 14 | keyFile = ../../certs/generated/coredns/coredns-kube-key.pem; 15 | destDir = "/var/lib/secrets/coredns"; 16 | user = "coredns"; 17 | }; 18 | "kube-ca.pem" = { 19 | keyFile = ../../certs/generated/kubernetes/ca.pem; 20 | destDir = "/var/lib/secrets/coredns"; 21 | user = "coredns"; 22 | }; 23 | }; 24 | 25 | services.coredns = { 26 | enable = true; 27 | config = '' 28 | .:53 { 29 | kubernetes cluster.local { 30 | endpoint https://${virtualIP} 31 | tls /var/lib/secrets/coredns/coredns-kube.pem /var/lib/secrets/coredns/coredns-kube-key.pem /var/lib/secrets/coredns/kube-ca.pem 32 | pods verified 33 | } 34 | forward . 1.1.1.1:53 1.0.0.1:53 35 | } 36 | ''; 37 | }; 38 | 39 | services.kubernetes.kubelet.clusterDns = nodeIP self; 40 | 41 | networking.firewall.interfaces.mynet.allowedTCPPorts = [ 53 ]; 42 | networking.firewall.interfaces.mynet.allowedUDPPorts = [ 53 ]; 43 | 44 | users.groups.coredns = { }; 45 | users.users.coredns = { 46 | group = "coredns"; 47 | isSystemUser = true; 48 | }; 49 | } 50 | -------------------------------------------------------------------------------- /modules/loadbalancer/default.nix: -------------------------------------------------------------------------------- 1 | { lib, resourcesByRole, self, ... }: 2 | let 3 | inherit (import ../../consts.nix) virtualIP; 4 | inherit (import ../../utils.nix) nodeIP; 5 | backends = map 6 | (r: "server ${r.values.name} ${nodeIP r}:6443") 7 | (resourcesByRole "controlplane"); 8 | in 9 | { 10 | services.haproxy = { 11 | enable = true; 12 | # TODO: backend healthchecks 13 | config = '' 14 | defaults 15 | timeout connect 10s 16 | 17 | frontend k8s 18 | mode tcp 19 | bind *:443 20 | default_backend controlplanes 21 | 22 | backend controlplanes 23 | mode tcp 24 | ${builtins.concatStringsSep "\n " backends} 25 | ''; 26 | }; 27 | 28 | services.keepalived = { 29 | enable = true; 30 | vrrpInstances.k8s = { 31 | # TODO: at least basic (hardcoded) auth or other protective measures 32 | interface = "ens3"; 33 | priority = 34 | # Prioritize loadbalancer1 over loadbalancer2 over loadbalancer3, etc. 35 | let number = lib.strings.toInt (lib.strings.removePrefix "loadbalancer" self.values.name); 36 | in 37 | 200 - number; 38 | virtualRouterId = 42; 39 | virtualIps = [ 40 | { 41 | addr = virtualIP; 42 | } 43 | ]; 44 | }; 45 | }; 46 | 47 | boot.kernel.sysctl."net.ipv4.ip_nonlocal_bind" = true; 48 | 49 | networking.firewall.allowedTCPPorts = [ 443 ]; 50 | networking.firewall.extraCommands = "iptables -A INPUT -p vrrp -j ACCEPT"; 51 | networking.firewall.extraStopCommands = "iptables -D INPUT -p vrrp -j ACCEPT || true"; 52 | } 53 | -------------------------------------------------------------------------------- /modules/etcd.nix: -------------------------------------------------------------------------------- 1 | { lib, resources, resourcesByRole, self, ... }: 2 | let 3 | inherit (import ../utils.nix) nodeIP; 4 | etcds = resourcesByRole "etcd"; 5 | cluster = map (r: "${r.values.name}=https://${nodeIP r}:2380") etcds; 6 | 7 | mkSecret = filename: { 8 | keyFile = ../certs/generated/etcd + "/${filename}"; 9 | destDir = "/var/lib/secrets/etcd"; 10 | user = "etcd"; 11 | }; 12 | in 13 | { 14 | deployment.keys = { 15 | "ca.pem" = mkSecret "ca.pem"; 16 | "peer.pem" = mkSecret "peer.pem"; 17 | "peer-key.pem" = mkSecret "peer-key.pem"; 18 | "server.pem" = mkSecret "server.pem"; 19 | "server-key.pem" = mkSecret "server-key.pem"; 20 | }; 21 | 22 | networking.firewall.allowedTCPPorts = [ 2379 2380 ]; 23 | 24 | services.etcd = { 25 | enable = true; 26 | 27 | advertiseClientUrls = [ "https://${nodeIP self}:2379" ]; 28 | initialAdvertisePeerUrls = [ "https://${nodeIP self}:2380" ]; 29 | initialCluster = lib.mkForce cluster; 30 | listenClientUrls = [ "https://${nodeIP self}:2379" "https://127.0.0.1:2379" ]; 31 | listenPeerUrls = [ "https://${nodeIP self}:2380" "https://127.0.0.1:2380" ]; 32 | 33 | clientCertAuth = true; 34 | peerClientCertAuth = true; 35 | 36 | certFile = "/var/lib/secrets/etcd/server.pem"; 37 | keyFile = "/var/lib/secrets/etcd/server-key.pem"; 38 | 39 | peerCertFile = "/var/lib/secrets/etcd/peer.pem"; 40 | peerKeyFile = "/var/lib/secrets/etcd/peer-key.pem"; 41 | 42 | peerTrustedCaFile = "/var/lib/secrets/etcd/ca.pem"; 43 | trustedCaFile = "/var/lib/secrets/etcd/ca.pem"; 44 | }; 45 | 46 | systemd.services.etcd = { 47 | wants = [ "network-online.target" ]; 48 | after = [ "network-online.target" ]; 49 | }; 50 | } 51 | -------------------------------------------------------------------------------- /hive.nix: -------------------------------------------------------------------------------- 1 | let 2 | pkgs = import (import ./nixpkgs.nix) { }; 3 | lib = pkgs.lib; 4 | 5 | inherit (pkgs.callPackage ./resources.nix { }) resources resourcesByRole; 6 | inherit (import ./utils.nix) nodeIP; 7 | 8 | etcdHosts = map (r: r.values.name) (resourcesByRole "etcd"); 9 | controlPlaneHosts = map (r: r.values.name) (resourcesByRole "controlplane"); 10 | workerHosts = map (r: r.values.name) (resourcesByRole "worker"); 11 | loadBalancerHosts = map (r: r.values.name) (resourcesByRole "loadbalancer"); 12 | 13 | etcdConf = { ... }: { 14 | imports = [ ./modules/etcd.nix ]; 15 | deployment.tags = [ "etcd" ]; 16 | }; 17 | 18 | controlPlaneConf = { ... }: { 19 | imports = [ ./modules/controlplane ]; 20 | deployment.tags = [ "controlplane" ]; 21 | }; 22 | 23 | workerConf = { ... }: { 24 | imports = [ ./modules/worker ]; 25 | deployment.tags = [ "worker" ]; 26 | }; 27 | 28 | loadBalancerConf = { ... }: { 29 | imports = [ ./modules/loadbalancer ]; 30 | deployment.tags = [ "loadbalancer" ]; 31 | }; 32 | in 33 | { 34 | meta = { 35 | nixpkgs = import (import ./nixpkgs.nix); 36 | }; 37 | 38 | defaults = { name, self, ... }: { 39 | imports = [ 40 | ./modules/autoresources.nix 41 | ./modules/base.nix 42 | ]; 43 | 44 | deployment.targetHost = nodeIP self; 45 | networking.hostName = name; 46 | }; 47 | } 48 | // builtins.listToAttrs (map (h: { name = h; value = etcdConf; }) etcdHosts) 49 | // builtins.listToAttrs (map (h: { name = h; value = controlPlaneConf; }) controlPlaneHosts) 50 | // builtins.listToAttrs (map (h: { name = h; value = loadBalancerConf; }) loadBalancerHosts) 51 | // builtins.listToAttrs (map (h: { name = h; value = workerConf; }) workerHosts) 52 | -------------------------------------------------------------------------------- /modules/worker/default.nix: -------------------------------------------------------------------------------- 1 | { config, name, resourcesByRole, ... }: 2 | let 3 | inherit (import ../../consts.nix) virtualIP; 4 | in 5 | { 6 | imports = [ ../kubernetes.nix ./coredns.nix ./flannel.nix ]; 7 | 8 | deployment.keys = { 9 | "kubelet.pem" = { 10 | keyFile = ../../certs/generated/kubernetes/kubelet + "/${name}.pem"; 11 | destDir = "/var/lib/secrets/kubernetes"; 12 | user = "kubernetes"; 13 | }; 14 | 15 | "kubelet-key.pem" = { 16 | keyFile = ../../certs/generated/kubernetes/kubelet + "/${name}-key.pem"; 17 | destDir = "/var/lib/secrets/kubernetes"; 18 | user = "kubernetes"; 19 | }; 20 | 21 | "proxy.pem" = { 22 | keyFile = ../../certs/generated/kubernetes/proxy.pem; 23 | destDir = "/var/lib/secrets/kubernetes"; 24 | user = "kubernetes"; 25 | }; 26 | 27 | "proxy-key.pem" = { 28 | keyFile = ../../certs/generated/kubernetes/proxy-key.pem; 29 | destDir = "/var/lib/secrets/kubernetes"; 30 | user = "kubernetes"; 31 | }; 32 | }; 33 | 34 | networking.firewall.allowedTCPPorts = [ 35 | config.services.kubernetes.kubelet.port 36 | ]; 37 | 38 | services.kubernetes.kubelet = rec { 39 | enable = true; 40 | unschedulable = false; 41 | kubeconfig = rec { 42 | certFile = tlsCertFile; 43 | keyFile = tlsKeyFile; 44 | server = "https://${virtualIP}"; 45 | }; 46 | tlsCertFile = "/var/lib/secrets/kubernetes/kubelet.pem"; 47 | tlsKeyFile = "/var/lib/secrets/kubernetes/kubelet-key.pem"; 48 | }; 49 | 50 | services.kubernetes.proxy = { 51 | enable = true; 52 | kubeconfig = { 53 | certFile = "/var/lib/secrets/kubernetes/proxy.pem"; 54 | keyFile = "/var/lib/secrets/kubernetes/proxy-key.pem"; 55 | server = "https://${virtualIP}"; 56 | }; 57 | }; 58 | } 59 | -------------------------------------------------------------------------------- /modules/worker/flannel.nix: -------------------------------------------------------------------------------- 1 | { config, lib, resourcesByRole, ... }: 2 | let 3 | etcdServers = map (r: "https://${r.values.name}:2379") (resourcesByRole "etcd"); 4 | in 5 | { 6 | deployment.keys = { 7 | "etcd-ca.pem" = { 8 | keyFile = ../../certs/generated/etcd/ca.pem; 9 | destDir = "/var/lib/secrets/flannel"; 10 | }; 11 | "etcd-client.pem" = { 12 | keyFile = ../../certs/generated/flannel/etcd-client.pem; 13 | destDir = "/var/lib/secrets/flannel"; 14 | }; 15 | "etcd-client-key.pem" = { 16 | keyFile = ../../certs/generated/flannel/etcd-client-key.pem; 17 | destDir = "/var/lib/secrets/flannel"; 18 | }; 19 | }; 20 | 21 | # https://github.com/NixOS/nixpkgs/blob/145084f62b6341fc4300ba3f8eb244d594168e9d/nixos/modules/services/cluster/kubernetes/flannel.nix#L41-L47 22 | networking = { 23 | dhcpcd.denyInterfaces = [ "mynet*" "flannel*" ]; 24 | firewall.allowedUDPPorts = [ 25 | 8285 # flannel udp 26 | 8472 # flannel vxlan 27 | ]; 28 | }; 29 | 30 | services.flannel = { 31 | enable = true; 32 | network = config.services.kubernetes.clusterCidr; 33 | 34 | storageBackend = "etcd"; # TODO: reconsider 35 | etcd = { 36 | endpoints = etcdServers; 37 | 38 | caFile = "/var/lib/secrets/flannel/etcd-ca.pem"; 39 | certFile = "/var/lib/secrets/flannel/etcd-client.pem"; 40 | keyFile = "/var/lib/secrets/flannel/etcd-client-key.pem"; 41 | }; 42 | }; 43 | 44 | # systemd.network module makes this true by default, however: 45 | # https://github.com/NixOS/nixpkgs/issues/114118 46 | services.resolved.enable = false; 47 | 48 | services.kubernetes.kubelet = { 49 | cni.config = [{ 50 | name = "mynet"; 51 | type = "flannel"; 52 | cniVersion = "0.3.1"; 53 | delegate = { 54 | isDefaultGateway = true; 55 | bridge = "mynet"; 56 | }; 57 | }]; 58 | }; 59 | } 60 | -------------------------------------------------------------------------------- /certs/default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import (import ../nixpkgs.nix) { } 2 | , cfssl ? pkgs.cfssl 3 | }: 4 | let 5 | caConfig = pkgs.writeText "ca-config.json" '' 6 | { 7 | "signing": { 8 | "profiles": { 9 | "client": { 10 | "expiry": "87600h", 11 | "usages": ["signing", "key encipherment", "client auth"] 12 | }, 13 | "peer": { 14 | "expiry": "87600h", 15 | "usages": ["signing", "key encipherment", "client auth", "server auth"] 16 | }, 17 | "server": { 18 | "expiry": "8760h", 19 | "usages": ["signing", "key encipherment", "client auth", "server auth"] 20 | } 21 | } 22 | } 23 | } 24 | ''; 25 | in 26 | # Only use Nix to generate the certificate generation ( :) ) script. 27 | # That way, we avoid certificates ending up in the world-readable Nix store 28 | pkgs.writeShellScriptBin "generate-certs" '' 29 | set -e 30 | 31 | # Generates a CA, if one does not exist, in the current directory. 32 | function genCa() { 33 | csrjson=$1 34 | [ -n "$csrjson" ] || { echo "Usage: genCa CSRJSON" && return 1; } 35 | [ -f ca.pem ] && { echo "$(realpath ca.pem) exists, not replacing the CA" && return 0; } 36 | ${cfssl}/bin/cfssl gencert -loglevel 2 -initca "$csrjson" | ${cfssl}/bin/cfssljson -bare ca 37 | } 38 | 39 | # Generates a certificate signed by ca.pem from the current directory 40 | # (convention over configuration). 41 | function genCert() { 42 | profile=$1 43 | output=$2 # e.g. `apiserver/client` will result in `apiserver/client.pem` and `apiserver/client-key.pem` 44 | csrjson=$3 45 | 46 | { [ -n "$profile" ] && [ -n "$output" ] && [ -n "$csrjson" ]; } \ 47 | || { echo "Usage: genCert PROFILE OUTPUT CSRJSON" && return 1; } 48 | 49 | ${cfssl}/bin/cfssl gencert \ 50 | -loglevel 2 \ 51 | -ca ca.pem \ 52 | -ca-key ca-key.pem \ 53 | -config ${caConfig} \ 54 | -profile "$profile" \ 55 | "$csrjson" \ 56 | | ${cfssl}/bin/cfssljson -bare "$output" 57 | } 58 | 59 | 60 | out=./certs/generated 61 | 62 | ${pkgs.callPackage ./etcd.nix { }} 63 | ${pkgs.callPackage ./kubernetes.nix { }} 64 | ${pkgs.callPackage ./coredns.nix { }} 65 | ${pkgs.callPackage ./flannel.nix { }} 66 | '' 67 | -------------------------------------------------------------------------------- /main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | libvirt = { 4 | source = "dmacvicar/libvirt" 5 | } 6 | } 7 | } 8 | 9 | provider "libvirt" { 10 | uri = "qemu:///system" 11 | } 12 | 13 | resource "libvirt_network" "k8s" { 14 | name = "k8s" 15 | mode = "nat" 16 | domain = "k8s.local" 17 | addresses = ["10.240.0.0/24"] 18 | 19 | dns { 20 | enabled = true 21 | } 22 | 23 | xml { 24 | # By default, DHCP range is the whole subnet. 25 | # We will eventually want virtual IPs, so try to make space for them. 26 | # XSLT (I have no idea what I'm doing), 27 | # because of https://github.com/dmacvicar/terraform-provider-libvirt/issues/794 28 | xslt = < 30 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 10.240.0.100 42 | 43 | 44 | 45 | 46 | EOF 47 | } 48 | } 49 | 50 | resource "libvirt_volume" "nixos_boot" { 51 | name = "boot" 52 | source = "boot/image/nixos.qcow2" 53 | } 54 | 55 | module "replicas" { 56 | for_each = { 57 | "etcd" : { 58 | "count" : var.etcd_instances, 59 | "memory" : 512, 60 | } 61 | "controlplane" : { 62 | "count" : var.control_plane_instances, 63 | "memory" : 512, 64 | } 65 | "worker" : { 66 | "count" : var.worker_instances, 67 | "memory" : 1024, 68 | } 69 | "loadbalancer" : { 70 | "count" : var.load_balancer_instances, 71 | "memory" : 512, 72 | } 73 | } 74 | 75 | source = "./replicas" 76 | 77 | name = each.key 78 | num_replicas = each.value.count 79 | memory = each.value.memory 80 | network_id = libvirt_network.k8s.id 81 | volume_id = libvirt_volume.nixos_boot.id 82 | } 83 | 84 | variable "etcd_instances" { 85 | type = number 86 | description = "Amount of etcd hosts to spawn" 87 | } 88 | 89 | variable "control_plane_instances" { 90 | type = number 91 | description = "Amount of control plane hosts to spawn" 92 | } 93 | 94 | variable "worker_instances" { 95 | type = number 96 | description = "Amount of worker hosts to spawn" 97 | } 98 | 99 | variable "load_balancer_instances" { 100 | type = number 101 | description = "Amount of control plane load balancer hosts to spawn" 102 | } 103 | -------------------------------------------------------------------------------- /modules/controlplane/apiserver.nix: -------------------------------------------------------------------------------- 1 | { lib, resourcesByRole, ... }: 2 | let 3 | etcdServers = map (r: "https://${r.values.name}:2379") (resourcesByRole "etcd"); 4 | 5 | mkSecret = filename: { 6 | keyFile = ../../certs/generated/kubernetes/apiserver + "/${filename}"; 7 | destDir = "/var/lib/secrets/kubernetes/apiserver"; 8 | user = "kubernetes"; 9 | }; 10 | 11 | corednsPolicies = map 12 | (r: { 13 | apiVersion = "abac.authorization.kubernetes.io/v1beta1"; 14 | kind = "Policy"; 15 | spec = { 16 | user = "system:coredns"; 17 | namespace = "*"; 18 | resource = r; 19 | readonly = true; 20 | }; 21 | }) [ "endpoints" "services" "pods" "namespaces" ] 22 | ++ lib.singleton 23 | { 24 | apiVersion = "abac.authorization.kubernetes.io/v1beta1"; 25 | kind = "Policy"; 26 | spec = { 27 | user = "system:coredns"; 28 | namespace = "*"; 29 | resource = "endpointslices"; 30 | apiGroup = "discovery.k8s.io"; 31 | readonly = true; 32 | }; 33 | }; 34 | in 35 | { 36 | deployment.keys = { 37 | "server.pem" = mkSecret "server.pem"; 38 | "server-key.pem" = mkSecret "server-key.pem"; 39 | 40 | "kubelet-client.pem" = mkSecret "kubelet-client.pem"; 41 | "kubelet-client-key.pem" = mkSecret "kubelet-client-key.pem"; 42 | 43 | "etcd-ca.pem" = { 44 | keyFile = ../../certs/generated/etcd/ca.pem; 45 | destDir = "/var/lib/secrets/kubernetes/apiserver"; 46 | user = "kubernetes"; 47 | }; 48 | "etcd-client.pem" = mkSecret "etcd-client.pem"; 49 | "etcd-client-key.pem" = mkSecret "etcd-client-key.pem"; 50 | }; 51 | 52 | networking.firewall.allowedTCPPorts = [ 6443 ]; 53 | 54 | services.kubernetes.apiserver = { 55 | enable = true; 56 | serviceClusterIpRange = "10.32.0.0/24"; 57 | 58 | # Using ABAC for CoreDNS running outside of k8s 59 | # is more simple in this case than using kube-addon-manager 60 | authorizationMode = [ "RBAC" "Node" "ABAC" ]; 61 | authorizationPolicy = corednsPolicies; 62 | 63 | etcd = { 64 | servers = etcdServers; 65 | caFile = "/var/lib/secrets/kubernetes/apiserver/etcd-ca.pem"; 66 | certFile = "/var/lib/secrets/kubernetes/apiserver/etcd-client.pem"; 67 | keyFile = "/var/lib/secrets/kubernetes/apiserver/etcd-client-key.pem"; 68 | }; 69 | 70 | kubeletClientCertFile = "/var/lib/secrets/kubernetes/apiserver/kubelet-client.pem"; 71 | kubeletClientKeyFile = "/var/lib/secrets/kubernetes/apiserver/kubelet-client-key.pem"; 72 | 73 | # TODO: separate from server keys 74 | serviceAccountKeyFile = "/var/lib/secrets/kubernetes/apiserver/server.pem"; 75 | serviceAccountSigningKeyFile = "/var/lib/secrets/kubernetes/apiserver/server-key.pem"; 76 | 77 | tlsCertFile = "/var/lib/secrets/kubernetes/apiserver/server.pem"; 78 | tlsKeyFile = "/var/lib/secrets/kubernetes/apiserver/server-key.pem"; 79 | }; 80 | } 81 | -------------------------------------------------------------------------------- /certs/kubernetes.nix: -------------------------------------------------------------------------------- 1 | { lib, pkgs, cfssl, kubectl }: 2 | let 3 | inherit (pkgs.callPackage ../resources.nix { }) resourcesByRole; 4 | inherit (import ../consts.nix) virtualIP; 5 | inherit (import ../utils.nix) nodeIP; 6 | inherit (pkgs.callPackage ./utils.nix { }) getAltNames mkCsr; 7 | 8 | caCsr = mkCsr "kubernetes-ca" { 9 | cn = "kubernetes-ca"; 10 | }; 11 | 12 | apiServerCsr = mkCsr "kube-api-server" { 13 | cn = "kubernetes"; 14 | altNames = 15 | lib.singleton virtualIP ++ 16 | # Alternative names remain, as they might be useful for debugging purposes 17 | getAltNames "controlplane" ++ 18 | getAltNames "loadbalancer" ++ 19 | lib.singleton "10.32.0.1" ++ 20 | [ "kubernetes" "kubernetes.default" "kubernetes.default.svc" "kubernetes.default.svc.cluster" "kubernetes.svc.cluster.local" ]; 21 | }; 22 | 23 | apiServerKubeletClientCsr = mkCsr "kube-api-server-kubelet-client" { 24 | cn = "kube-api-server"; 25 | altNames = getAltNames "controlplane"; 26 | organization = "system:masters"; 27 | }; 28 | 29 | cmCsr = mkCsr "kube-controller-manager" { 30 | cn = "system:kube-controller-manager"; 31 | organization = "system:kube-controller-manager"; 32 | }; 33 | 34 | adminCsr = mkCsr "admin" { 35 | cn = "admin"; 36 | organization = "system:masters"; 37 | }; 38 | 39 | etcdClientCsr = mkCsr "etcd-client" { 40 | cn = "kubernetes"; 41 | altNames = getAltNames "controlplane"; 42 | }; 43 | 44 | proxyCsr = mkCsr "kube-proxy" { 45 | cn = "system:kube-proxy"; 46 | organization = "system:node-proxier"; 47 | }; 48 | 49 | schedulerCsr = mkCsr "kube-scheduler" rec { 50 | cn = "system:kube-scheduler"; 51 | organization = cn; 52 | }; 53 | 54 | workerCsrs = map 55 | (r: { 56 | name = r.values.name; 57 | csr = mkCsr r.values.name { 58 | cn = "system:node:${r.values.name}"; 59 | organization = "system:nodes"; 60 | # TODO: unify with getAltNames? 61 | altNames = [ r.values.name (nodeIP r) ]; 62 | }; 63 | }) 64 | (resourcesByRole "worker"); 65 | 66 | workerScripts = map (csr: "genCert peer kubelet/${csr.name} ${csr.csr}") workerCsrs; 67 | in 68 | '' 69 | mkdir -p $out/kubernetes/{apiserver,controller-manager,kubelet} 70 | 71 | pushd $out/etcd > /dev/null 72 | genCert client ../kubernetes/apiserver/etcd-client ${etcdClientCsr} 73 | popd > /dev/null 74 | 75 | pushd $out/kubernetes > /dev/null 76 | 77 | genCa ${caCsr} 78 | genCert server apiserver/server ${apiServerCsr} 79 | genCert server apiserver/kubelet-client ${apiServerKubeletClientCsr} 80 | genCert client controller-manager ${cmCsr} 81 | genCert client proxy ${proxyCsr} 82 | genCert client scheduler ${schedulerCsr} 83 | genCert client admin ${adminCsr} 84 | 85 | ${builtins.concatStringsSep "\n" workerScripts} 86 | 87 | ${kubectl}/bin/kubectl --kubeconfig admin.kubeconfig config set-credentials admin \ 88 | --client-certificate=admin.pem \ 89 | --client-key=admin-key.pem 90 | ${kubectl}/bin/kubectl --kubeconfig admin.kubeconfig config set-cluster virt \ 91 | --certificate-authority=ca.pem \ 92 | --server=https://${virtualIP} 93 | ${kubectl}/bin/kubectl --kubeconfig admin.kubeconfig config set-context virt \ 94 | --user admin \ 95 | --cluster virt 96 | ${kubectl}/bin/kubectl --kubeconfig admin.kubeconfig config use-context virt > /dev/null 97 | 98 | popd > /dev/null 99 | '' 100 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Toy highly-available Kubernetes cluster on NixOS 2 | 3 | 4 | 5 | * [About](#about) 6 | * [Motivation](#motivation) 7 | * [Architecture](#architecture) 8 | * [Goals](#goals) 9 | * [Non-goals](#non-goals) 10 | * [Trying it out](#trying-it-out) 11 | * [Prerequisites](#prerequisites) 12 | * [Running](#running) 13 | * [Verifying](#verifying) 14 | * [Modifying](#modifying) 15 | * [Destroying](#destroying) 16 | * [Tips and tricks](#tips-and-tricks) 17 | * [Contributing](#contributing) 18 | * [Acknowledgements](#acknowledgements) 19 | 20 | 21 | 22 | ## About 23 | 24 | A recipe for a cluster of virtual machines managed by [Terraform](https://www.terraform.io/), 25 | running a highly-available Kubernetes cluster, 26 | deployed on NixOS using [Colmena](https://github.com/zhaofengli/colmena). 27 | 28 | ### Motivation 29 | 30 | NixOS provides a Kubernetes module, which is capable of running a `master` or `worker` node. 31 | The module even provides basic PKI, making running simple clusters easy. 32 | However, HA support is limited (see, for example, 33 | [this comment](https://github.com/NixOS/nixpkgs/blob/acab4d1d4dff1e1bbe95af639fdc6294363cce66/nixos/modules/services/cluster/kubernetes/pki.nix#L329) 34 | and an [empty section](https://nixos.wiki/wiki/Kubernetes#N_Masters_.28HA.29) 35 | for "N masters" in NixOS wiki). 36 | 37 | This project serves as an example of using the NixOS Kubernetes module in an advanced way, 38 | setting up a cluster that is highly-available on all levels. 39 | 40 | ### Architecture 41 | 42 | External etcd topology, 43 | [as described by Kubernetes docs](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#external-etcd-topology), 44 | is implemented. 45 | The cluster consists of: 46 | * 3 `etcd` nodes 47 | * 3 `controlplane` nodes, running 48 | `kube-apiserver`, `kube-controller-manager`, and `kube-scheduler`. 49 | * 2 `worker` nodes, running `kubelet`, `kube-proxy`, 50 | `coredns`, and a CNI network (currently `flannel`). 51 | * 2 `loadbalancer` nodes, running `keepalived` and `haproxy`, 52 | which proxies to the Kubernetes API. 53 | 54 | ### Goals 55 | * All infrastructure declaratively managed by Terraform and Nix (Colmena). 56 | Zero `kubectl apply -f foo.yaml` invocations required to get a functional cluster. 57 | * All the infrastructure-level services run directly on NixOS / systemd. 58 | Running `k get pods -A` after the cluster is spun up lists zero pods. 59 | * Functionality. The cluster should be able to run basic real-life deployments, 60 | although 100% parity with high-profile Kubernetes distributions is unlikely to be reached. 61 | * High-availability. 62 | A failure of a single service (of any kind) or a single machine (of any role) 63 | shall not leave the cluster in a non-functional state. 64 | 65 | ### Non-goals 66 | * Production-readiness. I am not an expert in any of: Nix, Terraform, Kubernetes, HA, etc. 67 | * Perfect security (see the above point). 68 | Some basic measures are taken: NixOS firewall is left turned on 69 | (although some overly permissive rules may be in place), 70 | Kubernetes uses ABAC and RBAC, 71 | and TLS auth is used between the services. 72 | 73 | ## Trying it out 74 | 75 | ### Prerequisites 76 | 77 | * Nix (only tested on NixOS, might work on other Linux distros). 78 | * Libvirtd running. For NixOS, put this in your config: 79 | ```nix 80 | { 81 | virtualisation.libvirtd.enable = true; 82 | users.users."yourname".extraGroups = [ "libvirtd" ]; 83 | } 84 | ``` 85 | * At least 6 GB of available RAM. 86 | * At least 15 GB of available disk space. 87 | * `10.240.0.0/24` IPv4 subnet available (as in, not used for your home network or similar). 88 | This is used by the "physical" network of the VMs. 89 | 90 | ### Running 91 | 92 | ```console 93 | $ nix-shell 94 | $ make-boot-image # Build the base NixOS image to boot VMs from 95 | $ ter init # Initialize terraform modules 96 | $ ter apply # Create the virtual machines 97 | $ make-certs # Generate TLS certificates for Kubernetes, etcd, and other daemons. 98 | $ colmena apply # Deploy to your cluster 99 | ``` 100 | 101 | Most of the steps can take several minutes each when running for the first time. 102 | 103 | ### Verifying 104 | 105 | ```console 106 | $ ./check.sh # Prints out diagnostic information about the cluster and tries to run a simple pod. 107 | $ k run --image nginx nginx # Run a simple pod. `k` is an alias of `kubectl` that uses the generated admin credentials. 108 | ``` 109 | 110 | ### Modifying 111 | 112 | The number of servers of each role can be changed by editing `terraform.tfvars` 113 | and issuing the following commands afterwards: 114 | 115 | ```console 116 | $ ter apply # Spin up or spin down machines 117 | $ make-certs # Regenerate the certs, as they are tied to machine IPs/hostnames 118 | $ colmena apply # Redeploy 119 | ``` 120 | 121 | ### Destroying 122 | 123 | ```console 124 | $ ter destroy # Destroy the virtual machines 125 | $ rm boot/image # Destroy the base image 126 | ``` 127 | 128 | ### Tips and tricks 129 | 130 | * After creating and destroying the cluster many times, your `.ssh/known_hosts` 131 | will get polluted with many entries with the virtual machine IPs. 132 | Due to this, you are likely to run into a "host key mismatch" errors while deploying. 133 | I use `:g/^10.240.0./d` in Vim to clean it up. 134 | You can probably do the same with `sed` or similar software of your choice. 135 | 136 | ## Contributing 137 | 138 | Contributions are welcome, although I might reject any that conflict with the project goals. 139 | See [TODOs](https://github.com/justinas/nixos-ha-kubernetes/search?q=TODO) in the repo 140 | for some rough edges you could work on. 141 | 142 | Make sure the `ci-lint` script succeeds. 143 | Make sure the `check.sh` script succeeds after a deploying a fresh cluster. 144 | 145 | ## Acknowledgements 146 | 147 | Both [Kubernetes The Hard Way](https://github.com/kelseyhightower/kubernetes-the-hard-way) 148 | and [Kubernetes The Hard Way on Bare Metal](https://github.com/Praqma/LearnKubernetes/blob/master/kamran/Kubernetes-The-Hard-Way-on-BareMetal.md) 149 | helped me immensely in this project. 150 | --------------------------------------------------------------------------------