├── configuration.nix ├── hardware-configuration.nix ├── common ├── bluetooth.nix ├── sub │ ├── ceph-module.nix │ ├── base-if-module.nix │ ├── fs-root-module.nix │ ├── base-fs-module.nix │ ├── constants.nix │ ├── base-dns-module.nix │ ├── vpn.nix │ ├── base-ssh.nix │ ├── base-hosts.nix │ ├── base-dnsmasq.nix │ ├── base-unbound.nix │ ├── base-ntpd.nix │ ├── base-firewall.nix │ ├── base-networking.nix │ ├── calculated.nix │ └── base-keepalived-module.nix ├── syncthing.nix ├── dyndns.nix ├── ceph.mount.nix ├── sshd.nix ├── ipfs.nix ├── xfs-root.nix ├── fs-root.nix ├── btrfs-root.nix ├── bcachefs-root.nix ├── ceph.nix ├── ddns.nix ├── tinc.nix ├── zfs-root.nix ├── base.nix ├── consul.nix └── wireguard.nix ├── services ├── etcd.nix ├── sub │ └── postgresql-module.nix ├── postgresql.nix ├── quassel.nix ├── sydent.nix ├── murmur.nix └── authdns.nix ├── gateway ├── base.nix ├── ntp.nix ├── dns.nix ├── sub │ ├── base-conntrack.nix │ ├── base-firewall.nix │ └── base-keepalived.nix ├── upnp.nix ├── load-balancer.nix └── dhcpd.nix ├── graphical ├── intel.nix ├── nvidia.nix └── base.nix ├── web ├── sub │ ├── acme-settings.nix │ └── ssl-settings.nix ├── hydra.nix ├── www-wak-io.nix ├── pub.nix ├── mumble.nix ├── triton-cache.nix ├── www-wkennington-com.nix ├── unifi.nix ├── base.nix ├── ipfs.nix ├── ceph.nix └── consul.nix ├── core ├── sub │ └── ctdbd.module.nix ├── glusterfs.nix ├── ipfs-cluster.nix ├── ceph.mds.nix ├── mongodb.nix ├── mesos.nix ├── zookeeper.nix ├── ceph.mon.nix ├── ceph.mgr.nix ├── ctdbd.nix └── ceph.osd.nix ├── scripts ├── zfs-root-mount └── zfs-root-provision ├── customization ├── tinc.nix ├── wireguard.nix └── vars.nix ├── laptop └── base.nix ├── README.md └── nas └── samba.nix /configuration.nix: -------------------------------------------------------------------------------- 1 | /conf/nixos-local/configuration.nix -------------------------------------------------------------------------------- /hardware-configuration.nix: -------------------------------------------------------------------------------- 1 | /conf/nixos-local/hardware-configuration.nix -------------------------------------------------------------------------------- /common/bluetooth.nix: -------------------------------------------------------------------------------- 1 | { ... }: 2 | { 3 | hardware.bluetooth.enable = true; 4 | } 5 | -------------------------------------------------------------------------------- /services/etcd.nix: -------------------------------------------------------------------------------- 1 | { ... }: 2 | { 3 | services.etcd = { 4 | enable = true; 5 | }; 6 | } 7 | -------------------------------------------------------------------------------- /common/sub/ceph-module.nix: -------------------------------------------------------------------------------- 1 | { lib, ... }: 2 | with lib; 3 | { 4 | options = { 5 | cephPackage = mkOption { }; 6 | }; 7 | } 8 | -------------------------------------------------------------------------------- /services/sub/postgresql-module.nix: -------------------------------------------------------------------------------- 1 | { lib, ... }: 2 | with lib; 3 | { 4 | options = { 5 | postgresqlPackage = mkOption { }; 6 | }; 7 | } 8 | -------------------------------------------------------------------------------- /gateway/base.nix: -------------------------------------------------------------------------------- 1 | { ... }: 2 | { 3 | imports = [ 4 | ./sub/base-keepalived.nix 5 | ./sub/base-conntrack.nix 6 | ./sub/base-firewall.nix 7 | ]; 8 | } 9 | -------------------------------------------------------------------------------- /common/sub/base-if-module.nix: -------------------------------------------------------------------------------- 1 | { lib, ... }: 2 | with lib; 3 | { 4 | options = { 5 | myNatIfs = mkOption { 6 | type = types.listOf types.str; 7 | default = [ ]; 8 | }; 9 | }; 10 | } 11 | -------------------------------------------------------------------------------- /common/sub/fs-root-module.nix: -------------------------------------------------------------------------------- 1 | { lib, ... }: 2 | 3 | with lib; 4 | { 5 | options = { 6 | 7 | serialConsole = mkOption { 8 | type = types.nullOr types.int; 9 | }; 10 | 11 | }; 12 | } 13 | -------------------------------------------------------------------------------- /common/sub/base-fs-module.nix: -------------------------------------------------------------------------------- 1 | { lib, ... }: 2 | with lib; 3 | { 4 | options = { 5 | rootUUID = mkOption { 6 | default = null; 7 | type = types.nullOr types.str; 8 | }; 9 | }; 10 | } 11 | -------------------------------------------------------------------------------- /common/sub/constants.nix: -------------------------------------------------------------------------------- 1 | { ... }: 2 | { 3 | privateIp4 = [ 4 | "127.0.0.0/8" 5 | "192.168.0.0/16" 6 | "172.16.0.0/12" 7 | "10.0.0.0/8" 8 | "100.64.0.0/10" 9 | ]; 10 | privateIp6 = [ 11 | "fc00::/7" 12 | ]; 13 | } 14 | -------------------------------------------------------------------------------- /graphical/intel.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, ... }: 2 | { 3 | imports = [ ./base.nix ]; 4 | services.kmscon.enable = false; 5 | services.xserver = { 6 | vaapiDrivers = [ pkgs.intel-vaapi-driver ]; 7 | videoDrivers = [ "modesetting" ]; 8 | }; 9 | } 10 | -------------------------------------------------------------------------------- /web/sub/acme-settings.nix: -------------------------------------------------------------------------------- 1 | { lib, ... }: 2 | 3 | with lib; 4 | { 5 | options = { 6 | 7 | acmeServers = mkOption { 8 | type = types.listOf types.string; 9 | default = [ 10 | "localhost:81" 11 | ]; 12 | }; 13 | 14 | }; 15 | } 16 | -------------------------------------------------------------------------------- /graphical/nvidia.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, ... }: 2 | { 3 | imports = [ ./base.nix ]; 4 | nixpkgs.config.allowUnfree = true; 5 | services.kmscon.enable = false; 6 | services.xserver = { 7 | vaapiDrivers = [ pkgs.vaapi-vdpau ]; 8 | videoDrivers = [ "nvidia-long" ]; 9 | }; 10 | } 11 | -------------------------------------------------------------------------------- /core/sub/ctdbd.module.nix: -------------------------------------------------------------------------------- 1 | { lib, ... }: 2 | 3 | let 4 | inherit (lib) 5 | mkOption 6 | types; 7 | in 8 | { 9 | options = { 10 | 11 | myCtdbd = { 12 | 13 | enable = mkOption { 14 | type = types.bool; 15 | default = false; 16 | }; 17 | 18 | }; 19 | 20 | }; 21 | } 22 | -------------------------------------------------------------------------------- /services/postgresql.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | 3 | with lib; 4 | { 5 | require = [ 6 | ./sub/postgresql-module.nix 7 | ]; 8 | services.postgresql = { 9 | enable = true; 10 | enableTCPIP = true; 11 | package = config.postgresqlPackage; 12 | }; 13 | users.extraUsers.postgres.useDefaultShell = true; 14 | } 15 | -------------------------------------------------------------------------------- /gateway/ntp.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | 3 | with lib; 4 | { 5 | networking.firewall.extraCommands = 6 | flip concatMapStrings config.myNatIfs (n: '' 7 | ip46tables -A INPUT -i ${n} -p udp --dport ntp -j ACCEPT 8 | ip46tables -A INPUT -i ${n} -p tcp --dport ntp -j ACCEPT 9 | ''); 10 | 11 | services.chrony.extraConfig = '' 12 | allow 0.0.0.0/0 13 | allow ::/0 14 | ''; 15 | } 16 | -------------------------------------------------------------------------------- /core/glusterfs.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | let 3 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 4 | in 5 | { 6 | environment.systemPackages = [ 7 | pkgs.glusterfs 8 | ]; 9 | networking.firewall.extraCommands = '' 10 | iptables -A INPUT -p tcp --dport 24007 -s ${calculated.myInternalIp4Net} -j ACCEPT 11 | iptables -A INPUT -p tcp --dport 49152 -s ${calculated.myInternalIp4Net} -j ACCEPT 12 | ''; 13 | } 14 | -------------------------------------------------------------------------------- /scripts/zfs-root-mount: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | mount -t zfs root /mnt 5 | mount -t zfs {root,/mnt}/conf 6 | mount -o defaults,bind /mnt/{conf,etc}/nixos 7 | mount -t zfs {root,/mnt/var}/log 8 | mount -t zfs {root,/mnt}/state 9 | mount -o defaults,bind /mnt/state/lib /mnt/var/lib 10 | mount -o defaults,bind /mnt/state/lib /mnt/var/db 11 | mount -o defaults,bind /mnt/state/home /mnt/home 12 | mount -o defaults,bind /mnt/state/home/root /mnt/root 13 | mount -t zfs root/tmp /mnt/tmp 14 | -------------------------------------------------------------------------------- /services/quassel.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, ... }: 2 | { 3 | imports = [ 4 | ./postgresql.nix 5 | ]; 6 | networking.firewall.extraCommands = '' 7 | # Allow all access 8 | ip46tables -A INPUT -p tcp --dport 4242 -j ACCEPT 9 | 10 | # DB Access 11 | ip46tables -A OUTPUT -m owner --uid-owner quassel -o lo -p tcp --dport 5432 -j ACCEPT 12 | ''; 13 | services.quassel = { 14 | enable = true; 15 | dataDir = "/var/lib/quassel"; 16 | interfaces = [ "0.0.0.0" ]; 17 | }; 18 | systemd.services.quassel.serviceConfig.Restart = "always"; 19 | } 20 | -------------------------------------------------------------------------------- /core/ipfs-cluster.nix: -------------------------------------------------------------------------------- 1 | { pkgs, ... }: 2 | { 3 | environment.systemPackages = [ 4 | pkgs.ipfs-cluster 5 | ]; 6 | services.ipfs-cluster.enable = true; 7 | networking.firewall.extraCommands = '' 8 | # Allow communicating with the local ipfs daemon 9 | ip46tables -A OUTPUT -m owner --uid-owner ipfs-cluster -o lo -p tcp --dport 5001 -j ACCEPT 10 | 11 | # Allow ipfs-cluster to communicate with other ipfs-clusters 12 | ip46tables -A INPUT -p tcp --dport 9096 -j ACCEPT 13 | ip46tables -A OUTPUT -m owner --uid-owner ipfs-cluster -p tcp --dport 9096 -j ACCEPT 14 | ''; 15 | } 16 | -------------------------------------------------------------------------------- /common/syncthing.nix: -------------------------------------------------------------------------------- 1 | { ... }: 2 | { 3 | networking.firewall.extraCommands = '' 4 | # Allow input to the sync port 5 | ip46tables -A INPUT -p tcp --dport 22000 -j ACCEPT 6 | ip46tables -A INPUT -p udp --dport 21027 -j ACCEPT 7 | 8 | # Allow local syncthings to access each other 9 | ip46tables -A OUTPUT -m owner --uid-owner syncthing -p tcp --dport 22000 -j ACCEPT 10 | ip46tables -A OUTPUT -m owner --uid-owner syncthing -p udp --dport 21027 -j ACCEPT 11 | 12 | # Allow privileged users to access syncthing's webui 13 | ip46tables -A OUTPUT -m owner --gid-owner wheel -o lo -p tcp --dport 8384 -j ACCEPT 14 | ''; 15 | services.syncthing.enable = true; 16 | } 17 | -------------------------------------------------------------------------------- /web/sub/ssl-settings.nix: -------------------------------------------------------------------------------- 1 | { domain }: 2 | '' 3 | ssl on; 4 | ssl_protocols TLSv1.2; 5 | ssl_ciphers EECDH+CHACHA20:EECDH+AESGCM; 6 | ssl_ecdh_curve secp384r1; 7 | ssl_prefer_server_ciphers on; 8 | ssl_certificate /conf/ssl/${domain}.crt; 9 | ssl_certificate_key /conf/ssl/${domain}.key; 10 | ssl_session_cache builtin:1000 shared:SSL:10m; 11 | ssl_session_ticket_key /conf/ssl/nginx/ticket1.key; 12 | ssl_session_ticket_key /conf/ssl/nginx/ticket2.key; 13 | ssl_session_tickets on; 14 | ssl_session_timeout 10m; 15 | ssl_stapling on; 16 | ssl_stapling_verify on; 17 | resolver 127.0.0.1; 18 | add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; 19 | '' 20 | -------------------------------------------------------------------------------- /graphical/base.nix: -------------------------------------------------------------------------------- 1 | { pkgs, lib, ... }: 2 | with lib; 3 | { 4 | # Undo minimalistic settings 5 | fonts.fontconfig.enable = true; 6 | hardware.pulseaudio.enable = true; 7 | security.pam.services.su.forwardXAuth = true; 8 | sound.enable = true; 9 | 10 | environment.systemPackages = [ 11 | pkgs.flashrom 12 | ]; 13 | security.sudo.enable = true; 14 | services = { 15 | kmscon.hwRender = true; 16 | pcscd = { 17 | enable = true; 18 | allowedGroups = [ "users" ]; 19 | }; 20 | udev.packages = [ 21 | pkgs.libu2f-host 22 | ]; 23 | xserver = { 24 | enable = true; 25 | windowManager.default = "none"; 26 | desktopManager.default = "none"; 27 | displayManager.lightdm.enable = true; 28 | }; 29 | }; 30 | } 31 | -------------------------------------------------------------------------------- /common/dyndns.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, ... }: 2 | { 3 | systemd.services.ddclient = { 4 | after = [ "network.target" ]; 5 | wantedBy = [ "multi-user.target" ]; 6 | preStart = '' 7 | mkdir -p /var/cache/ddclient 8 | cp /conf/ddclient.conf /tmp/ddclient.conf 9 | chown ddclient /var/cache/ddclient /tmp/ddclient.conf 10 | ''; 11 | environment.SSL_CERT_FILE="/etc/ssl/certs/ca-bundle.crt"; 12 | serviceConfig = { 13 | Type = "simple"; 14 | ExecStart = "@${pkgs.ddclient}/bin/ddclient ddclient -quiet -file /tmp/ddclient.conf"; 15 | PermissionsStartOnly="true"; 16 | User = "ddclient"; 17 | Group = "nogroup"; 18 | }; 19 | }; 20 | users.extraUsers.ddclient = { 21 | uid = config.ids.uids.ddclient; 22 | description = "ddclient daemon user"; 23 | }; 24 | } 25 | -------------------------------------------------------------------------------- /gateway/dns.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | with lib; 3 | { 4 | assertions = [ 5 | { 6 | assertion = config.services.unbound.enable || config.services.dnsmasq.enable; 7 | message = "You must enable at least one of `unbound` or `dnsmasq`"; 8 | } 9 | ]; 10 | networking.firewall.extraCommands = 11 | flip concatMapStrings config.myNatIfs (n: '' 12 | ip46tables -A INPUT -i ${n} -p udp --dport domain -j ACCEPT 13 | ip46tables -A INPUT -i ${n} -p tcp --dport domain -j ACCEPT 14 | ''); 15 | 16 | services.dnsmasq.extraConfig = 17 | mkIf (config.services.dnsmasq.enable) 18 | (flip concatMapStrings config.myNatIfs (n: '' 19 | interface=${n} 20 | no-dhcp-interface=${n} 21 | '')); 22 | 23 | services.unbound.extraConfig = '' 24 | server: 25 | prefetch: yes 26 | prefetch-key: yes 27 | ''; 28 | } 29 | -------------------------------------------------------------------------------- /common/sub/base-dns-module.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | with lib; 3 | { 4 | options = { 5 | myDns = rec { 6 | forwardZones = mkOption { 7 | default = { }; 8 | type = types.attrsOf (types.listOf types.optionSet); 9 | options = { 10 | zone = mkOption { 11 | server = mkOption { 12 | type = types.str; 13 | }; 14 | 15 | port = mkOption { 16 | type = types.int; 17 | }; 18 | }; 19 | }; 20 | }; 21 | 22 | forwardZones' = mkOption { 23 | default = [ ]; 24 | type = types.listOf types.str; 25 | }; 26 | }; 27 | }; 28 | 29 | config = { 30 | assertions = [ 31 | { 32 | assertion = config.myDns.forwardZones' == attrNames config.myDns.forwardZones; 33 | message = "Not all of the forward zones were processed"; 34 | } 35 | ]; 36 | }; 37 | } 38 | -------------------------------------------------------------------------------- /common/ceph.mount.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | with lib; 3 | let 4 | calculated = (import ./sub/calculated.nix { inherit config lib; }); 5 | in 6 | { 7 | imports = [ 8 | ./ceph.nix 9 | ]; 10 | systemd.automounts = [ { 11 | wantedBy = [ "remote-fs.target" ]; 12 | where = "/ceph"; 13 | } ]; 14 | systemd.mounts = [ { 15 | wants = [ "network-online.target" ]; 16 | wantedBy = [ "remote-fs.target" ]; 17 | after = [ "network.target" "network-interfaces.target" "network-online.target" "ceph-mds.service" "ceph-mon.service" ]; 18 | type = "ceph"; 19 | what = "${concatStringsSep "," calculated.myCeph.monIps}:/"; 20 | where = "/ceph"; 21 | options = [ 22 | "name=admin" 23 | secretfile=/etc/ceph/ceph.client.admin.key" 24 | ]; 25 | /*options = [ 26 | "name=admin" 27 | "secretfile=/etc/ceph/ceph.client.admin.key" 28 | "fsc" 29 | "dcache" 30 | ];*/ 31 | } ]; 32 | } 33 | -------------------------------------------------------------------------------- /common/sshd.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, ... }: 2 | { 3 | environment.systemPackages = with pkgs; [ mosh ]; 4 | networking.firewall.allowedUDPPortRanges = [ 5 | { from = 60000; to = 61000; } 6 | ]; 7 | services.openssh = { 8 | enable = true; 9 | hostKeys = [ 10 | { path = "/etc/ssh/ssh_host_rsa_key"; type = "rsa"; bits = 4096; } 11 | { path = "/etc/ssh/ssh_host_ed25519_key"; type = "ed25519"; bits = 256; } 12 | ]; 13 | forwardX11 = false; 14 | passwordAuthentication = false; 15 | challengeResponseAuthentication = false; 16 | extraConfig = pkgs.lib.mkAfter '' 17 | AllowAgentForwarding yes 18 | AllowTcpForwarding no 19 | UseDNS no 20 | 21 | Ciphers aes256-gcm@openssh.com,chacha20-poly1305@openssh.com 22 | KexAlgorithms curve25519-sha256,curve25519-sha256@libssh.org 23 | MACs hmac-sha2-512-etm@openssh.com 24 | 25 | Match User root 26 | AllowTcpForwarding yes 27 | ''; 28 | }; 29 | } 30 | -------------------------------------------------------------------------------- /core/ceph.mds.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | let 3 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 4 | 5 | stateDir = "/var/lib/ceph/mds/ceph-${config.networking.hostName}"; 6 | in 7 | { 8 | imports = [ 9 | ../common/ceph.nix 10 | ]; 11 | 12 | systemd.services.ceph-mds = { 13 | wantedBy = [ "multi-user.target" ]; 14 | after = [ "network.target" ]; 15 | 16 | restartTriggers = [ config.environment.etc."ceph/ceph.conf".source ]; 17 | 18 | serviceConfig = { 19 | Type = "simple"; 20 | ExecStart = "@${config.cephPackage}/bin/ceph-mds ceph-mds -i ${config.networking.hostName} -f --hot-standby 0"; 21 | User = "ceph-mds"; 22 | Group = "ceph"; 23 | PermissionsStartOnly = true; 24 | Restart = "always"; 25 | }; 26 | 27 | preStart = '' 28 | mkdir -p ${stateDir} 29 | chmod 0700 ${stateDir} 30 | chown -R ceph-mds:ceph ${stateDir} 31 | mkdir -p -m 0775 /var/run/ceph 32 | chown ceph-mon:ceph /var/run/ceph 33 | ''; 34 | }; 35 | } 36 | -------------------------------------------------------------------------------- /web/hydra.nix: -------------------------------------------------------------------------------- 1 | { lib, ... }: 2 | let 3 | vars = (import ./customization/vars.nix { inherit lib; }); 4 | 5 | domain = "hydra.${vars.domain}"; 6 | hydraInstance = "10.1.2.30:3000"; 7 | in 8 | { 9 | services.nginx.config = '' 10 | server { 11 | listen *:443 ssl http2; 12 | listen [::]:443 ssl http2; 13 | server_name ${domain}; 14 | location / { 15 | proxy_set_header Accept-Encoding ""; 16 | proxy_set_header Host $http_host; 17 | proxy_set_header X-Real-IP $remote_addr; 18 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 19 | proxy_set_header X-Forwarded-Proto $scheme; 20 | 21 | proxy_pass http://${hydraInstance}/; 22 | proxy_set_header Front-End-Https on; 23 | proxy_redirect off; 24 | } 25 | 26 | ${import sub/ssl-settings.nix { inherit domain; }} 27 | } 28 | 29 | server { 30 | listen *:80; 31 | listen [::]:80; 32 | server_name ${domain}; 33 | rewrite ^(.*) https://${domain}$1 permanent; 34 | } 35 | ''; 36 | } 37 | -------------------------------------------------------------------------------- /services/sydent.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | let 3 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 4 | 5 | varDir = "/var/lib/sydent"; 6 | in 7 | { 8 | networking.firewall.extraCommands = '' 9 | iptables -I INPUT -p tcp --dport 8090 -s "${calculated.myInternalIp4Net}" -j ACCEPT 10 | ''; 11 | 12 | systemd.services.sydent = { 13 | description = "Sydent daemon"; 14 | after = [ "network.target" ]; 15 | wantedBy = [ "multi-user.target" ]; 16 | preStart = '' 17 | mkdir -p "${varDir}" 18 | chown sydent "${varDir}" 19 | chmod 0700 "${varDir}" 20 | ''; 21 | serviceConfig = { 22 | Type = "simple"; 23 | PermissionsStartOnly = true; 24 | User = "sydent"; 25 | WorkingDirectory = varDir; 26 | ExecStart = "${pkgs.sydent}/bin/sydent"; 27 | }; 28 | }; 29 | 30 | users.extraUsers = lib.singleton { 31 | name = "sydent"; 32 | uid = config.ids.uids.sydent; 33 | description = "Sydent daemon user"; 34 | home = varDir; 35 | isSystemUser = true; 36 | }; 37 | } 38 | -------------------------------------------------------------------------------- /customization/tinc.nix: -------------------------------------------------------------------------------- 1 | { lib, ... }: 2 | let 3 | vars = (import ./vars.nix { inherit lib; }); 4 | in 5 | with lib; 6 | { 7 | dedicated = [ 8 | "atomic" 9 | "newton" 10 | "page" 11 | "quest" 12 | "legend" 13 | ]; 14 | hosts = { 15 | atomic = '' 16 | Address = atomic.wak.io 17 | Ed25519PublicKey = CyQA7TSZ1hksT5YlM6JA1ZKsGwy4wufkG2UDg8+BGaA 18 | ''; 19 | prodigy = '' 20 | Ed25519PublicKey = kKcEmjbD+1Fx8llu6xlAQsBiuSmb2wJp8PzhAnGtezI 21 | ''; 22 | lotus = '' 23 | Ed25519PublicKey = M0pHIXS55y831Hhs4zvB8PbdBfXV6T5hFMRSYMW8WeC 24 | ''; 25 | newton = '' 26 | Address = newton.wak.io 27 | Ed25519PublicKey = Y3l5+f/+GyOKcA99pXN5h2/DY5eYWYGSzVylKlv10lB 28 | ''; 29 | page = '' 30 | Address = page.wak.io 31 | Ed25519PublicKey = sGpVBKdBn9ALe7Z6gldb7j2d4v/lAxDfhYpvOn8dFLJ 32 | ''; 33 | quest = '' 34 | Address = quest.wak.io 35 | Ed25519PublicKey = nXRjIs3FLanILhsDF36XGq39lnojdL0VJJsvI52cAEA 36 | ''; 37 | legend = '' 38 | Address = legend.wak.io 39 | Ed25519PublicKey = zSAeyznGvqmaQAOmv9F4LOQGWpGQJNGGzP5DrEK9tAE 40 | ''; 41 | }; 42 | } 43 | -------------------------------------------------------------------------------- /laptop/base.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, ... }: 2 | { 3 | environment.etc."wpa_supplicant.conf" = { 4 | enable = true; 5 | mode = "0600"; 6 | source = "/conf/wpa_supplicant.conf"; 7 | }; 8 | environment.systemPackages = with pkgs; [ 9 | powertop 10 | ]; 11 | networking.wireless = { 12 | enable = true; 13 | interfaces = [ "wifi" ]; 14 | }; 15 | services.xserver = { 16 | multitouch.enable = true; 17 | synaptics = { 18 | enable = true; 19 | tapButtons = false; 20 | twoFingerScroll = true; 21 | additionalOptions = '' 22 | Option "RTCornerButton" "2" 23 | ''; 24 | }; 25 | }; 26 | services.udev.extraRules = '' 27 | ACTION=="add", SUBSYSTEM=="pci", ATTR{power/control}="auto" 28 | ACTION=="add", SUBSYSTEM=="usb", TEST=="power/control", ATTR{power/control}="auto" 29 | ACTION=="add", SUBSYSTEM=="scsi_host", TEST=="link_power_management_policy", ATTR{link_power_management_policy}="min_power" 30 | ACTION=="add", SUBSYSTEM=="module", TEST=="parameters/power_save", ATTR{parameters/power_save}="1" 31 | ACTION=="add", SUBSYSTEM=="net", KERNEL=="eth*", RUN+="${pkgs.ethtool}/bin/ethtool -s $name wol d" 32 | ''; 33 | } 34 | -------------------------------------------------------------------------------- /common/sub/vpn.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | 3 | with lib; 4 | let 5 | vars = (import ../../customization/vars.nix { inherit lib; }); 6 | calculated = (import ./calculated.nix { inherit config lib; }); 7 | 8 | id = vars.vpn.idMap.${config.networking.hostName}; 9 | 10 | remoteNets = if calculated.iAmRemote then vars.netMaps else 11 | flip filterAttrs vars.netMaps 12 | (n: { priv4, ... }: priv4 != calculated.myNetMap.priv4); 13 | extraRoutes = mapAttrsToList (n: { priv4, ... }: "${priv4}0.0/16") remoteNets; 14 | in 15 | { 16 | networking = { 17 | interfaces."${vars.domain}.vpn" = { 18 | ip4 = optionals (vars.vpn ? subnet4) [ 19 | { address = "${vars.vpn.subnet4}${toString id}"; prefixLength = 24; } 20 | ]; 21 | ip6 = optionals (vars.vpn ? subnet6) [ 22 | { address = "${vars.vpn.subnet6}${toString id}"; prefixLength = 64; } 23 | ]; 24 | }; 25 | 26 | localCommands = optionalString (calculated.iAmGateway || calculated.iAmRemote) ( 27 | flip concatMapStrings extraRoutes (n: '' 28 | ip route del "${n}" dev "${vars.domain}.vpn" >/dev/null 2>&1 || true 29 | ip route add "${n}" dev "${vars.domain}.vpn" 30 | '')); 31 | }; 32 | } 33 | -------------------------------------------------------------------------------- /scripts/zfs-root-provision: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | POOL_ARGS=$@ 5 | 6 | # Pool building 7 | zpool create -m legacy root $POOL_ARGS 8 | zfs set compression=lz4 root 9 | zfs set xattr=sa root 10 | zfs set atime=off root 11 | mkdir -p /mnt 12 | mount -t zfs root /mnt 13 | 14 | zfs create root/conf 15 | zfs set setuid=off root/conf 16 | zfs set devices=off root/conf 17 | mkdir -p /mnt/conf 18 | mount -t zfs {root,/mnt}/conf 19 | mkdir -p /mnt/{conf,etc}/nixos 20 | mount -o defaults,bind /mnt/{conf,etc}/nixos 21 | 22 | zfs create root/log 23 | zfs set setuid=off root/log 24 | zfs set devices=off root/log 25 | mkdir -p /mnt/var/log 26 | mount -t zfs {root,/mnt/var}/log 27 | 28 | zfs create root/state 29 | zfs set setuid=off root/state 30 | zfs set devices=off root/state 31 | mkdir -p /mnt/state 32 | mount -t zfs {root,/mnt}/state 33 | mkdir -p /mnt/{state/{lib,home/root},var/{lib,db},home} 34 | mount -o defaults,bind /mnt/state/lib /mnt/var/lib 35 | mount -o defaults,bind /mnt/state/lib /mnt/var/db 36 | mount -o defaults,bind /mnt/state/home /mnt/home 37 | mount -o defaults,bind /mnt/state/home/root /mnt/root 38 | 39 | zfs create root/tmp 40 | zfs set sync=disabled root/tmp 41 | zfs set setuid=off root/tmp 42 | zfs set devices=off root/tmp 43 | mkdir -p /mnt/tmp 44 | mount -t zfs root/tmp /mnt/tmp 45 | chmod 1777 /mnt/tmp 46 | -------------------------------------------------------------------------------- /common/sub/base-ssh.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | let 4 | inherit (lib) 5 | concatMap 6 | flip 7 | mapAttrs 8 | optionals; 9 | 10 | calculated = (import ./calculated.nix { inherit config lib; }); 11 | vars = (import ../../customization/vars.nix { inherit lib; }); 12 | in 13 | { 14 | programs.ssh = { 15 | knownHosts = flip mapAttrs vars.sshHostKeys (host: key: { 16 | hostNames = let 17 | dc = calculated.dc host; 18 | netMap = vars.netMaps."${dc}"; 19 | netData = netMap.internalMachineMap."${host}"; 20 | in [ 21 | host 22 | "${host}.${vars.domain}" 23 | ] ++ optionals (calculated.isRemote host) [ 24 | "${host}.remote.${vars.domain}" 25 | "${host}.remote" 26 | ] ++ optionals (!calculated.isRemote host) ([ 27 | "${host}.${dc}.${vars.domain}" 28 | "${host}.${dc}" 29 | ] ++ flip concatMap netData.vlans (vlan: [ 30 | "${host}.${vlan}.${vars.domain}" 31 | "${host}.${vlan}" 32 | "${host}.${vlan}.${dc}.${vars.domain}" 33 | "${host}.${vlan}.${dc}" 34 | ])) ++ optionals (vars.vpn.idMap ? "${host}") [ 35 | "${host}.vpn.${vars.domain}" 36 | "${host}.vpn" 37 | ]; 38 | publicKey = key; 39 | }); 40 | package = pkgs.openssh; 41 | }; 42 | } 43 | -------------------------------------------------------------------------------- /common/ipfs.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | let 3 | inherit (lib) 4 | concatLists 5 | flip 6 | optionals; 7 | 8 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 9 | 10 | addresses = optionals (calculated.myPublicIp4 != null) [ 11 | "/ip4/${calculated.myPublicIp4}" 12 | ] ++ optionals (calculated.myPublicIp6 != null) [ 13 | "/ip6/${calculated.myPublicIp6}" 14 | ]; 15 | 16 | addresses' = 17 | if addresses != [] then 18 | addresses 19 | else 20 | [ "/ip4/0.0.0.0" "/ip6/::" ]; 21 | 22 | swarm = concatLists (flip map addresses' (n: [ 23 | "${n}/tcp/4001" 24 | ] ++ optionals (config.services.ipfs.quic) [ 25 | "${n}/udp/4001/quic" 26 | ])); 27 | in 28 | { 29 | environment.systemPackages = with pkgs; [ 30 | ipfs 31 | ]; 32 | 33 | networking.firewall = { 34 | allowedTCPPorts = [ 4001 ]; 35 | allowedUDPPorts = [ 4001 ]; 36 | 37 | extraCommands = '' 38 | # Allow all other processes to acccess the gateway 39 | ip46tables -A OUTPUT -o lo -p tcp --dport 8001 -j ACCEPT 40 | ''; 41 | }; 42 | 43 | services.ipfs = { 44 | enable = true; 45 | extraAttrs = { 46 | Addresses.Swarm = swarm; 47 | Gateway.NoFetch = true; 48 | }; 49 | }; 50 | 51 | systemd.services.ipfs = { 52 | serviceConfig = { 53 | MemoryMax = "4G"; 54 | CPUQuota = "100%"; 55 | }; 56 | }; 57 | } 58 | -------------------------------------------------------------------------------- /gateway/sub/base-conntrack.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | let 4 | inherit (lib) 5 | filter 6 | length 7 | mkIf; 8 | 9 | calculated = import ../../common/sub/calculated.nix { inherit config lib; }; 10 | otherGateways = filter (n: config.networking.hostName != n) calculated.myNetMap.gateways; 11 | in 12 | mkIf (length calculated.myNetMap.gateways >= 2) { 13 | environment.systemPackages = [ pkgs.conntrack-tools ]; 14 | 15 | networking.firewall.extraCommands = '' 16 | iptables -I INPUT -i tlan -d 225.0.0.50 -j ACCEPT 17 | ''; 18 | 19 | services.conntrackd = { 20 | enable = true; 21 | interface = "tlan"; 22 | localAddress = calculated.internalIp4 config.networking.hostName "tlan"; 23 | ignoreAddresses = [ 24 | calculated.myVpnIp4 25 | ] ++ map (vlan: calculated.internalIp4 config.networking.hostName vlan) calculated.myNetData.vlans; 26 | }; 27 | 28 | services.keepalived.syncGroups.gateway = { 29 | notifyMaster = "${pkgs.conntrack-tools}/libexec/primary-backup.sh primary"; 30 | notifyBackup = "${pkgs.conntrack-tools}/libexec/primary-backup.sh backup"; 31 | notifyFault = "${pkgs.conntrack-tools}/libexec/primary-backup.sh fault"; 32 | notifyStop = "${pkgs.conntrack-tools}/libexec/primary-backup.sh fault"; 33 | }; 34 | 35 | systemd.services.keepalived = { 36 | requires = [ "conntrackd.service" ]; 37 | after = [ "conntrackd.service" ]; 38 | bindsTo = [ "conntrackd.service" ]; 39 | partOf = [ "conntrackd.service" ]; 40 | }; 41 | } 42 | -------------------------------------------------------------------------------- /common/xfs-root.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | with lib; 3 | { 4 | imports = [ ./fs-root.nix ]; 5 | boot.initrd.supportedFilesystems = [ "xfs" ]; 6 | fileSystems = mkMerge [ 7 | (mkOrder 0 [ 8 | { 9 | mountPoint = "/"; 10 | fsType = "xfs"; 11 | device = "/dev/disk/by-uuid/${config.rootUUID}"; 12 | options = [ 13 | "defaults" 14 | "noatime" 15 | ]; 16 | neededForBoot = true; 17 | } 18 | ]) 19 | (mkOrder 2 [ 20 | { 21 | mountPoint = "/etc/nixos"; 22 | fsType = "none"; 23 | device = "/conf/nixos"; 24 | neededForBoot = true; 25 | options = [ 26 | "defaults" 27 | "bind" 28 | ]; 29 | } 30 | { 31 | mountPoint = "/home"; 32 | fsType = "none"; 33 | device = "/state/home"; 34 | neededForBoot = true; 35 | options = [ 36 | "defaults" 37 | "bind" 38 | ]; 39 | } 40 | { 41 | mountPoint = "/root"; 42 | fsType = "none"; 43 | device = "/state/home/root"; 44 | neededForBoot = true; 45 | options = [ 46 | "defaults" 47 | "bind" 48 | ]; 49 | } 50 | { 51 | mountPoint = "/var/lib"; 52 | fsType = "none"; 53 | device = "/state/lib"; 54 | neededForBoot = true; 55 | options = [ 56 | "defaults" 57 | "bind" 58 | ]; 59 | } 60 | { 61 | mountPoint = "/var/db"; 62 | fsType = "none"; 63 | device = "/state/lib"; 64 | neededForBoot = true; 65 | options = [ 66 | "defaults" 67 | "bind" 68 | ]; 69 | } 70 | ]) 71 | ]; 72 | } 73 | -------------------------------------------------------------------------------- /common/fs-root.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }:with lib; 2 | { 3 | require = [ 4 | ./sub/fs-root-module.nix 5 | ]; 6 | boot = { 7 | kernelParams = optionals (config.serialConsole != null) [ 8 | "console=tty0" 9 | "console=ttyS${toString config.serialConsole},115200n8" 10 | ]; 11 | loader = { 12 | efi = { 13 | canTouchEfiVariables = true; 14 | efiSysMountPoint = "/boot"; 15 | }; 16 | grub = { 17 | efiSupport = true; 18 | extraConfig = optionalString (config.serialConsole != null) '' 19 | serial --unit=${toString config.serialConsole} --speed=115200 20 | terminal_input --append serial 21 | terminal_output --append serial 22 | ''; 23 | }; 24 | timeout = 1; 25 | }; 26 | }; 27 | 28 | fileSystems = mkMerge [ 29 | (mkOrder 1 (flip map config.boot.loader.grub.mirroredBoots 30 | (arg: 31 | assert arg.path != "/boot"; # We should never see the default path 32 | assert length arg.devices == 1; # There should always be a 1 - 1 map between paths and devices 33 | { 34 | mountPoint = arg.path; 35 | device = "${head arg.devices}-part2"; 36 | fsType = "vfat"; 37 | options = [ 38 | "defaults" 39 | "noatime" 40 | ]; 41 | neededForBoot = true; 42 | }) 43 | )) 44 | (mkOrder 1 [ 45 | { 46 | mountPoint = "/tmp"; 47 | device = "tmpfs"; 48 | fsType = "tmpfs"; 49 | options = [ 50 | "defaults" 51 | "noatime" 52 | ]; 53 | neededForBoot = true; 54 | } 55 | ]) 56 | ]; 57 | 58 | system.extraDependencies = with pkgs; [ 59 | grub_bios-i386 grub_efi-x86_64 grub_efi-i386 60 | ]; 61 | } 62 | -------------------------------------------------------------------------------- /core/mongodb.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | let 3 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 4 | 5 | serverIps = 6 | if calculated.iAmRemote then 7 | null 8 | else 9 | calculated.myMongodb.serverIps; 10 | in 11 | with lib; 12 | { 13 | networking.firewall = { 14 | extraCommands = mkMerge [ 15 | (mkOrder 0 '' 16 | # Cleanup if we haven't already 17 | iptables -D INPUT -p tcp --dport 27017 -j mongodb || true 18 | iptables -F mongodb || true 19 | iptables -X mongodb || true 20 | ipset destroy mongodb || true 21 | '') 22 | (mkIf (serverIps != null) '' 23 | # Allow remote mongodb replicas to communicate 24 | ipset create mongodb hash:ip family inet 25 | ${flip concatMapStrings serverIps (n: '' 26 | ipset add mongodb "${n}" 27 | '')} 28 | iptables -N mongodb 29 | iptables -A mongodb -m set --match-set mongodb src -j ACCEPT 30 | iptables -A mongodb -j RETURN 31 | iptables -A INPUT -p tcp --dport 27017 -j mongodb 32 | 33 | # Allow mongodb to connect to itself and other nodes 34 | ip46tables -A OUTPUT -p tcp --dport 27017 -m owner --uid-owner mongodb -j ACCEPT 35 | '') 36 | ]; 37 | extraStopCommands = '' 38 | iptables -D INPUT -p tcp --dport 27017 -j mongodb || true 39 | iptables -F mongodb || true 40 | iptables -X mongodb || true 41 | ipset destroy mongodb || true 42 | ''; 43 | }; 44 | services.mongodb = { 45 | enable = true; 46 | bind_ip = "0.0.0.0"; 47 | replSetName = mkIf (serverIps != null) calculated.myDomain; 48 | extraConfig = mkIf (serverIps != null) '' 49 | keyFile = /conf/mongodb/keyfile 50 | ''; 51 | }; 52 | } 53 | -------------------------------------------------------------------------------- /gateway/upnp.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | let 3 | calculated = (import ../../common/sub/calculated.nix { inherit config lib; }); 4 | in 5 | { 6 | networking.firewall.extraCommands = lib.flip lib.concatMapStrings config.myNatIfs (i: '' 7 | # Allow traffic to the daemon 8 | ip46tables -A INPUT -i ${i} -p udp --dport 1900 -j ACCEPT 9 | ip46tables -A INPUT -i ${i} -p tcp --dport 1901 -j ACCEPT 10 | ip46tables -A INPUT -i ${i} -p udp --dport 5351 -j ACCEPT 11 | 12 | '') + '' 13 | # Add upnp tables if they don't exist 14 | if ! iptables -L 2>&1 | grep -q 'upnp-forward'; then 15 | ip46tables -N upnp-forward 16 | ip46tables -A upnp-forward -j RETURN 17 | fi 18 | if ! iptables -t nat -L 2>&1 | grep -q 'upnp-nat'; then 19 | ip46tables -t nat -N upnp-nat 20 | ip46tables -t nat -A upnp-nat -j RETURN 21 | fi 22 | 23 | # Add the mappings from the other chains 24 | ip46tables -A FORWARD -j upnp-forward 25 | ip46tables -t nat -A PREROUTING -j upnp-nat 26 | ''; 27 | 28 | services.miniupnpd = { 29 | enable = true; 30 | externalInterface = "wan"; 31 | internalIPs = config.myNatIfs; 32 | natpmp = true; 33 | upnp = true; 34 | appendConfig = '' 35 | http_port=1901 36 | 37 | upnp_forward_chain=upnp-forward 38 | upnp_nat_chain=upnp-nat 39 | 40 | secure_mode=yes 41 | 42 | system_uptime=yes 43 | 44 | lease_file=/var/lib/miniupnpd/leases 45 | 46 | uuid=797e683b-e968-4e21-af2a-56b53f8e06e7 47 | 48 | allow 40000-50000 ${calculated.myNetMap.priv4}0.0/16 40000-50000 49 | deny 0-65535 0.0.0.0/0 0-65535 50 | ''; 51 | }; 52 | 53 | systemd.services.miniupnpd.preStart = '' 54 | mkdir -p /var/lib/miniupnpd 55 | touch /var/lib/miniupnpd/leases 56 | ''; 57 | } 58 | -------------------------------------------------------------------------------- /common/sub/base-hosts.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | 3 | with lib; 4 | let 5 | vars = (import ../../customization/vars.nix { inherit lib; }); 6 | calculated = (import ./calculated.nix { inherit config lib; }); 7 | 8 | ipHosts' = concatLists (flip mapAttrsToList vars.netMaps (dc: dcd: 9 | concatLists (flip mapAttrsToList dcd.internalMachineMap (host: data: [ 10 | # We actually don't want this because it conflicts with public names 11 | #{ "${calculated.internalIp4 host (head data.vlans)}" = "${host}.${vars.domain}"; } 12 | #{ "${calculated.internalIp4 host (head data.vlans)}" = "${host}.${dc}.${vars.domain}"; } 13 | { "${calculated.bmcIp4 host}" = "${host}-bmc.${vars.domain}"; } 14 | { "${calculated.bmcIp4 host}" = "${host}-bmc.${dc}.${vars.domain}"; } 15 | ] ++ flip concatMap data.vlans (vlan: [ 16 | { "${calculated.internalIp4 host vlan}" = "${host}.${vlan}.${vars.domain}"; } 17 | { "${calculated.internalIp4 host vlan}" = "${host}.${vlan}.${dc}.${vars.domain}"; } 18 | ]) 19 | )) 20 | )) ++ concatLists (flip mapAttrsToList vars.vpn.idMap (host: id: 21 | [ 22 | { "${calculated.vpnIp6 host}" = "${host}.vpn.${vars.domain}"; } 23 | { "${calculated.vpnIp4 host}" = "${host}.vpn.${vars.domain}"; } 24 | ] 25 | )) ++ concatLists (flip map (filter (n: vars.vpn.idMap ? "${n}") vars.remotes) (host: 26 | [ 27 | { "${calculated.vpnGwIp6 host}" = "${host}.remote.${vars.domain}"; } 28 | { "${calculated.vpnGwIp4 host}" = "${host}.remote.${vars.domain}"; } 29 | ] 30 | )); 31 | 32 | ipHosts = foldAttrs (n: a: [ n ] ++ a) [ ] ipHosts'; 33 | in 34 | { 35 | networking.extraHosts = concatStrings (flip mapAttrsToList ipHosts (ip: hosts: '' 36 | ${ip} ${concatStringsSep " " hosts} 37 | '')); 38 | } 39 | -------------------------------------------------------------------------------- /common/btrfs-root.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | with lib; 3 | { 4 | imports = [ ./fs-root.nix ]; 5 | boot.initrd.supportedFilesystems = [ "btrfs" ]; 6 | fileSystems = mkMerge [ 7 | (mkOrder 0 [ 8 | { 9 | mountPoint = "/"; 10 | fsType = "btrfs"; 11 | device = "/dev/disk/by-uuid/${config.rootUUID}"; 12 | options = [ 13 | "defaults" 14 | "noatime" 15 | "space_cache" 16 | "compress=lzo" 17 | ]; 18 | neededForBoot = true; 19 | } 20 | ]) 21 | (mkOrder 2 [ 22 | { 23 | mountPoint = "/etc/nixos"; 24 | fsType = "none"; 25 | device = "/conf/nixos"; 26 | neededForBoot = true; 27 | options = [ 28 | "defaults" 29 | "bind" 30 | ]; 31 | } 32 | { 33 | mountPoint = "/home"; 34 | fsType = "none"; 35 | device = "/state/home"; 36 | neededForBoot = true; 37 | options = [ 38 | "defaults" 39 | "bind" 40 | ]; 41 | } 42 | { 43 | mountPoint = "/root"; 44 | fsType = "none"; 45 | device = "/state/home/root"; 46 | neededForBoot = true; 47 | options = [ 48 | "defaults" 49 | "bind" 50 | ]; 51 | } 52 | { 53 | mountPoint = "/var/lib"; 54 | fsType = "none"; 55 | device = "/state/lib"; 56 | neededForBoot = true; 57 | options = [ 58 | "defaults" 59 | "bind" 60 | ]; 61 | } 62 | { 63 | mountPoint = "/var/db"; 64 | fsType = "none"; 65 | device = "/state/lib"; 66 | neededForBoot = true; 67 | options = [ 68 | "defaults" 69 | "bind" 70 | ]; 71 | } 72 | ]) 73 | ]; 74 | } 75 | -------------------------------------------------------------------------------- /common/bcachefs-root.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | with lib; 3 | { 4 | imports = [ ./fs-root.nix ]; 5 | 6 | boot = { 7 | initrd.supportedFilesystems = [ "bcachefs" ]; 8 | kernelPackages = lib.mkOverride 0 pkgs.linuxPackages_bcachefs; 9 | }; 10 | 11 | fileSystems = mkMerge [ 12 | (mkOrder 0 [ 13 | { 14 | mountPoint = "/"; 15 | fsType = "bcachefs"; 16 | device = "/dev/disk/by-uuid/${config.rootUUID}"; 17 | options = [ 18 | "defaults" 19 | "noatime" 20 | ]; 21 | neededForBoot = true; 22 | } 23 | ]) 24 | (mkOrder 2 [ 25 | { 26 | mountPoint = "/etc/nixos"; 27 | fsType = "none"; 28 | device = "/conf/nixos"; 29 | neededForBoot = true; 30 | options = [ 31 | "defaults" 32 | "bind" 33 | ]; 34 | } 35 | { 36 | mountPoint = "/home"; 37 | fsType = "none"; 38 | device = "/state/home"; 39 | neededForBoot = true; 40 | options = [ 41 | "defaults" 42 | "bind" 43 | ]; 44 | } 45 | { 46 | mountPoint = "/root"; 47 | fsType = "none"; 48 | device = "/state/home/root"; 49 | neededForBoot = true; 50 | options = [ 51 | "defaults" 52 | "bind" 53 | ]; 54 | } 55 | { 56 | mountPoint = "/var/lib"; 57 | fsType = "none"; 58 | device = "/state/lib"; 59 | neededForBoot = true; 60 | options = [ 61 | "defaults" 62 | "bind" 63 | ]; 64 | } 65 | { 66 | mountPoint = "/var/db"; 67 | fsType = "none"; 68 | device = "/state/lib"; 69 | neededForBoot = true; 70 | options = [ 71 | "defaults" 72 | "bind" 73 | ]; 74 | } 75 | ]) 76 | ]; 77 | } 78 | -------------------------------------------------------------------------------- /web/www-wak-io.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | let 3 | inherit (lib) 4 | concatMapStrings 5 | head 6 | flip; 7 | 8 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 9 | vars = (import ../customization/vars.nix { inherit lib; }); 10 | 11 | consulService = "www-wak-io"; 12 | consulDomain = "${consulService}.service.consul.${vars.domain}"; 13 | checkDomain = "${consulService}.${config.networking.hostName}.${vars.domain}"; 14 | 15 | dc = calculated.dc config.networking.hostName; 16 | 17 | domains = [ 18 | "www.${dc}.wak.io" 19 | "${dc}.wak.io" 20 | "www.wak.io" 21 | "wak.io" 22 | consulDomain 23 | checkDomain 24 | ]; 25 | 26 | path = "/ceph/${consulService}"; 27 | in 28 | { 29 | imports = [ ./base.nix ]; 30 | 31 | services.nginx.config = '' 32 | server { 33 | listen *:443 ssl http2; 34 | listen [::]:443 ssl http2; 35 | '' + flip concatMapStrings domains (n: '' 36 | server_name ${n}; 37 | '') + '' 38 | 39 | location / { 40 | root ${path}; 41 | autoindex on; 42 | } 43 | 44 | error_page 500 502 503 504 /50x.html; 45 | 46 | ${import sub/ssl-settings.nix { domain = head domains; }} 47 | } 48 | 49 | '' + flip concatMapStrings domains (n: '' 50 | server { 51 | listen *:80; 52 | listen [::]:80; 53 | server_name ${n}; 54 | 55 | location / { 56 | rewrite ^(.*) https://${n}$1 permanent; 57 | } 58 | 59 | location /.well-known/acme-challenge { 60 | proxy_pass http://acme; 61 | } 62 | } 63 | ''); 64 | 65 | environment.etc."consul.d/${consulService}.json".text = builtins.toJSON { 66 | service = { 67 | name = consulService; 68 | port = 443; 69 | checks = [ 70 | { 71 | http = "https://${checkDomain}:443"; 72 | tls_skip_verify = true; 73 | interval = "10s"; 74 | timeout = "1s"; 75 | } 76 | ]; 77 | }; 78 | }; 79 | } 80 | -------------------------------------------------------------------------------- /web/pub.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | let 3 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 4 | vars = (import ../customization/vars.nix { inherit lib; }); 5 | 6 | domain = "pub.${calculated.myDomain}"; 7 | topDomain = "pub.${vars.domain}"; 8 | consulService = "pub"; 9 | consulDomain = "${consulService}.service.consul.${vars.domain}"; 10 | checkDomain = "${consulService}.${config.networking.hostName}.${vars.domain}"; 11 | 12 | path = "/ceph/www-pub"; 13 | in 14 | { 15 | imports = [ ./base.nix ]; 16 | 17 | services.nginx.config = '' 18 | server { 19 | listen *:443 ssl http2; 20 | listen [::]:443 ssl http2; 21 | server_name ${domain}; 22 | server_name ${topDomain}; 23 | server_name ${consulDomain}; 24 | server_name ${checkDomain}; 25 | 26 | location / { 27 | root ${path}; 28 | autoindex on; 29 | autoindex_exact_size off; 30 | } 31 | 32 | error_page 500 502 503 504 /50x.html; 33 | 34 | ${import sub/ssl-settings.nix { inherit domain; }} 35 | } 36 | 37 | server { 38 | listen *:80; 39 | listen [::]:80; 40 | server_name ${domain}; 41 | server_name ${topDomain}; 42 | server_name ${consulDomain}; 43 | server_name ${checkDomain}; 44 | 45 | location / { 46 | root ${path}; 47 | autoindex on; 48 | autoindex_exact_size off; 49 | } 50 | 51 | location /.well-known/acme-challenge { 52 | proxy_pass http://acme; 53 | } 54 | 55 | error_page 500 502 503 504 /50x.html; 56 | } 57 | ''; 58 | 59 | environment.etc."consul.d/${consulService}.json".text = builtins.toJSON { 60 | service = { 61 | name = consulService; 62 | port = 443; 63 | checks = [ 64 | { 65 | http = "https://${checkDomain}:443"; 66 | tls_skip_verify = true; 67 | interval = "10s"; 68 | timeout = "1s"; 69 | } 70 | ]; 71 | }; 72 | }; 73 | } 74 | -------------------------------------------------------------------------------- /web/mumble.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | let 3 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 4 | vars = (import ../customization/vars.nix { inherit lib; }); 5 | 6 | domain = "mumble.${calculated.myDomain}"; 7 | topDomain = "mumble.${vars.domain}"; 8 | consulService = "mumble-web"; 9 | consulDomain = "${consulService}.service.consul.${vars.domain}"; 10 | checkDomain = "${consulService}.${config.networking.hostName}.${vars.domain}"; 11 | 12 | path = "/ceph/www-mumble"; 13 | in 14 | { 15 | imports = [ ./base.nix ]; 16 | 17 | services.nginx.config = '' 18 | server { 19 | listen *:443 ssl http2; 20 | listen [::]:443 ssl http2; 21 | server_name ${domain}; 22 | server_name ${topDomain}; 23 | server_name ${consulDomain}; 24 | server_name ${checkDomain}; 25 | 26 | location / { 27 | root ${path}; 28 | autoindex on; 29 | autoindex_exact_size off; 30 | } 31 | 32 | error_page 500 502 503 504 /50x.html; 33 | 34 | ${import sub/ssl-settings.nix { inherit domain; }} 35 | } 36 | 37 | server { 38 | listen *:80; 39 | listen [::]:80; 40 | server_name ${domain}; 41 | server_name ${topDomain}; 42 | server_name ${consulDomain}; 43 | server_name ${checkDomain}; 44 | 45 | location / { 46 | root ${path}; 47 | autoindex on; 48 | autoindex_exact_size off; 49 | } 50 | 51 | location /.well-known/acme-challenge { 52 | proxy_pass http://acme; 53 | } 54 | 55 | error_page 500 502 503 504 /50x.html; 56 | } 57 | ''; 58 | 59 | environment.etc."consul.d/${consulService}.json".text = builtins.toJSON { 60 | service = { 61 | name = consulService; 62 | port = 443; 63 | checks = [ 64 | { 65 | http = "https://${checkDomain}:443"; 66 | tls_skip_verify = true; 67 | interval = "10s"; 68 | timeout = "1s"; 69 | } 70 | ]; 71 | }; 72 | }; 73 | } 74 | -------------------------------------------------------------------------------- /services/murmur.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | let 3 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 4 | vars = (import ../customization/vars.nix { inherit lib; }); 5 | 6 | varDir = "/var/lib/murmur"; 7 | 8 | murmur = pkgs.murmur_git; 9 | 10 | murmurUser = "murmur"; 11 | murmurUid = 200000; 12 | 13 | sslCert = "/conf/ssl/mumble.${calculated.myDomain}.crt"; 14 | sslKey = "/conf/ssl/mumble.${calculated.myDomain}.key"; 15 | 16 | configFile = pkgs.writeText "murmur.ini" '' 17 | database=${varDir}/murmur.sqlite 18 | icesecretread= 19 | icesecretwrite= 20 | logfile=/var/log/murmur.log 21 | pidfile=/run/murmur.pid 22 | welcometext="
Welcome to mumble.${vars.domain}" 23 | port=64738 24 | host=0.0.0.0 :: 25 | serverpassword= 26 | bandwidth=192000 27 | users=100 28 | textmessagelength=5000000 29 | imagemessagelength=100000000 30 | allowhtml=true 31 | registerName=William's Chat 32 | sslCert=${sslCert} 33 | sslKey=${sslKey} 34 | uname=${murmurUser} 35 | certrequired=True 36 | 37 | [Ice] 38 | Ice.Warn.UnkownProperties=1 39 | Ice.MessageSizeMax=65536 40 | ''; 41 | in 42 | { 43 | environment.systemPackages = [ murmur ]; 44 | 45 | networking.firewall = { 46 | allowedTCPPorts = [ 64738 ]; 47 | allowedUDPPorts = [ 64738 ]; 48 | }; 49 | 50 | systemd.services.murmur = { 51 | description = "Murmur daemon"; 52 | after = [ "network.target" ]; 53 | wantedBy = [ "multi-user.target" ]; 54 | path = [ murmur ]; 55 | preStart = '' 56 | mkdir -p ${varDir} 57 | chown murmur ${varDir} 58 | chmod 0700 ${varDir} 59 | ''; 60 | serviceConfig = { 61 | Type = "simple"; 62 | ExecStart = "${murmur}/bin/murmurd -ini ${configFile} -fg"; 63 | }; 64 | }; 65 | 66 | users.extraUsers = lib.singleton { 67 | name = murmurUser; 68 | uid = murmurUid; 69 | description = "Murmur daemon user"; 70 | home = varDir; 71 | isSystemUser = true; 72 | }; 73 | } 74 | -------------------------------------------------------------------------------- /web/triton-cache.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | let 3 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 4 | vars = (import ../customization/vars.nix { inherit lib; }); 5 | 6 | domain = "cache.triton.${calculated.myDomain}"; 7 | topDomain = "cache.triton.${vars.domain}"; 8 | consulService = "triton-cache"; 9 | consulDomain = "${consulService}.service.consul.${vars.domain}"; 10 | checkDomain = "${consulService}.${config.networking.hostName}.${vars.domain}"; 11 | 12 | path = "/ceph/triton-cache"; 13 | in 14 | { 15 | imports = [ ./base.nix ]; 16 | 17 | services.nginx.config = '' 18 | server { 19 | listen *:443 ssl http2; 20 | listen [::]:443 ssl http2; 21 | server_name ${domain}; 22 | server_name ${topDomain}; 23 | server_name ${consulDomain}; 24 | server_name ${checkDomain}; 25 | 26 | location / { 27 | root ${path}; 28 | autoindex off; 29 | autoindex_exact_size off; 30 | } 31 | 32 | error_page 500 502 503 504 /50x.html; 33 | 34 | ${import sub/ssl-settings.nix { inherit domain; }} 35 | } 36 | 37 | server { 38 | listen *:80; 39 | listen [::]:80; 40 | server_name ${domain}; 41 | server_name ${topDomain}; 42 | server_name ${consulDomain}; 43 | server_name ${checkDomain}; 44 | 45 | location / { 46 | root ${path}; 47 | autoindex off; 48 | autoindex_exact_size off; 49 | } 50 | 51 | location /.well-known/acme-challenge { 52 | proxy_pass http://acme; 53 | } 54 | 55 | error_page 500 502 503 504 /50x.html; 56 | } 57 | ''; 58 | 59 | environment.etc."consul.d/${consulService}.json".text = builtins.toJSON { 60 | service = { 61 | name = consulService; 62 | port = 443; 63 | checks = [ 64 | { 65 | http = "https://${checkDomain}:443"; 66 | tls_skip_verify = true; 67 | interval = "10s"; 68 | timeout = "1s"; 69 | } 70 | ]; 71 | }; 72 | }; 73 | } 74 | -------------------------------------------------------------------------------- /web/www-wkennington-com.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | let 3 | inherit (lib) 4 | concatMapStrings 5 | head 6 | flip; 7 | 8 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 9 | vars = (import ../customization/vars.nix { inherit lib; }); 10 | 11 | consulService = "www-wkennington-com"; 12 | consulDomain = "${consulService}.service.consul.${vars.domain}"; 13 | checkDomain = "${consulService}.${config.networking.hostName}.${vars.domain}"; 14 | 15 | dc = calculated.dc config.networking.hostName; 16 | 17 | domains = [ 18 | "www.${dc}.wkennington.com" 19 | "${dc}.wkennington.com" 20 | "www.wkennington.com" 21 | "wkennington.com" 22 | consulDomain 23 | checkDomain 24 | ]; 25 | 26 | path = "/ceph/${consulService}"; 27 | in 28 | { 29 | imports = [ ./base.nix ]; 30 | 31 | services.nginx.config = '' 32 | server { 33 | listen *:443 ssl http2; 34 | listen [::]:443 ssl http2; 35 | '' + flip concatMapStrings domains (n: '' 36 | server_name ${n}; 37 | '') + '' 38 | 39 | location / { 40 | root ${path}; 41 | autoindex on; 42 | } 43 | 44 | error_page 500 502 503 504 /50x.html; 45 | 46 | ${import sub/ssl-settings.nix { domain = head domains; }} 47 | } 48 | 49 | '' + flip concatMapStrings domains (n: '' 50 | server { 51 | listen *:80; 52 | listen [::]:80; 53 | server_name ${n}; 54 | 55 | location / { 56 | rewrite ^(.*) https://${n}$1 permanent; 57 | } 58 | 59 | location /.well-known/acme-challenge { 60 | proxy_pass http://acme; 61 | } 62 | } 63 | ''); 64 | 65 | environment.etc."consul.d/${consulService}.json".text = builtins.toJSON { 66 | service = { 67 | name = consulService; 68 | port = 443; 69 | checks = [ 70 | { 71 | http = "https://${checkDomain}:443"; 72 | tls_skip_verify = true; 73 | interval = "10s"; 74 | timeout = "1s"; 75 | } 76 | ]; 77 | }; 78 | }; 79 | } 80 | -------------------------------------------------------------------------------- /customization/wireguard.nix: -------------------------------------------------------------------------------- 1 | { ... }: 2 | 3 | { 4 | hosts = { 5 | atomic = { 6 | publicKey = "b2g2dvgFgqZHu554r6g0Pzt7LegFC7PnYLsRQKmN/xk="; 7 | }; 8 | newton = { 9 | publicKey = "cBrmihVXtv8HeqUwXZ77q1ciXJkSSPXBeJ8ih14OpD0="; 10 | endpoint = "newton.wak.io:656"; 11 | }; 12 | page = { 13 | publicKey = "rJ4hsXQfUX4yatWCSgunbE7q3Zkmixt2RYpKlu70jyk="; 14 | endpoint = "page.wak.io:656"; 15 | }; 16 | quest = { 17 | publicKey = "xvX4tER03tr6X0xyOh4NsLL617y4eJQwabZ+OAnw5xo="; 18 | endpoint = "quest.wak.io:656"; 19 | }; 20 | ferrari = { 21 | publicKey = "nD2YwYj+9AECqPS/InmiHOEhhN0VKUQYY9MBipXjS3U="; 22 | }; 23 | atlas = { 24 | publicKey = "Z2daI/c0gwn9aNQondelzTSg3jRCHRGypMDEICXbak8="; 25 | }; 26 | nevada = { 27 | publicKey = "f7m+rqpaUOHDvH/Wwb2SXBm0HGG0ET1pifUmEvnCvnI="; 28 | }; 29 | exodus = { 30 | publicKey = "14tgyEgxQUzndg12uB95iIgPdkqvmq0hEEMNI/Gib1w="; 31 | }; 32 | elite = { 33 | publicKey = "9acg1dC1O5jASDh0lcDv0LSFqEJzQjs77PpGGNgNsmg="; 34 | }; 35 | delta = { 36 | publicKey = "ZWzGAxNhG/qXOuCiFhtbh2vlpQ4QKUINiyxfVuJHdHk="; 37 | }; 38 | legend = { 39 | publicKey = "97NWxPo82wuChdJZYEbMsD/nQm4yo5TnOEM3NyfsTSQ="; 40 | }; 41 | lake = { 42 | publicKey = "oKtsZpQBgL5ZGkYImWQJqQUDGTZdyVLSHOF1Z9YiBnk="; 43 | }; 44 | jupiter = { 45 | publicKey = "pkEAsNK4eFQiAIPeFSNm8YzQUATj+UZOzxegdA6tgHo="; 46 | }; 47 | "gw.abe-p" = { 48 | publicKey = "IaftkRMWBpV1hb16aZR92FJAmMJ2a4vV/EEB9UiFkS0="; 49 | endpoint = "gw.abe-p.wak.io:657"; 50 | }; 51 | "gw.fmt-1" = { 52 | publicKey = "mKvSVqiig7zRgacHBvTAAWzgB390XgxnANhCbCyPLGA="; 53 | endpoint = "gw.fmt-1.wak.io:657"; 54 | }; 55 | "gw.mtv-w" = { 56 | publicKey = "OzHYvVZ1giTFZ4wfONtRjNVhXe5JvDR7EmdMkeWc4H4="; 57 | endpoint = "gw.mtv-w.wak.io:657"; 58 | }; 59 | "gw.nyc-1" = { 60 | publicKey = "IvLzaIsx9V7zUQUQN4MIvOCp9Ht8MBr1aUxj5LNABzc="; 61 | endpoint = "gw.nyc-1.wak.io:657"; 62 | }; 63 | }; 64 | } 65 | -------------------------------------------------------------------------------- /gateway/load-balancer.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | with lib; 3 | let 4 | vars = (import ../customization/vars.nix { inherit lib; }); 5 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 6 | 7 | internalVlanMap = listToAttrs (flip map (calculated.myNetData.vlans ++ [ "lan" ]) (v: 8 | nameValuePair v vars.internalVlanMap.${v} 9 | )); 10 | 11 | lbMap = calculated.myNetMap.loadBalancerMap; 12 | machines = attrNames map; 13 | 14 | lbPrioMap = flip mapAttrs lbMap (lb: _: priorities (orderList lb (attrNames lbMap) (attrValues lbMap))); 15 | 16 | orderList = lb: lbs: machines: if head lbs == lb then machines 17 | else orderList lb ((tail lbs) ++ [ (head lbs) ]) ((tail machines) ++ [ (head machines) ]); 18 | priorities = machines: listToAttrs (zipListsWith (machine: prio: nameValuePair machine (256 - prio)) 19 | machines (range 1 (length machines))); 20 | in 21 | { 22 | services.keepalived = { 23 | enable = true; 24 | syncGroups = flip mapAttrs' lbPrioMap (lb: _: nameValuePair "${lb}g" { group = [ "${lb}-4" "${lb}-6" ]; }); 25 | instances = flip mapAttrs' lbPrioMap (lb: priorities: nameValuePair "${lb}-4" { 26 | interface = "tlan"; 27 | trackInterfaces = [ "wan" ]; 28 | virtualRouterId = calculated.myNetMap.vrrpMap."${lb}-4"; 29 | priority = priorities."${config.networking.hostName}"; 30 | authType = "PASS"; 31 | authPass = "none"; 32 | virtualIpAddresses = [ 33 | { ip = "${calculated.myNetMap.pub4}${toString calculated.myNetMap.pub4MachineMap."${lb}"}/32"; device = "wan"; } 34 | ]; 35 | }) // flip mapAttrs' lbPrioMap (lb: priorities: nameValuePair "${lb}-6" { 36 | interface = "tlan"; 37 | trackInterfaces = [ "wan" ]; 38 | virtualRouterId = calculated.myNetMap.vrrpMap."${lb}-6"; 39 | priority = priorities."${config.networking.hostName}"; 40 | authType = "PASS"; 41 | authPass = "none"; 42 | virtualIpAddresses = [ 43 | { ip = "${calculated.myNetMap.pub6}${toString calculated.myNetMap.pub6MachineMap."${lb}"}/128"; device = "wan"; } 44 | ]; 45 | }); 46 | }; 47 | } 48 | -------------------------------------------------------------------------------- /core/mesos.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | with lib; 3 | let 4 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 5 | 6 | isMaster = flip any (attrNames calculated.myZookeeper.servers) 7 | (name: config.networking.hostName == name); 8 | myIp = calculated.myVpnIp4; 9 | servers = map (n: "${n}:2181") calculated.myZookeeper.serverIps; 10 | zkUrl = "zk://${concatStringsSep "," servers}/mesos"; 11 | in 12 | { 13 | imports = [ ./zookeeper.nix ]; 14 | 15 | networking.firewall = { 16 | extraCommands = mkMerge [ 17 | (mkOrder 0 '' 18 | # Cleanup if we haven't already 19 | iptables -D INPUT -p tcp --dport 5050 -j mesos || true 20 | iptables -D INPUT -p tcp --dport 5051 -j mesos || true 21 | iptables -F mesos || true 22 | iptables -X mesos || true 23 | ipset destroy mesos || true 24 | '') 25 | ('' 26 | # Allow remote mesos to communicate 27 | ipset create mesos hash:ip family inet 28 | ${flip concatMapStrings calculated.myZookeeper.serverIps (n: '' 29 | ipset add mesos "${n}" 30 | '')} 31 | iptables -N mesos 32 | iptables -A mesos -m set --match-set mesos src -j ACCEPT 33 | iptables -A mesos -j RETURN 34 | iptables -A INPUT -p tcp --dport 5050 -j mesos 35 | iptables -A INPUT -p tcp --dport 5051 -j mesos 36 | '') 37 | ]; 38 | extraStopCommands = '' 39 | iptables -D INPUT -p tcp --dport 5050 -j mesos || true 40 | iptables -D INPUT -p tcp --dport 5051 -j mesos || true 41 | iptables -F mesos || true 42 | iptables -X mesos || true 43 | ipset destroy mesos || true 44 | ''; 45 | }; 46 | 47 | services.mesos = { 48 | slave = { 49 | enable = true; 50 | master = zkUrl; 51 | extraCmdLineOptions = [ "--ip=${myIp}" ]; 52 | }; 53 | master = mkIf isMaster { 54 | enable = true; 55 | # We must have a majority of servers in the quorum 56 | quorum = (length calculated.myZookeeper.serverIps + 2) / 2; 57 | zk = zkUrl; 58 | extraCmdLineOptions = [ "--ip=${myIp}" ]; 59 | }; 60 | }; 61 | virtualisation.docker.enable = true; 62 | } 63 | -------------------------------------------------------------------------------- /common/sub/base-dnsmasq.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | with lib; 3 | let 4 | trustAnchor = readFile (pkgs.stdenv.mkDerivation { 5 | name = "dnssec-anchor-dnsmasq"; 6 | builder = pkgs.writeText "builder.sh" '' 7 | ${pkgs.gawk}/bin/awk ' 8 | { 9 | if (/Domain:/) { domain=$2; } 10 | if (/Id:/) { id=$2; } 11 | if (/Algorithm:/) { algorithm=$2; } 12 | if (/HashType:/) { hashtype=$2; } 13 | if (/Hash:/) { hash=$2; } 14 | } 15 | END { 16 | print domain "," id "," algorithm "," hashtype "," hash; 17 | } 18 | ' "${pkgs.dnssec-root}/share/dnssec/iana-root.txt" > $out 19 | ''; 20 | }); 21 | in 22 | { 23 | networking.firewall.extraCommands = '' 24 | # Allow dnsmasq to access dns servers 25 | ip46tables -A OUTPUT -m owner --uid-owner dnsmasq -p udp --dport domain -j ACCEPT 26 | ip46tables -A OUTPUT -m owner --uid-owner dnsmasq -p tcp --dport domain -j ACCEPT 27 | 28 | # Allow all users to access dns 29 | ip46tables -A OUTPUT -o lo -p udp --dport domain -j ACCEPT 30 | ip46tables -A OUTPUT -o lo -p tcp --dport domain -j ACCEPT 31 | ''; 32 | 33 | services.dnsmasq = { 34 | enable = true; 35 | resolveLocalQueries = true; 36 | extraConfig = '' 37 | domain-needed 38 | bogus-priv 39 | filterwin2k 40 | # Always reload when hosts change 41 | no-hosts 42 | addn-hosts=${config.environment.etc.hosts.source} 43 | 44 | # Enable Dnssec (disable for now until better resolvers can be added) 45 | #dnssec 46 | #trust-anchor=${trustAnchor} 47 | #dnssec-check-unsigned 48 | #dnssec-timestamp=/var/lib/dnsmasq/timestamp 49 | 50 | interface=lo 51 | no-dhcp-interface=lo 52 | 53 | bind-dynamic 54 | 55 | expand-hosts 56 | '' + concatStrings (flip mapAttrsToList config.myDns.forwardZones (zone: servers: 57 | flip concatMapStrings servers ({ server, port }: '' 58 | server=/${zone}/${server}#${port} 59 | '') 60 | )); 61 | }; 62 | 63 | systemd.services.dnsmasq.preStart = '' 64 | mkdir -p /var/lib/dnsmasq 65 | chmod 0700 /var/lib/dnsmasq 66 | chown dnsmasq /var/lib/dnsmasq 67 | ''; 68 | } 69 | -------------------------------------------------------------------------------- /web/unifi.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | let 3 | vars = (import ../customization/vars.nix { inherit lib; }); 4 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 5 | constants = (import ../common/sub/constants.nix { }); 6 | 7 | domain = "unifi.${calculated.myDomain}"; 8 | host = if calculated.iAmRemote then calculated.myVpnIp4 else calculated.myInternalIp4; 9 | in 10 | with lib; 11 | { 12 | imports = [ 13 | ./base.nix 14 | ]; 15 | 16 | networking.extraHosts = '' 17 | ${host} ${domain} 18 | ''; 19 | 20 | networking.firewall.extraCommands = '' 21 | # Allow access points to attach to the controller 22 | ip46tables -A INPUT -i mlan -p tcp --dport 8080 -j ACCEPT 23 | 24 | # Allow nginx to access unifi 25 | ip46tables -A OUTPUT -o lo -m owner --uid-owner nginx -p tcp --dport 8443 -j ACCEPT 26 | 27 | # Allow unifi to access mongodb 28 | ip46tables -A OUTPUT -o lo -m owner --uid-owner unifi -p tcp --dport 27117 -j ACCEPT 29 | ''; 30 | 31 | nixpkgs.config.allowUnfree = true; 32 | 33 | services = { 34 | nginx.config = '' 35 | server { 36 | listen *:443 ssl http2; 37 | listen [::]:443 ssl http2; 38 | server_name ${domain}; 39 | location / { 40 | ${flip concatMapStrings constants.privateIp4 (ip: '' 41 | allow ${ip}; 42 | '')} 43 | deny all; 44 | 45 | proxy_set_header Accept-Encoding ""; 46 | proxy_set_header Host $http_host; 47 | proxy_set_header X-Real-IP $remote_addr; 48 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 49 | proxy_set_header X-Forwarded-Proto $scheme; 50 | 51 | proxy_pass https://localhost:8443/; 52 | proxy_set_header Front-End-Https on; 53 | proxy_redirect off; 54 | } 55 | 56 | ${import sub/ssl-settings.nix { inherit domain; }} 57 | } 58 | 59 | server { 60 | listen *:80; 61 | listen [::]:80; 62 | server_name ${domain}; 63 | rewrite ^(.*) https://${domain}$1 permanent; 64 | } 65 | ''; 66 | 67 | unifi.enable = true; 68 | }; 69 | 70 | systemd.services.unifi.wantedBy = mkOverride 10 [ ]; 71 | } 72 | -------------------------------------------------------------------------------- /core/zookeeper.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | with lib; 3 | let 4 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 5 | 6 | servers = zipLists calculated.myZookeeper.serverIps 7 | (attrValues calculated.myZookeeper.servers); 8 | myId = calculated.myZookeeper.servers.${config.networking.hostName}; 9 | in 10 | { 11 | networking.firewall = { 12 | extraCommands = mkMerge [ 13 | (mkOrder 0 '' 14 | # Cleanup if we haven't already 15 | iptables -D INPUT -p tcp --dport 2181 -j zookeeper || true 16 | iptables -D INPUT -p tcp --dport 2888 -j zookeeper || true 17 | iptables -D INPUT -p tcp --dport 3888 -j zookeeper || true 18 | iptables -F zookeeper || true 19 | iptables -X zookeeper || true 20 | ipset destroy zookeeper || true 21 | '') 22 | ('' 23 | # Allow remote zookeepers to communicate 24 | ipset create zookeeper hash:ip family inet 25 | ${flip concatMapStrings calculated.myZookeeper.serverIps (n: '' 26 | ipset add zookeeper "${n}" 27 | '')} 28 | iptables -N zookeeper 29 | iptables -A zookeeper -m set --match-set zookeeper src -j ACCEPT 30 | iptables -A zookeeper -j RETURN 31 | iptables -A INPUT -p tcp --dport 2181 -j zookeeper 32 | iptables -A INPUT -p tcp --dport 2888 -j zookeeper 33 | iptables -A INPUT -p tcp --dport 3888 -j zookeeper 34 | 35 | # Allow zookeeper to connect to itself and other nodes 36 | ip46tables -A OUTPUT -p tcp --dport 2888 -m owner --uid-owner zookeeper -j ACCEPT 37 | ip46tables -A OUTPUT -p tcp --dport 3888 -m owner --uid-owner zookeeper -j ACCEPT 38 | '') 39 | ]; 40 | extraStopCommands = '' 41 | iptables -D INPUT -p tcp --dport 2181 -j zookeeper || true 42 | iptables -D INPUT -p tcp --dport 2888 -j zookeeper || true 43 | iptables -D INPUT -p tcp --dport 3888 -j zookeeper || true 44 | iptables -F zookeeper || true 45 | iptables -X zookeeper || true 46 | ipset destroy zookeeper || true 47 | ''; 48 | }; 49 | services.zookeeper = { 50 | enable = true; 51 | id = myId; 52 | servers = flip concatMapStrings servers ({ fst, snd }: '' 53 | server.${toString snd}=${fst}:2888:3888 54 | ''); 55 | }; 56 | } 57 | -------------------------------------------------------------------------------- /common/ceph.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | let 3 | numCephUsers = 48; 4 | calculated = (import ./sub/calculated.nix { inherit config lib; }); 5 | in 6 | with lib; 7 | { 8 | require = [ ./sub/ceph-module.nix ]; 9 | environment.systemPackages = [ config.cephPackage ]; 10 | environment.etc."ceph/ceph.conf".text = '' 11 | [global] 12 | fsid = ${calculated.myCeph.fsId}; 13 | mon initial members = ${concatStringsSep ", " calculated.myNetMap.ceph.mons} 14 | mon host = ${concatStringsSep ", " calculated.myCeph.monIps} 15 | log file = /dev/null 16 | log to stderr = false 17 | err to stderr = false 18 | log to syslog = true 19 | err to syslog = true 20 | mon cluster log to syslog = true 21 | mon cluster log file = /dev/null 22 | mgr module path = ${config.cephPackage.lib}/lib/ceph/mgr 23 | public network = ${calculated.myInternalIp4Net} 24 | auth cluster required = cephx 25 | auth service required = cephx 26 | auth client required = cephx 27 | ''; 28 | fileSystems = [ 29 | { 30 | mountPoint = "/etc/ceph"; 31 | fsType = "none"; 32 | device = "/conf/ceph"; 33 | neededForBoot = true; 34 | options = [ 35 | "defaults" 36 | "bind" 37 | ]; 38 | } 39 | ]; 40 | networking.firewall.extraCommands = '' 41 | # Allow inbound connections to ceph daemons 42 | # TODO(wak): Only enable when osd or mds is enabled 43 | iptables -A INPUT -p tcp --dport 6800:6900 -s ${calculated.myInternalIp4Net} -j ACCEPT 44 | 45 | # Allow connections to ceph mons 46 | iptables -A OUTPUT -d ${calculated.myInternalIp4Net} -p tcp --dport 6789 -j ACCEPT 47 | 48 | # Allow connections to other ceph services 49 | iptables -A OUTPUT -d ${calculated.myInternalIp4Net} -p tcp --dport 6800:6900 -j ACCEPT 50 | ''; 51 | users = { 52 | extraUsers = { 53 | ceph-mon = { 54 | uid = 100000; 55 | group = "ceph"; 56 | }; 57 | ceph-mds = { 58 | uid = 100001; 59 | group = "ceph"; 60 | }; 61 | ceph-mgr = { 62 | uid = 100002; 63 | group = "ceph"; 64 | }; 65 | } // listToAttrs (flip map (range 0 (numCephUsers - 1)) (n: 66 | nameValuePair "ceph-osd${toString n}" { uid = 100100 + n; group = "ceph"; useDefaultShell = true; })); 67 | extraGroups.ceph.gid = 100000; 68 | }; 69 | } 70 | -------------------------------------------------------------------------------- /common/ddns.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | let 4 | inherit (lib) 5 | length; 6 | 7 | vars = (import ../customization/vars.nix { inherit lib; }); 8 | 9 | host = "${config.networking.hostName}.${vars.domain}"; 10 | 11 | stateDir = "/var/lib/ddclient"; 12 | in 13 | { 14 | systemd.services."${host}-update" = { 15 | wantedBy = [ 16 | "multi-user.target" 17 | ]; 18 | wants = [ 19 | "network-online.target" 20 | ]; 21 | after = [ 22 | "network-online.target" 23 | ]; 24 | 25 | serviceConfig = { 26 | Type = "simple"; 27 | Restart = "on-failure"; 28 | RestartSec = "5s"; 29 | RestartPreventExitStatus = "80"; 30 | User = "ddclient"; 31 | }; 32 | 33 | path = [ 34 | pkgs.coreutils 35 | pkgs.curl 36 | pkgs.bind_tools 37 | pkgs.gawk 38 | pkgs.gnugrep 39 | pkgs.knot 40 | ]; 41 | 42 | script = '' 43 | set -e 44 | set -o pipefail 45 | 46 | # Make sure we can open our key file 47 | cat "/conf/ddns/${host}.key" >/dev/null || exit 80 48 | 49 | ip4="$(curl -4 -s "https://ifconfig.co")" 50 | if echo "$ip4" | grep -q 'Too Many Requests'; then 51 | echo "Too many attempts, waiting 60 seconds" 52 | sleep 60 53 | exit 1 54 | fi 55 | echo "Request got: $ip4" >&2 56 | echo "$ip4" | grep -q '^[0-9]\{1,3\}\(\.[0-9]\{1,3\}\)\{3\}''$' 57 | 58 | rc=0 59 | nameservers=() 60 | for nameserver in $(dig NS wak.io | grep 'IN\s\+NS\s\+.*wak.io' | awk '{print $5}'); do 61 | echo "Using nameservers: $nameserver" >&2 62 | 63 | commands="server $nameserver\n" 64 | commands+="ttl 300\n" 65 | commands+="zone ${host}.\n" 66 | commands+="origin ${host}.\n" 67 | commands+="del ${host}.\n" 68 | commands+="add ${host}. A $ip4\n" 69 | commands+="send\n" 70 | echo -n -e "$commands" | knsupdate -t 10 -r 3 -k "/conf/ddns/${host}.key" || rc=$? 71 | done 72 | exit $rc 73 | ''; 74 | }; 75 | 76 | systemd.timers."${host}-update" = { 77 | wantedBy = [ "multi-user.target" ]; 78 | 79 | timerConfig = { 80 | OnUnitActiveSec = "5m"; 81 | }; 82 | }; 83 | 84 | users.extraUsers.ddclient = { 85 | uid = config.ids.uids.ddclient; 86 | description = "ddclient daemon user"; 87 | home = stateDir; 88 | }; 89 | } 90 | -------------------------------------------------------------------------------- /common/tinc.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | with lib; 4 | let 5 | vars = (import ../customization/vars.nix { inherit lib; }); 6 | tincConfig = (import ../customization/tinc.nix { inherit lib; }); 7 | calculated = (import ./sub/calculated.nix { inherit config lib; }); 8 | in 9 | { 10 | imports = [ 11 | ./sub/vpn.nix 12 | ]; 13 | 14 | myNatIfs = [ 15 | "${vars.domain}.vpn" 16 | ]; 17 | 18 | environment.systemPackages = [ 19 | config.services.tinc.networks."${vars.domain}.vpn".package 20 | ]; 21 | 22 | fileSystems = [ 23 | { 24 | mountPoint = "/etc/tinc"; 25 | fsType = "none"; 26 | device = "/conf/tinc"; 27 | neededForBoot = true; 28 | options = [ 29 | "defaults" 30 | "bind" 31 | ]; 32 | } 33 | ]; 34 | 35 | networking = { 36 | firewall = { 37 | allowedTCPPorts = [ 655 ]; 38 | allowedUDPPorts = [ 655 ]; 39 | extraCommands = '' 40 | ip46tables -A OUTPUT -m owner --uid-owner ${vars.domain}.vpn -p udp --dport 655 -j ACCEPT 41 | ip46tables -A OUTPUT -m owner --uid-owner ${vars.domain}.vpn -p tcp --dport 655 -j ACCEPT 42 | ''; 43 | }; 44 | }; 45 | 46 | services.tinc.networks."${vars.domain}.vpn" = { 47 | package = pkgs.tinc_1_1; 48 | name = config.networking.hostName; 49 | extraConfig = '' 50 | StrictSubnets yes 51 | TunnelServer yes 52 | DirectOnly yes 53 | AutoConnect yes 54 | '' + flip concatMapStrings tincConfig.dedicated (n: '' 55 | ConnectTo ${n} 56 | ''); 57 | hosts = mkMerge [ 58 | tincConfig.hosts 59 | (flip mapAttrs tincConfig.hosts (host: _: 60 | let 61 | remote = calculated.isRemote host; 62 | netMap = vars.netMaps.${calculated.dc host}; 63 | hostMap = netMap.internalMachineMap.${host}; 64 | gateway = any (n: n == host) netMap.gateways; 65 | in '' 66 | Subnet = ${vars.vpn.subnet4}${toString vars.vpn.idMap.${host}}/32 67 | Subnet = ${vars.vpn.subnet6}${toString vars.vpn.idMap.${host}}/128 68 | '' + optionalString (!remote) ( 69 | flip concatMapStrings (hostMap.vlans) (vlan: '' 70 | Subnet = ${calculated.internalIp4 host vlan}/32 71 | '') + optionalString gateway '' 72 | Subnet = ${netMap.priv4}0.0/16 73 | '' 74 | ) 75 | )) 76 | ]; 77 | }; 78 | } 79 | -------------------------------------------------------------------------------- /common/sub/base-unbound.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | 3 | with lib; 4 | let 5 | vars = (import ../../customization/vars.nix { inherit lib; }); 6 | 7 | cores = config.nix.maxJobs; 8 | pwr = num: 9 | if num < 2 then 10 | 1 11 | else 12 | (pwr (builtins.div num 2))*2; 13 | p2cores = 14 | let 15 | p2bottom = pwr cores; 16 | in if p2bottom < cores then p2bottom*2 else p2bottom; 17 | in 18 | { 19 | # Make sure we always use unbound 20 | networking.hasLocalResolver = true; 21 | 22 | networking.firewall.extraCommands = '' 23 | # Allow unbound to access dns servers 24 | ip46tables -A OUTPUT -m owner --uid-owner unbound -p udp --dport domain -j ACCEPT 25 | ip46tables -A OUTPUT -m owner --uid-owner unbound -p tcp --dport domain -j ACCEPT 26 | 27 | # Allow all users to access dns 28 | ip46tables -A OUTPUT -o lo -p udp --dport domain -j ACCEPT 29 | ip46tables -A OUTPUT -o lo -p tcp --dport domain -j ACCEPT 30 | ''; 31 | 32 | services.unbound = { 33 | enable = true; 34 | extraConfig = concatStrings (flip mapAttrsToList config.myDns.forwardZones (name: servers: 35 | "forward-zone:\n name: ${name}\n" + flip concatMapStrings servers ({ server, port }: 36 | " forward-addr: \"${server}@${toString port}\"\n" 37 | ) 38 | )) + '' 39 | server: 40 | num-threads: ${toString cores} 41 | msg-cache-slabs: ${toString p2cores} 42 | rrset-cache-slabs: ${toString p2cores} 43 | infra-cache-slabs: ${toString p2cores} 44 | key-cache-slabs: ${toString p2cores} 45 | rrset-cache-size: 100m 46 | msg-cache-size: 50m 47 | outgoing-range: 8192 48 | num-queries-per-thread: 4096 49 | so-rcvbuf: 4m 50 | so-sndbuf: 4m 51 | so-reuseport: yes 52 | 53 | interface: 0.0.0.0 54 | interface: ::0 55 | access-control: 0.0.0.0/0 allow 56 | local-zone: "vpn.${vars.domain}" refuse 57 | ${flip concatMapStrings ([ null ] ++ attrNames vars.netMaps) (dc: 58 | flip concatMapStrings (attrNames vars.internalVlanMap) (lan: 59 | " local-zone: \"${lan}.${optionalString (dc != null) "${dc}."}${vars.domain}\" refuse\n" 60 | ) 61 | )} 62 | remote-control: 63 | control-enable: yes 64 | control-use-cert: no 65 | ''; 66 | }; 67 | 68 | myDns.forwardZones' = attrNames config.myDns.forwardZones; 69 | } 70 | -------------------------------------------------------------------------------- /web/base.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | with lib; 4 | { 5 | require = [ 6 | ./sub/acme-settings.nix 7 | ]; 8 | 9 | networking.firewall.allowedTCPPorts = [ 80 443 ]; 10 | 11 | services.nginx = { 12 | enable = true; 13 | package = pkgs.nginx; 14 | config = mkMerge [ 15 | (mkBefore ('' 16 | worker_processes 4; 17 | events { 18 | worker_connections 1024; 19 | } 20 | error_log syslog:server=unix:/dev/log crit; 21 | http { 22 | include ${config.services.nginx.package}/etc/nginx/mime.types; 23 | default_type application/octet-stream; 24 | access_log off; 25 | sendfile on; 26 | tcp_nopush on; 27 | aio threads; 28 | directio 4m; 29 | output_buffers 1 64k; 30 | keepalive_timeout 60; 31 | 32 | upstream acme { 33 | '' + flip concatMapStrings config.acmeServers (n: '' 34 | server ${n} max_fails=0; 35 | '') + '' 36 | } 37 | 38 | server { 39 | listen *:80; 40 | listen [::]:80; 41 | server_name default; 42 | location / { 43 | root ${config.services.nginx.package}/share/nginx/html; 44 | index index.html; 45 | } 46 | location /.well-known/acme-challenge { 47 | proxy_pass http://acme; 48 | } 49 | error_page 500 502 503 504 /50x.html; 50 | } 51 | server { 52 | listen *:443 ssl http2; 53 | listen [::]:443 ssl http2; 54 | server_name default; 55 | location / { 56 | root ${config.services.nginx.package}/share/nginx/html; 57 | index index.html; 58 | } 59 | error_page 500 502 503 504 /50x.html; 60 | 61 | ${import sub/ssl-settings.nix { domain = "nginx/default"; }} 62 | } 63 | '')) 64 | (mkAfter '' 65 | } 66 | '') 67 | ]; 68 | }; 69 | systemd.services.nginx = { 70 | wants = [ "ceph.mount" ]; 71 | after = [ "ceph.mount" ]; 72 | }; 73 | 74 | environment.etc."consul.d/web-base.json".text = builtins.toJSON { 75 | check = { 76 | id = "web-base"; 77 | name = "Nginx Serving Default"; 78 | http = "http://localhost"; 79 | interval = "10s"; 80 | timeout = "1s"; 81 | }; 82 | }; 83 | } 84 | -------------------------------------------------------------------------------- /core/ceph.mon.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, lib, ... }: 2 | with lib; 3 | let 4 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 5 | 6 | stateDir = "/var/lib/ceph/mon/ceph-${config.networking.hostName}"; 7 | monCfg = pkgs.writeText "ceph-mon.conf" '' 8 | [global] 9 | fsid = ${calculated.myCeph.fsId} 10 | mon initial members = ${concatStringsSep ", " calculated.myNetMap.ceph.mons} 11 | mon host = ${concatStringsSep ", " calculated.myCeph.monIps} 12 | mon data = ${stateDir} 13 | admin socket = /run/ceph/ceph-mon.${config.networking.hostName}.asok 14 | log file = /dev/null 15 | log to stderr = false 16 | err to stderr = false 17 | log to syslog = true 18 | err to syslog = true 19 | mon cluster log to syslog = true 20 | mon cluster log file = /dev/null 21 | public network = ${calculated.myInternalIp4Net} 22 | auth cluster required = cephx 23 | auth service required = cephx 24 | auth client required = cephx 25 | osd journal size = 1024 26 | filestore xattr use omap = true 27 | osd pool default size = 2 28 | osd pool default min size = 1 29 | osd pool default pg num = 333 30 | osd pool default pgp num = 333 31 | osd crush chooseleaf type = 1 32 | ''; 33 | in 34 | { 35 | imports = [ 36 | ../common/ceph.nix 37 | ]; 38 | networking.firewall.extraCommands = '' 39 | iptables -A INPUT -p tcp --dport 6789 -s ${calculated.myInternalIp4Net} -j ACCEPT 40 | ''; 41 | 42 | systemd.services.ceph-mon = { 43 | wantedBy = [ "multi-user.target" ]; 44 | after = [ "network.target" "time-syncd.target" ]; 45 | requires = [ "time-syncd.target" ]; 46 | 47 | # Updating crush maps requires crushtool in the path 48 | path = [ config.cephPackage ]; 49 | 50 | serviceConfig = { 51 | Type = "simple"; 52 | ExecStart = "@${config.cephPackage}/bin/ceph-mon ceph-mon -i ${config.networking.hostName} -c ${monCfg} -f"; 53 | User = "ceph-mon"; 54 | Group = "ceph"; 55 | PermissionsStartOnly = true; 56 | Restart = "always"; 57 | }; 58 | 59 | preStart = '' 60 | mkdir -p -m 0775 /var/lib/ceph/mon 61 | if [ ! -d "${stateDir}" ]; then 62 | ceph-mon -i ${config.networking.hostName} --mkfs --keyring /etc/ceph/ceph.client.admin.keyring 63 | fi 64 | chmod 0700 ${stateDir} 65 | chown -R ceph-mon ${stateDir} 66 | 67 | mkdir -p -m 0775 /var/run/ceph 68 | chown ceph-mon:ceph /var/run/ceph 69 | ''; 70 | }; 71 | } 72 | -------------------------------------------------------------------------------- /services/authdns.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | let 3 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 4 | 5 | outboundIp = "${calculated.myNetMap.pub4}${toString calculated.myNetMap.pub4MachineMap.outbound}"; 6 | in 7 | { 8 | environment.systemPackages = [ 9 | pkgs.knot 10 | ]; 11 | 12 | # Forward all of the incoming traffic to the correct port. 13 | networking.firewall.extraCommands = '' 14 | ip46tables -I INPUT -p udp --dport 1153 -j ACCEPT 15 | ip46tables -I INPUT -p tcp --dport 1153 -j ACCEPT 16 | ip46tables -t nat -A PREROUTING -i wan -p udp --dport 53 -j REDIRECT --to-port 1153 17 | ip46tables -t nat -A PREROUTING -i wan -p tcp --dport 53 -j REDIRECT --to-port 1153 18 | 19 | # Rewrite traffic to replicate from the correct ip 20 | iptables -t mangle -X nixos-knot-output || true 21 | iptables -t mangle -N nixos-knot-output 22 | iptables -t mangle -A nixos-knot-output -o lo -j RETURN 23 | iptables -t mangle -A nixos-knot-output -j MARK --set-mark 0x20 24 | iptables -t mangle -A OUTPUT -m owner -p tcp --dport 53 --uid-owner knot -j nixos-knot-output 25 | iptables -t mangle -A OUTPUT -m owner -p udp --dport 53 --uid-owner knot -j nixos-knot-output 26 | iptables -t nat -A POSTROUTING -m mark --mark 0x20 -j SNAT --to-source ${calculated.myInternalIp4} 27 | ''; 28 | 29 | networking.localCommands = '' 30 | ${pkgs.iproute}/bin/ip route del table 10 default || true 31 | ${pkgs.iproute}/bin/ip route add table 10 default via ${calculated.myGatewayIp4} 32 | ${pkgs.iproute}/bin/ip rule del fwmark 0x20 || true 33 | ${pkgs.iproute}/bin/ip rule add fwmark 0x20 table 10 34 | ''; 35 | 36 | systemd.services.knot = { 37 | after = [ "network.target" ]; 38 | wantedBy = [ "multi-user.target" ]; 39 | description = "knot authoritative dns"; 40 | preStart = '' 41 | rm -f /etc/knot 42 | ln -sv /ceph/knot/conf /etc/knot 43 | 44 | mkdir -p /run/knot 45 | chown knot:root /run/knot 46 | chmod 0755 /run/knot 47 | 48 | mkdir -p /var/lib/knot 49 | chown knot:root /var/lib/knot 50 | chmod 0700 /var/lib/knot 51 | rm -f /var/lib/knot/zones 52 | ln -sv /ceph/knot/zones /var/lib/knot/zones 53 | ''; 54 | 55 | serviceConfig = { 56 | ExecStart = "${pkgs.knot}/bin/knotd"; 57 | User = "knot"; 58 | PermissionsStartOnly = true; 59 | }; 60 | }; 61 | 62 | users.extraUsers.knot = { 63 | uid = config.ids.uids.knot; 64 | description = "knot daemon user"; 65 | }; 66 | } 67 | -------------------------------------------------------------------------------- /core/ceph.mgr.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | let 3 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 4 | 5 | stateDir = "/var/lib/ceph/mgr/ceph-${config.networking.hostName}"; 6 | 7 | pythonPackages = pkgs.python2Packages; 8 | 9 | cephMgr = pkgs.stdenv.mkDerivation { 10 | name = "ceph-mgr-wrapped"; 11 | 12 | nativeBuildInputs = [ 13 | pkgs.makeWrapper 14 | pythonPackages.python 15 | ]; 16 | 17 | pythonPath = with pythonPackages; [ 18 | cherrypy 19 | config.cephPackage.lib 20 | jinja2 21 | ]; 22 | 23 | unpackPhase = "true"; 24 | 25 | installPhase = '' 26 | declare -A pythonPathsSeen 27 | makePythonPath() { 28 | if [ -n "''${pythonPathsSeen[$1]}" ]; then return; fi 29 | pythonPathsSeen[$1]=1 30 | local ppath="$(toPythonPath "$1")" 31 | if [ -e "$ppath" ]; then 32 | addToSearchPath PYTHONPATH "$ppath" 33 | fi 34 | local prop="$1/nix-support/propagated-native-build-inputs" 35 | if [ -e "$prop" ]; then 36 | local new_path 37 | for new_path in $(cat "$prop"); do 38 | makePythonPath $new_path 39 | done 40 | fi 41 | } 42 | for lib in $pythonPath; do 43 | makePythonPath "$lib" 44 | done 45 | 46 | mkdir -p "$out"/bin 47 | ln -sv "${config.cephPackage}"/bin/ceph-mgr "$out"/bin 48 | wrapProgram "$out"/bin/ceph-mgr --prefix PYTHONPATH : "$PYTHONPATH" 49 | ''; 50 | }; 51 | in 52 | { 53 | imports = [ 54 | ../common/ceph.nix 55 | ]; 56 | 57 | networking.firewall.extraCommands = '' 58 | iptables -A INPUT -p tcp --dport 7000 -s ${calculated.myInternalIp4Net} -j ACCEPT 59 | ''; 60 | 61 | systemd.services.ceph-mgr = { 62 | wantedBy = [ "multi-user.target" ]; 63 | after = [ "network.target" ]; 64 | 65 | restartTriggers = [ config.environment.etc."ceph/ceph.conf".source ]; 66 | 67 | serviceConfig = { 68 | Type = "simple"; 69 | ExecStart = "@${cephMgr}/bin/ceph-mgr ceph-mgr -i ${config.networking.hostName} -f"; 70 | ExecStopPost = "${config.cephPackage}/bin/ceph mgr fail ${config.networking.hostName}"; 71 | User = "ceph-mgr"; 72 | Group = "ceph"; 73 | PermissionsStartOnly = true; 74 | Restart = "always"; 75 | }; 76 | 77 | preStart = '' 78 | mkdir -p ${stateDir} 79 | chmod 0700 ${stateDir} 80 | chown -R ceph-mgr:ceph ${stateDir} 81 | mkdir -p -m 0775 /var/run/ceph 82 | chown ceph-mon:ceph /var/run/ceph 83 | ''; 84 | }; 85 | } 86 | -------------------------------------------------------------------------------- /gateway/sub/base-firewall.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | with lib; 3 | let 4 | vars = (import ../../customization/vars.nix { inherit lib; }); 5 | calculated = (import ../../common/sub/calculated.nix { inherit config lib; }); 6 | 7 | natval = let 8 | outboundOrNull = calculated.myNetMap.pub4MachineMap.outbound or null; 9 | pub4OrNull = calculated.myNetMap.pub4 or null; 10 | in if outboundOrNull == null || pub4OrNull == null then "MASQUERADE" else "SNAT --to ${pub4OrNull}${toString outboundOrNull}"; 11 | 12 | nat6val = let 13 | outboundOrNull = calculated.myNetMap.pub6MachineMap.outbound or null; 14 | pub6OrNull = calculated.myNetMap.pub6 or null; 15 | in if outboundOrNull == null || pub6OrNull == null then "MASQUERADE" else "SNAT --to ${pub6OrNull}${toString outboundOrNull}"; 16 | in 17 | { 18 | networking.firewall.extraCommands = mkOrder 2 '' 19 | # Forward Outbound Connections 20 | ${concatStrings (map (n: '' 21 | ip46tables -w -A FORWARD -i ${n} -o wan -j ACCEPT 22 | ip46tables -w -A FORWARD -i ${n} -o gwan -j ACCEPT 23 | ip6tables -w -A FORWARD -i ${n} -o hurricane -j ACCEPT 24 | ip46tables -A FORWARD -i ${n} -o ${vars.domain}.vpn -j ACCEPT 25 | ip46tables -A FORWARD -i ${n} -o gw.${vars.domain}.vpn -j ACCEPT 26 | '') (filter (n: n != "${vars.domain}.vpn") config.myNatIfs))} 27 | 28 | # Masquerade all private connections 29 | iptables -t mangle -A PREROUTING -m set --match-set private src -j MARK --set-mark 0x10 30 | ip6tables -t mangle -A PREROUTING -m set --match-set private6 src -j MARK --set-mark 0x10 31 | #iptables -t nat -A POSTROUTING -m mark --mark 0x10 -j MASQUERADE 32 | 33 | # Masquerade all vpn connections 34 | # If we hit a node with vpn support we might have asymmetric routing otherwise. 35 | #iptables -t mangle -A PREROUTING -s "${vars.vpn.subnet4}0/24" -j MARK --set-mark 0x11 36 | #iptables -t mangle -A PREROUTING -s "${vars.vpn.remote4}0/24" -j MARK --set-mark 0x11 37 | #iptables -t nat -A POSTROUTING -m mark --mark 0x11 -j MASQUERADE 38 | 39 | # Masquerade all public connections 40 | iptables -t nat -A POSTROUTING -o wan -m mark --mark 0x10 -j ${natval} 41 | iptables -t nat -A POSTROUTING -o gwan -m mark --mark 0x10 -j ${natval} 42 | ip6tables -t nat -A POSTROUTING -o wan -m mark --mark 0x10 -j ${nat6val} 43 | ip6tables -t nat -A POSTROUTING -o gwan -m mark --mark 0x10 -j ${nat6val} 44 | 45 | # Allow access to servers 46 | ${concatStrings (map (n: '' 47 | ip46tables -A FORWARD -i ${n} -o slan -j ACCEPT 48 | '') (filter (n: n != "slan") config.myNatIfs))} 49 | ''; 50 | } 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # William's NixOS Config System 2 | 3 | TODO 4 | 5 | ## ZFS on Root Configuration 6 | The expected configuration of the root zpool for the system is as follows. 7 | 8 | ``` 9 | $ zfs list 10 | NAME USED AVAIL REFER MOUNTPOINT 11 | root 2.61G 88.4G 3.62M legacy 12 | root/conf 2.85M 88.4G 2.85M legacy 13 | root/log 148M 88.4G 148M legacy 14 | root/nix 1.79G 88.4G 1.79G legacy 15 | root/state 675M 88.4G 675M legacy 16 | root/state/postgresql 272K 88.4G 272K legacy 17 | root/tmp 1.20M 88.4G 1.20M legacy 18 | ``` 19 | 20 | ``` 21 | $ zfs get all | grep local 22 | root mountpoint legacy local 23 | root compression lz4 local 24 | root atime off local 25 | root xattr sa local 26 | root/conf devices off local 27 | root/conf setuid off local 28 | root/log devices off local 29 | root/log setuid off local 30 | root/state devices off local 31 | root/state setuid off local 32 | root/state/postgresql recordsize 8K local 33 | root/state/postgresql primarycache metadata local 34 | root/state/postgresql secondarycache metadata local 35 | root/state/postgresql logbias throughput local 36 | root/tmp devices off local 37 | root/tmp setuid off local 38 | root/tmp sync disabled local 39 | ``` 40 | 41 | ## ZFS on Ceph OSDs 42 | The expected configuration of the pool servicing a ceph osd. 43 | 44 | ``` 45 | $ zfs get all osdN | grep local 46 | osdN mountpoint legacy local 47 | osdN compression lz4 local 48 | osdN atime off local 49 | osdN devices off local 50 | osdN exec off local 51 | osdN setuid off local 52 | osdN xattr sa local 53 | ``` 54 | -------------------------------------------------------------------------------- /common/zfs-root.nix: -------------------------------------------------------------------------------- 1 | { lib, ... }: 2 | with lib; 3 | { 4 | imports = [ ./fs-root.nix ]; 5 | boot = { 6 | initrd.supportedFilesystems = [ "zfs" ]; 7 | zfs = { 8 | forceImportRoot = false; 9 | forceImportAll = false; 10 | }; 11 | }; 12 | fileSystems = mkMerge [ 13 | (mkOrder 0 [ 14 | { 15 | mountPoint = "/"; 16 | fsType = "zfs"; 17 | device = "root"; 18 | neededForBoot = true; 19 | } 20 | ]) 21 | (mkOrder 1 [ 22 | { 23 | mountPoint = "/conf"; 24 | fsType = "zfs"; 25 | device = "root/conf"; 26 | neededForBoot = true; 27 | } 28 | { 29 | mountPoint = "/nix"; 30 | fsType = "zfs"; 31 | device = "root/nix"; 32 | neededForBoot = true; 33 | } 34 | { 35 | mountPoint = "/state"; 36 | fsType = "zfs"; 37 | device = "root/state"; 38 | neededForBoot = true; 39 | } 40 | { 41 | mountPoint = "/tmp"; 42 | fsType = "zfs"; 43 | device = "root/tmp"; 44 | neededForBoot = true; 45 | } 46 | { 47 | mountPoint = "/var/log"; 48 | fsType = "zfs"; 49 | device = "root/log"; 50 | neededForBoot = true; 51 | } 52 | ]) 53 | (mkOrder 2 [ 54 | { 55 | mountPoint = "/etc/nixos"; 56 | fsType = "none"; 57 | device = "/conf/nixos"; 58 | neededForBoot = true; 59 | options = [ 60 | "defaults" 61 | "bind" 62 | ]; 63 | } 64 | { 65 | mountPoint = "/home"; 66 | fsType = "none"; 67 | device = "/state/home"; 68 | neededForBoot = true; 69 | options = [ 70 | "defaults" 71 | "bind" 72 | ]; 73 | } 74 | { 75 | mountPoint = "/root"; 76 | fsType = "none"; 77 | device = "/state/home/root"; 78 | neededForBoot = true; 79 | options = [ 80 | "defaults" 81 | "bind" 82 | ]; 83 | } 84 | { 85 | mountPoint = "/var/lib"; 86 | fsType = "none"; 87 | device = "/state/lib"; 88 | neededForBoot = true; 89 | options = [ 90 | "defaults" 91 | "bind" 92 | ]; 93 | } 94 | { 95 | mountPoint = "/var/db"; 96 | fsType = "none"; 97 | device = "/state/lib"; 98 | neededForBoot = true; 99 | options = [ 100 | "defaults" 101 | "bind" 102 | ]; 103 | } 104 | ]) 105 | ]; 106 | } 107 | -------------------------------------------------------------------------------- /common/sub/base-ntpd.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | let 4 | calculated = (import ./calculated.nix { inherit config lib; }); 5 | 6 | timeSyncdScript = pkgs.writeScript "time-syncd-script" '' 7 | #! ${pkgs.stdenv.shell} -e 8 | export PATH="/var/setuid-wrappers:${pkgs.chrony}/bin:${pkgs.gawk}/bin:${pkgs.gnugrep}/bin" 9 | out="$(sudo chronyc tracking)" || exit 1 10 | echo "$out" >&2 11 | sudo chronyc sourcestats 12 | if [ "$(echo "$out" | awk -F'[ ]*:[ ]*' '/Stratum/{print $2;}')" -eq "0" ]; then 13 | exit 1 14 | fi 15 | if ! echo "$out" | awk -F'[ ]*:[ ]*' '/System time/{print $2;}' | grep -q '^0.000'; then 16 | exit 1 17 | fi 18 | exit 0 19 | ''; 20 | in 21 | with lib; 22 | { 23 | services = { 24 | ntp = { 25 | servers = map ({ server, weight }: server) calculated.myNtpServers; 26 | }; 27 | chrony = { 28 | enable = true; 29 | initstepslew = { 30 | enable = true; 31 | threshold = "0.1"; 32 | }; 33 | extraConfig = '' 34 | leapsecmode slew 35 | maxslewrate 1000 36 | smoothtime 400 0.001 37 | leapsectz right/UTC 38 | ''; 39 | }; 40 | }; 41 | 42 | networking.firewall.extraCommands = '' 43 | ip46tables -A OUTPUT -m owner --uid-owner chrony -p udp --dport ntp -j ACCEPT 44 | ''; 45 | 46 | security.sudo = { 47 | enable = true; 48 | extraConfig = '' 49 | ALL ALL=(root) NOPASSWD: ${pkgs.chrony}/bin/chronyc tracking 50 | ALL ALL=(root) NOPASSWD: ${pkgs.chrony}/bin/chronyc sourcestats 51 | ''; 52 | }; 53 | 54 | systemd.targets.time-syncd = { 55 | description = "This target is met when the time is in sync with upstream servers."; 56 | requires = [ 57 | "time-syncd.service" 58 | ]; 59 | after = [ 60 | "time-syncd.service" 61 | ]; 62 | }; 63 | 64 | systemd.services.time-syncd = { 65 | serviceConfig = { 66 | Type = "oneshot"; 67 | TimeoutStartSec = "0"; 68 | }; 69 | 70 | script = '' 71 | echo "Checking for time sync" >&2 72 | while ! ${timeSyncdScript}; do 73 | sleep 5 74 | done 75 | echo "Time is in sync" >&2 76 | ''; 77 | }; 78 | 79 | environment.etc."consul.d/chronyd.json".text = builtins.toJSON { 80 | check = { 81 | id = "chronyd"; 82 | name = "Chrony Clock Sync"; 83 | args = [ (pkgs.writeScript "consul-check-time-syncd" '' 84 | #! ${pkgs.stdenv.shell} -e 85 | ${timeSyncdScript} || exit 2 # Exit 2 means critical 86 | '') ]; 87 | interval = "10s"; 88 | }; 89 | }; 90 | 91 | systemd.services.chronyd.postStart = lib.optionalString config.services.consul.enable '' 92 | while [ ! -e "/run/chrony/chronyd.sock" ]; do 93 | sleep 1 94 | done 95 | ${pkgs.acl}/bin/setfacl -m u:consul:rw /run/chrony/chronyd.sock 96 | ''; 97 | } 98 | -------------------------------------------------------------------------------- /gateway/sub/base-keepalived.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | let 4 | inherit (lib) 5 | attrNames 6 | concatMap 7 | flip 8 | length 9 | listToAttrs 10 | mapAttrs' 11 | mkIf 12 | nameValuePair; 13 | 14 | vars = (import ../../customization/vars.nix { inherit lib; }); 15 | calculated = (import ../../common/sub/calculated.nix { inherit config lib; }); 16 | 17 | internalVlanMap = listToAttrs (flip map (calculated.myNetData.vlans ++ [ "lan" ]) (v: 18 | nameValuePair v vars.internalVlanMap.${v} 19 | )); 20 | in 21 | mkIf (length calculated.myNetMap.gateways >= 2) { 22 | systemd.services.keepalived = { 23 | path = [ pkgs.iputils pkgs.iproute pkgs.gnugrep pkgs.gnused ]; 24 | preStart = '' 25 | gateway="" 26 | while [ -z "$gateway" ]; do 27 | gateway="$(ip route | grep default | sed -n 's,.*via \([^ ]*\).*,\1,p')" 28 | sleep 1 29 | done 30 | while ! ping -c 1 "$gateway" -W 1; do 31 | true 32 | done 33 | ''; 34 | }; 35 | 36 | services.keepalived = { 37 | enable = true; 38 | syncGroups.gateway.group = [ "wan-4" "wan-6" ] ++ concatMap (n: [ "${n}-4" "${n}-6" ]) (attrNames internalVlanMap); 39 | instances = flip mapAttrs' internalVlanMap (n: id: nameValuePair "${n}-4" { 40 | preempt = false; 41 | interface = "tlan"; 42 | trackInterfaces = [ n ]; 43 | virtualRouterId = calculated.myNetMap.vrrpMap."${n}-4"; 44 | priority = calculated.myNetData.id; 45 | authType = "PASS"; 46 | authPass = "none"; 47 | virtualIpAddresses = [ 48 | { ip = "${calculated.gatewayIp4 config.networking.hostName n}/32"; device = n; } 49 | ]; 50 | }) // flip mapAttrs' internalVlanMap (n: id: nameValuePair "${n}-6" { 51 | preempt = false; 52 | interface = "tlan"; 53 | trackInterfaces = [ n ]; 54 | virtualRouterId = calculated.myNetMap.vrrpMap."${n}-6"; 55 | priority = calculated.myNetData.id; 56 | authType = "PASS"; 57 | authPass = "none"; 58 | virtualIpAddresses = [ 59 | { ip = "${calculated.gatewayIp6 config.networking.hostName n}/128"; device = n; } 60 | ]; 61 | }) // { 62 | "wan-4" = { 63 | preempt = false; 64 | interface = "tlan"; 65 | trackInterfaces = [ "wan" ]; 66 | virtualRouterId = calculated.myNetMap.vrrpMap.wan-4; 67 | priority = calculated.myNetData.id; 68 | authType = "PASS"; 69 | authPass = "none"; 70 | virtualIpAddresses = [ 71 | { ip = "${calculated.myNetMap.pub4}${toString calculated.myNetMap.pub4MachineMap.outbound}/32"; device = "wan"; } 72 | ]; 73 | }; 74 | 75 | "wan-6" = { 76 | preempt = false; 77 | interface = "tlan"; 78 | trackInterfaces = [ "wan" ]; 79 | virtualRouterId = calculated.myNetMap.vrrpMap.wan-6; 80 | priority = calculated.myNetData.id; 81 | authType = "PASS"; 82 | authPass = "none"; 83 | virtualIpAddresses = [ 84 | { ip = "${calculated.myNetMap.pub6}${toString calculated.myNetMap.pub6MachineMap.outbound}/128"; device = "wan"; } 85 | ]; 86 | }; 87 | }; 88 | }; 89 | } 90 | -------------------------------------------------------------------------------- /web/ipfs.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, lib, ... }: 2 | let 3 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 4 | constants = (import ../common/sub/constants.nix { }); 5 | vars = (import ../customization/vars.nix { inherit lib; }); 6 | 7 | domain = "ipfs.${calculated.myDomain}"; 8 | topDomain = "ipfs.${vars.domain}"; 9 | consulService = "ipfs-gateway"; 10 | consulDomain = "${consulService}.service.consul.${vars.domain}"; 11 | checkDomain = "${consulService}.${config.networking.hostName}.${vars.domain}"; 12 | in 13 | with lib; 14 | { 15 | imports = [ 16 | ../common/ipfs.nix 17 | ]; 18 | 19 | networking.extraHosts = '' 20 | ${calculated.myInternalIp4} ${checkDomain} 21 | ''; 22 | 23 | services = { 24 | nginx.config = '' 25 | server { 26 | listen *:443 ssl http2; 27 | listen [::]:443 ssl http2; 28 | server_name ${domain}; 29 | server_name ${topDomain}; 30 | server_name ${consulDomain}; 31 | server_name ${checkDomain}; 32 | 33 | location / { 34 | proxy_set_header Accept-Encoding ""; 35 | proxy_set_header Host $http_host; 36 | proxy_set_header X-Real-IP $remote_addr; 37 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 38 | proxy_set_header X-Forwarded-Proto $scheme; 39 | 40 | proxy_pass http://localhost:8001/; 41 | proxy_set_header Front-End-Https on; 42 | proxy_redirect off; 43 | 44 | limit_except GET { 45 | deny all; 46 | } 47 | } 48 | 49 | ${import sub/ssl-settings.nix { inherit domain; }} 50 | } 51 | 52 | server { 53 | listen *:80; 54 | listen [::]:80; 55 | server_name ${domain}; 56 | 57 | location / { 58 | rewrite ^(.*) https://${domain}$1 permanent; 59 | } 60 | 61 | location /.well-known/acme-challenge { 62 | proxy_pass http://acme; 63 | } 64 | } 65 | 66 | server { 67 | listen *:80; 68 | listen [::]:80; 69 | server_name ${consulDomain}; 70 | location / { 71 | rewrite ^(.*) https://${consulDomain}$1 permanent; 72 | } 73 | 74 | location /.well-known/acme-challenge { 75 | proxy_pass http://acme; 76 | } 77 | } 78 | 79 | server { 80 | listen *:80; 81 | listen [::]:80; 82 | server_name ${topDomain}; 83 | location / { 84 | rewrite ^(.*) https://${topDomain}$1 permanent; 85 | } 86 | 87 | location /.well-known/acme-challenge { 88 | proxy_pass http://acme; 89 | } 90 | } 91 | ''; 92 | }; 93 | 94 | environment.etc."consul.d/${consulService}.json".text = builtins.toJSON { 95 | service = { 96 | name = consulService; 97 | port = 443; 98 | checks = [ 99 | { 100 | http = "https://${checkDomain}:443"; 101 | tls_skip_verify = true; 102 | interval = "10s"; 103 | timeout = "1s"; 104 | } 105 | ]; 106 | }; 107 | }; 108 | } 109 | -------------------------------------------------------------------------------- /common/sub/base-firewall.nix: -------------------------------------------------------------------------------- 1 | { pkgs, lib, ... }: 2 | let 3 | constants = (import ./constants.nix { inherit lib; }); 4 | in 5 | with lib; 6 | { 7 | boot.kernel.sysctl = { 8 | "net.ipv4.conf.all.forwarding" = true; 9 | "net.ipv4.conf.default.forwarding" = true; 10 | "net.ipv6.conf.all.forwarding" = true; 11 | "net.ipv6.conf.default.forwarding" = true; 12 | }; 13 | networking.firewall = { 14 | enable = true; 15 | checkReversePath = true; 16 | extraPackages = [ 17 | pkgs.ipset 18 | ]; 19 | extraCommands = mkMerge [ 20 | (mkOrder 1 '' 21 | # Default Policy 22 | ip46tables -P INPUT DROP 23 | ip46tables -P FORWARD DROP 24 | ip46tables -P OUTPUT DROP 25 | 26 | # Flush Old Rules 27 | ip46tables -F INPUT 28 | ip46tables -F OUTPUT 29 | ip46tables -F FORWARD 30 | ip46tables -t nat -F 31 | ip46tables -t nat -X 32 | ip46tables -t mangle -F 33 | ip46tables -t mangle -X 34 | 35 | # Remove old ipsets 36 | ipset destroy || true 37 | 38 | # Create an ipset for private ip4 addresses 39 | ipset create private hash:net family inet 40 | ${flip concatMapStrings constants.privateIp4 (i: '' 41 | ipset add private "${i}" 42 | '')} 43 | ipset create private6 hash:net family inet6 44 | ${flip concatMapStrings constants.privateIp6 (i: '' 45 | ipset add private6 "${i}" 46 | '')} 47 | 48 | # Allow Established Connnections 49 | ip46tables -A FORWARD -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT 50 | ip46tables -A OUTPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT 51 | 52 | # Allow root to make local connections 53 | ip46tables -A OUTPUT -m owner --uid-owner root -j ACCEPT 54 | 55 | # Allow ping + icmpv6 56 | iptables -A OUTPUT -p icmp --icmp-type echo-request -j ACCEPT 57 | ip6tables -A OUTPUT -p icmpv6 -j ACCEPT 58 | 59 | # Allow ssh 60 | ip46tables -A OUTPUT -p tcp --dport 22 -j ACCEPT 61 | # Allow mosh 62 | ip46tables -A OUTPUT -p udp --dport 60000:61000 -j ACCEPT 63 | # Allow http 64 | ip46tables -A OUTPUT -p tcp --dport 80 -j ACCEPT 65 | ip46tables -A OUTPUT -p tcp --dport 443 -j ACCEPT 66 | # Allow upnp 67 | ip46tables -A OUTPUT -p udp --dport 1900 -j ACCEPT 68 | ip46tables -A OUTPUT -p tcp --dport 1901 -j ACCEPT 69 | '') 70 | (mkAfter '' 71 | # Allow everything to make external connections 72 | ip46tables -A OUTPUT -o lo -j DROP 73 | iptables -A OUTPUT -m set --match-set private dst -j DROP 74 | ip6tables -A OUTPUT -m set --match-set private6 dst -j DROP 75 | ip46tables -A OUTPUT -j ACCEPT 76 | 77 | ip46tables -A FORWARD -j DROP 78 | '') 79 | ]; 80 | extraStopCommands = mkAfter '' 81 | # Flush Old Rules 82 | ip46tables -F INPUT || true 83 | ip46tables -F OUTPUT || true 84 | ip46tables -F FORWARD || true 85 | ip46tables -t nat -F || true 86 | ip46tables -t nat -X || true 87 | ip46tables -t mangle -F || true 88 | ip46tables -t mangle -X || true 89 | 90 | # Undo Default Policy 91 | ip46tables -P INPUT ACCEPT || true 92 | ip46tables -P FORWARD ACCEPT || true 93 | ip46tables -P OUTPUT ACCEPT || true 94 | 95 | # Remove old ipsets 96 | ipset destroy || true 97 | ''; 98 | }; 99 | } 100 | -------------------------------------------------------------------------------- /common/sub/base-networking.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | with lib; 3 | let 4 | vars = (import ../../customization/vars.nix { inherit lib; }); 5 | calculated = (import ./calculated.nix { inherit config lib; }); 6 | 7 | net = calculated.myNetMap; 8 | networkId = net.internalMachineMap.${config.networking.hostName}.id; 9 | hasWanIf = config.networking.interfaces ? "wan"; 10 | in 11 | { 12 | myNatIfs = calculated.myNetData.vlans; 13 | 14 | networking = mkMerge [ 15 | { 16 | dhcpcd.extraConfig = '' 17 | noipv4ll 18 | nohook mtu # Break Cable modem negotiation 19 | ''; 20 | } 21 | (mkIf (!calculated.iAmRemote) { 22 | defaultGateway = mkDefault (if !hasWanIf then calculated.myGatewayIp4 23 | else if calculated.myPublicIp4 != null then net.pub4Gateway else null); 24 | 25 | defaultGateway6 = mkDefault (if !hasWanIf then calculated.myGatewayIp6 26 | else if calculated.myPublicIp6 != null then net.pub6Gateway else null); 27 | 28 | interfaces = mkMerge [ 29 | { lan = { }; } 30 | (listToAttrs (flip map (calculated.myNetData.vlans ++ [ "lan" ]) (vlan: 31 | let 32 | vid = vars.internalVlanMap.${vlan}; 33 | in 34 | nameValuePair vlan { 35 | ip4 = mkOverride 0 ([ 36 | { address = "${net.priv4}${toString vid}.${toString networkId}"; prefixLength = 24; } 37 | ] ++ optional calculated.iAmOnlyGateway 38 | { address = "${net.priv4}${toString vid}.1"; prefixLength = 32; }); 39 | ip6 = mkOverride 0 ([ 40 | { address = "${net.priv6}${toString vid}::${toString networkId}"; prefixLength = 64; } 41 | ] ++ optionals (net ? pub6Routed) [ 42 | { address = "${net.pub6Routed}${toString vid}::${toString networkId}"; prefixLength = 64; } 43 | ]); 44 | } 45 | ))) 46 | (mkIf (calculated.myPublicIp4 != null) ({ 47 | wan = { 48 | ip4 = [ { 49 | address = calculated.myPublicIp4; 50 | prefixLength = net.pub4PrefixLength; 51 | } ]; 52 | ip6 = [ { 53 | address = calculated.myPublicIp6; 54 | prefixLength = net.pub6PrefixLength; 55 | } ]; 56 | }; 57 | })) 58 | ]; 59 | 60 | vlans = listToAttrs (flip map calculated.myNetData.vlans (vlan: 61 | nameValuePair vlan { 62 | id = vars.internalVlanMap.${vlan}; 63 | interface = if config.networking.interfaces ? "10glan" && vlan == "slan" then "10glan" else "lan"; 64 | } 65 | )); 66 | 67 | useDHCP = calculated.iAmGateway && ! (calculated.myNetMap ? pub4); 68 | 69 | nameservers = calculated.myDnsServers; 70 | }) 71 | (mkIf calculated.iAmRemote { 72 | useDHCP = true; 73 | }) 74 | ]; 75 | 76 | # Make sure that all of our local networks are routed internally 77 | systemd.services = mkIf (!calculated.iAmRemote && !calculated.iAmGateway) { 78 | "network-link-up-${head calculated.myNetData.vlans}" = let 79 | intf = head calculated.myNetData.vlans; 80 | dependency = "network-addresses-${intf}.service"; 81 | in { 82 | requires = [ dependency ]; 83 | after = [ dependency ]; 84 | postStart = concatStrings (flip mapAttrsToList vars.netMaps (dc: { priv4, ...}: '' 85 | ip route add "${priv4}0.0/16" dev "${intf}" via "${calculated.myGatewayIp4}" src "${calculated.myInternalIp4}" 86 | '')) + '' 87 | ip route add "${vars.vpn.remote4}0/24" dev "${intf}" via "${calculated.myGatewayIp4}" src "${calculated.myInternalIp4}" 88 | ''; 89 | }; 90 | }; 91 | } 92 | -------------------------------------------------------------------------------- /web/ceph.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, lib, ... }: 2 | let 3 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 4 | constants = (import ../common/sub/constants.nix { }); 5 | vars = (import ../customization/vars.nix { inherit lib; }); 6 | 7 | domain = "ceph.${calculated.myDomain}"; 8 | topDomain = "ceph.${vars.domain}"; 9 | consulService = "ceph-web"; 10 | consulDomain = "${consulService}.service.consul.${vars.domain}"; 11 | checkDomain = "${consulService}.${config.networking.hostName}.${vars.domain}"; 12 | in 13 | with lib; 14 | { 15 | imports = [ 16 | ./base.nix 17 | ]; 18 | 19 | networking.firewall.extraCommands = '' 20 | # Allow nginx to access ceph mgrs 21 | ip46tables -A OUTPUT -m owner --uid-owner nginx -p tcp --dport 7000 -j ACCEPT 22 | ''; 23 | 24 | networking.extraHosts = '' 25 | ${calculated.myInternalIp4} ${checkDomain} 26 | ''; 27 | 28 | services = { 29 | nginx.config = '' 30 | upstream ceph-backend { 31 | '' + (flip concatMapStrings calculated.myCeph.monIps (n: '' 32 | server ${n}:7000; 33 | '')) + '' 34 | } 35 | server { 36 | listen *:443 ssl http2; 37 | listen [::]:443 ssl http2; 38 | server_name ${domain}; 39 | server_name ${topDomain}; 40 | server_name ${consulDomain}; 41 | server_name ${checkDomain}; 42 | 43 | location / { 44 | proxy_set_header Accept-Encoding ""; 45 | proxy_set_header Host $http_host; 46 | proxy_set_header X-Real-IP $remote_addr; 47 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 48 | proxy_set_header X-Forwarded-Proto $scheme; 49 | 50 | proxy_pass http://ceph-backend; 51 | proxy_set_header Front-End-Https on; 52 | proxy_redirect off; 53 | } 54 | 55 | ${import sub/ssl-settings.nix { inherit domain; }} 56 | } 57 | 58 | server { 59 | listen *:80; 60 | listen [::]:80; 61 | server_name ${domain}; 62 | 63 | location / { 64 | rewrite ^(.*) https://${domain}$1 permanent; 65 | } 66 | 67 | location /.well-known/acme-challenge { 68 | proxy_pass http://acme; 69 | } 70 | } 71 | 72 | server { 73 | listen *:80; 74 | listen [::]:80; 75 | server_name ${consulDomain}; 76 | 77 | location / { 78 | rewrite ^(.*) https://${consulDomain}$1 permanent; 79 | } 80 | 81 | location /.well-known/acme-challenge { 82 | proxy_pass http://acme; 83 | } 84 | } 85 | 86 | server { 87 | listen *:80; 88 | listen [::]:80; 89 | server_name ${topDomain}; 90 | 91 | location / { 92 | rewrite ^(.*) https://${topDomain}$1 permanent; 93 | } 94 | 95 | location /.well-known/acme-challenge { 96 | proxy_pass http://acme; 97 | } 98 | } 99 | ''; 100 | }; 101 | 102 | environment.etc."consul.d/${consulService}.json".text = builtins.toJSON { 103 | service = { 104 | name = consulService; 105 | port = 443; 106 | checks = [ 107 | { 108 | script = '' 109 | export SSL_CERT_FILE="/etc/ssl/certs/ca-certificates.crt" 110 | # TODO: Get a new cert and remove -k 111 | if ${pkgs.curl}/bin/curl -k https://${checkDomain}/; then 112 | exit 0 113 | fi 114 | exit 2 # Critical 115 | ''; 116 | interval = "60s"; 117 | } 118 | ]; 119 | }; 120 | }; 121 | } 122 | -------------------------------------------------------------------------------- /web/consul.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, lib, ... }: 2 | let 3 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 4 | constants = (import ../common/sub/constants.nix { }); 5 | vars = (import ../customization/vars.nix { inherit lib; }); 6 | 7 | domain = "consul.${calculated.myDomain}"; 8 | topDomain = "consul.${vars.domain}"; 9 | consulService = "consul-web"; 10 | consulDomain = "${consulService}.service.consul.${vars.domain}"; 11 | checkDomain = "${consulService}.${config.networking.hostName}.${vars.domain}"; 12 | in 13 | with lib; 14 | { 15 | imports = [ 16 | ./base.nix 17 | ../common/consul.nix 18 | ]; 19 | 20 | networking.firewall.extraCommands = '' 21 | # Allow nginx to access consul http 22 | ip46tables -A OUTPUT -o lo -m owner --uid-owner nginx -p tcp --dport 8500 -j ACCEPT 23 | ''; 24 | 25 | networking.extraHosts = '' 26 | ${calculated.myInternalIp4} ${checkDomain} 27 | ''; 28 | 29 | services = { 30 | consul.extraConfig = { 31 | ui = true; 32 | }; 33 | nginx.config = '' 34 | server { 35 | listen *:443 ssl http2; 36 | listen [::]:443 ssl http2; 37 | server_name ${domain}; 38 | server_name ${topDomain}; 39 | server_name ${consulDomain}; 40 | server_name ${checkDomain}; 41 | 42 | location / { 43 | proxy_set_header Accept-Encoding ""; 44 | proxy_set_header Host $http_host; 45 | proxy_set_header X-Real-IP $remote_addr; 46 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 47 | proxy_set_header X-Forwarded-Proto $scheme; 48 | 49 | proxy_pass http://localhost:8500/; 50 | proxy_set_header Front-End-Https on; 51 | proxy_redirect off; 52 | 53 | limit_except GET { 54 | deny all; 55 | } 56 | } 57 | 58 | ${import sub/ssl-settings.nix { inherit domain; }} 59 | } 60 | 61 | server { 62 | listen *:80; 63 | listen [::]:80; 64 | server_name ${domain}; 65 | 66 | location / { 67 | rewrite ^(.*) https://${domain}$1 permanent; 68 | } 69 | 70 | location /.well-known/acme-challenge { 71 | proxy_pass http://acme; 72 | } 73 | } 74 | 75 | server { 76 | listen *:80; 77 | listen [::]:80; 78 | server_name ${consulDomain}; 79 | 80 | location / { 81 | rewrite ^(.*) https://${consulDomain}$1 permanent; 82 | } 83 | 84 | location /.well-known/acme-challenge { 85 | proxy_pass http://acme; 86 | } 87 | } 88 | 89 | server { 90 | listen *:80; 91 | listen [::]:80; 92 | server_name ${topDomain}; 93 | 94 | location / { 95 | rewrite ^(.*) https://${topDomain}$1 permanent; 96 | } 97 | 98 | location /.well-known/acme-challenge { 99 | proxy_pass http://acme; 100 | } 101 | } 102 | ''; 103 | }; 104 | 105 | environment.etc."consul.d/${consulService}.json".text = builtins.toJSON { 106 | service = { 107 | name = consulService; 108 | port = 443; 109 | checks = [ 110 | { 111 | script = '' 112 | export SSL_CERT_FILE="/etc/ssl/certs/ca-certificates.crt" 113 | # TODO: Get a new cert and remove -k 114 | if ${pkgs.curl}/bin/curl -k https://${checkDomain}/ui/; then 115 | exit 0 116 | fi 117 | exit 2 # Critical 118 | ''; 119 | interval = "60s"; 120 | } 121 | ]; 122 | }; 123 | }; 124 | } 125 | -------------------------------------------------------------------------------- /core/ctdbd.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | with lib; 4 | let 5 | needed = [ 6 | "functions" 7 | "events.d/00.ctdb" 8 | "events.d/01.reclock" 9 | "events.d/10.interface" 10 | "events.d/11.routing" 11 | "events.d/99.timeout" 12 | ]; 13 | files = { 14 | "ctdb/ctdbd.conf".text = '' 15 | CTDBD=${samba}/bin/ctdbd 16 | CTDB_PIDFILE=/run/ctdb/ctdb.pid 17 | CTDB_BASE=/etc/ctdb 18 | CTDB_DBDIR=/var/lib/ctdb 19 | CTDB_DBDIR_PERSISTENT=/var/lib/ctdb/persistent 20 | CTDB_DBDIR_STATE=/var/lib/ctdb/state 21 | CTDB_DEBUGLEVEL=5 22 | CTDB_EVENT_SCRIPT_DIR=/etc/ctdb/events.d 23 | CTDB_LOGGING=syslog 24 | CTDB_NODES=/etc/ctdb/nodes 25 | CTDB_PUBLIC_ADDRESSES=/etc/ctdb/public_addresses 26 | CTDB_RECOVERY_LOCK=/ceph/ctdb/reclock 27 | CTDB_SOCKET=/run/ctdb/ctdbd.socket 28 | CTDB_STARTUP_TIMEOUT=60 29 | 30 | CTDB_VARDIR=/var/lib/ctdb 31 | 32 | CTDB_NOSETSCHED=yes 33 | ''; 34 | "ctdb/public_addresses".text = '' 35 | ''; 36 | "ctdb/nodes".text = '' 37 | ''; 38 | } // listToAttrs (flip map needed 39 | (n: nameValuePair "ctdb/${n}" { source = "${samba}/etc/ctdb/${n}"; })); 40 | 41 | samba = config.services.samba.package; 42 | 43 | ctdbPath = pkgs.buildEnv { 44 | name = "ctdb-path"; 45 | paths = [ 46 | samba 47 | pkgs.coreutils 48 | pkgs.gawk 49 | pkgs.gnugrep 50 | pkgs.gnused 51 | pkgs.ethtool 52 | pkgs.iproute 53 | pkgs.iptables 54 | pkgs.net-tools 55 | pkgs.procps 56 | pkgs.tdb 57 | pkgs.util-linux_full 58 | pkgs.which 59 | ]; 60 | pathsToLink = [ 61 | "/bin" 62 | ]; 63 | ignoreCollisions = true; 64 | }; 65 | in 66 | { 67 | require = [ 68 | ./sub/ctdbd.module.nix 69 | ]; 70 | 71 | networking.firewall = { 72 | extraCommands = mkIf config.myCtdbd.enable (mkMerge [ 73 | (mkOrder 0 '' 74 | # Cleanup if we haven't already 75 | iptables -D INPUT -p tcp --dport 4379 -j ctdb || true 76 | iptables -F ctdb || true 77 | iptables -X ctdb || true 78 | ipset destroy ctdb || true 79 | '') 80 | ('' 81 | # Allow remote ctdb instances to sync up 82 | ipset create ctdb hash:ip family inet 83 | iptables -N ctdb 84 | iptables -A ctdb -m set --match-set ctdb src -j ACCEPT 85 | iptables -A ctdb -j RETURN 86 | iptables -A INPUT -p tcp --dport 4379 -j ctdb 87 | 88 | # Allow ctdb to connect to itself and other nodes 89 | ip46tables -A OUTPUT -p tcp --dport 4379 -m owner --uid-owner root -j ACCEPT 90 | '') 91 | ]); 92 | 93 | extraStopCommands = mkIf config.myCtdbd.enable '' 94 | iptables -D INPUT -p tcp --dport 4379 -j ctdb || true 95 | iptables -F ctdb || true 96 | iptables -X ctdb || true 97 | ipset destroy ctdb || true 98 | ''; 99 | }; 100 | 101 | environment.etc = mkIf config.myCtdbd.enable files; 102 | 103 | systemd.services.ctdbd = mkIf config.myCtdbd.enable { 104 | description = "CTDB Daemon"; 105 | wantedBy = [ "multi-user.target" ]; 106 | requires = [ "ceph.mount" "network.target" ]; 107 | after = [ "ceph.mount" "network.target" ]; 108 | 109 | path = [ ctdbPath ]; 110 | 111 | environment.CTDBD_CONF = "/etc/ctdb/ctdbd.conf"; 112 | 113 | restartTriggers = flip mapAttrsToList files 114 | (name: data: config.environment.etc.${name}.source); 115 | 116 | preStop = '' 117 | ${samba}/bin/ctdb stop 118 | ${samba}/bin/ctdb shutdown 119 | ''; 120 | 121 | serviceConfig = { 122 | Type = "forking"; 123 | ExecStart = "@${samba}/bin/ctdbd_wrapper ctdbd /run/ctdb/ctdbd.pid start"; 124 | PIDFile = "/run/ctdb/ctdbd.pid"; 125 | }; 126 | }; 127 | } 128 | -------------------------------------------------------------------------------- /gateway/dhcpd.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, utils, ... }: 2 | with lib; 3 | let 4 | vars = (import ../customization/vars.nix { inherit lib; }); 5 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 6 | 7 | internalVlanMap = listToAttrs (flip map (calculated.myNetData.vlans ++ [ "lan" ]) (v: 8 | nameValuePair v vars.internalVlanMap.${v} 9 | )); 10 | systemdDevices = flip map (attrNames internalVlanMap) 11 | (n: "sys-subsystem-net-devices-${utils.escapeSystemdPath n}.device"); 12 | 13 | net = calculated.myNetMap; 14 | 15 | primary = config.networking.hostName == head net.dhcpServers; 16 | peer = if length net.dhcpServers < 2 then null 17 | else if primary then head (tail net.dhcpServers) else head net.dhcpServers; 18 | in 19 | { 20 | assertions = [ { 21 | assertion = length net.dhcpServers < 3; 22 | message = "You must not have more than 2 dhcp servers."; 23 | } ]; 24 | 25 | networking.firewall.extraCommands = '' 26 | ip46tables -A INPUT -i tlan -p tcp --dport 647 -j ACCEPT 27 | ip46tables -A OUTPUT -o tlan -m owner --uid-owner dhcpd -p tcp --dport 647 -j ACCEPT 28 | '' + flip concatMapStrings (attrNames internalVlanMap) (n: '' 29 | iptables -w -A INPUT -i ${n} -p udp --dport 67 -j ACCEPT 30 | ''); 31 | 32 | services.dhcpd = { 33 | enable = true; 34 | interfaces = attrNames internalVlanMap; 35 | configFile = pkgs.writeText "dhcpd.conf" ('' 36 | authoritative; 37 | ddns-updates off; 38 | log-facility local1; # see dhcpd.nix 39 | 40 | max-lease-time 86400; 41 | default-lease-time 86400; 42 | 43 | option subnet-mask 255.255.255.0; 44 | '' + optionalString (peer != null) '' 45 | failover peer failover { 46 | ${if primary then "primary" else "secondary"}; 47 | address ${calculated.internalIp4 config.networking.hostName "tlan"}; 48 | port 647; 49 | peer address ${calculated.internalIp4 peer "tlan"}; 50 | peer port 647; 51 | max-response-delay 30; 52 | max-unacked-updates 20; 53 | mclt 3600; 54 | ${optionalString primary "split 128;"} 55 | load balance max seconds 3; 56 | } 57 | '' + concatStrings (flip mapAttrsToList internalVlanMap (vlan: vid: 58 | let 59 | subnet = "${net.priv4}${toString vid}"; 60 | nameservers = concatStringsSep ", " (calculated.dnsIp4 vlan); 61 | timeservers = concatStringsSep ", " (map ({ server, ... }: server) (calculated.ntpIp4 vlan)); 62 | dhcpLower = "${subnet}.${toString vars.gateway.dhcpRange.lower}"; 63 | dhcpUpper = "${subnet}.${toString vars.gateway.dhcpRange.upper}"; 64 | in '' 65 | subnet ${subnet}.0 netmask 255.255.255.0 { 66 | option broadcast-address ${subnet}.255; 67 | option routers ${subnet}.1; 68 | option domain-name-servers ${nameservers}; 69 | option time-servers ${timeservers}; 70 | option domain-name "${calculated.myDomain}"; 71 | pool { 72 | ${optionalString (peer != null) "failover peer \"failover\";"} 73 | range ${dhcpLower} ${dhcpUpper}; 74 | } 75 | } 76 | '' 77 | )) + concatStrings (flip mapAttrsToList calculated.myNetMap.internalMachineMap (host: data: '' 78 | '' + optionalString (data ? mac) '' 79 | host ${host} { 80 | hardware ethernt ${data.mac}; 81 | ${flip concatMapStrings data.vlans (vlan: '' 82 | fixed-address ${calculated.internalIp4 host vlan}; 83 | '')} 84 | } 85 | '' + optionalString (data ? bmcMac) '' 86 | host ${host}-bmc { 87 | hardware ethernet ${data.bmcMac}; 88 | fixed-address ${calculated.bmcIp4 host}; 89 | } 90 | ''))); 91 | }; 92 | 93 | systemd.services.dhcpd = { 94 | after = systemdDevices; 95 | bindsTo = systemdDevices; 96 | partOf = systemdDevices; 97 | }; 98 | } 99 | -------------------------------------------------------------------------------- /common/base.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | with lib; 3 | let 4 | vars = (import ../customization/vars.nix { inherit lib; }); 5 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 6 | in 7 | { 8 | imports = [ 9 | # ./sub/base-dnsmasq.nix # We are using unbound now 10 | ./sub/base-firewall.nix 11 | ./sub/base-hosts.nix 12 | ./sub/base-networking.nix 13 | ./sub/base-ntpd.nix 14 | ./sub/base-ssh.nix 15 | ./sub/base-unbound.nix 16 | ]; 17 | require = [ 18 | ./sub/base-dns-module.nix 19 | ./sub/base-fs-module.nix 20 | ./sub/base-if-module.nix 21 | ./sub/base-keepalived-module.nix 22 | ]; 23 | boot = { 24 | kernelPackages = pkgs.linuxPackages_latest; 25 | zfs.useDev = true; # We want encryption 26 | supportedFilesystems = [ "ext4" "xfs" "btrfs" "vfat" "ntfs" ]; 27 | extraModprobeConfig = '' 28 | options kvm-amd nested=1 29 | options kvm-intel nested=1 30 | ''; 31 | kernel.sysctl = { 32 | "net.ipv4.ip_nonlocal_bind" = 1; 33 | "net.ipv6.conf.all.use_tempaddr" = 2; 34 | "net.ipv6.conf.default.use_tempaddr" = 2; 35 | }; 36 | }; 37 | environment.etc."ssl/openssl.cnf".source = "${pkgs.openssl}/etc/ssl/openssl.cnf"; 38 | environment.systemPackages = with pkgs; [ 39 | acme-sh 40 | acpi 41 | aria2 42 | bind_tools 43 | borgbackup 44 | curl 45 | dmidecode 46 | edac-utils 47 | efibootmgr 48 | efivar 49 | elvish 50 | fish 51 | git 52 | gnupg 53 | gptfdisk 54 | hdparm 55 | htop 56 | iftop 57 | iotop 58 | iperf 59 | ipmitool 60 | ipset 61 | iptables 62 | jq 63 | ldns 64 | lftp 65 | lm-sensors 66 | mcelog 67 | mtr 68 | ncdu 69 | nftables 70 | nmap 71 | nvme-cli 72 | config.programs.ssh.package 73 | openssl 74 | pciutils 75 | pinentry 76 | psmisc 77 | rsync 78 | smartmontools 79 | sysstat 80 | tcpdump 81 | time 82 | tmux 83 | vim 84 | unzip 85 | usbutils 86 | ]; 87 | hardware.cpu = { 88 | intel.updateMicrocode = true; 89 | amd.updateMicrocode = true; 90 | }; 91 | networking.domain = calculated.myDomain; 92 | networking.search = [ vars.domain ]; 93 | nix = { 94 | buildCores = config.nix.maxJobs; 95 | allowedUsers = [ "@wheel" ]; 96 | }; 97 | programs = { 98 | bash = { 99 | enableCompletion = true; 100 | promptInit = "PS1=\"[\\u@\\h:\\w]\\\\$ \"\n"; 101 | }; 102 | }; 103 | services = { 104 | journald.extraConfig = "SystemMaxUse=1G"; 105 | logind.extraConfig = "HandleLidSwitch=suspend"; 106 | }; 107 | system.extraDependencies = with pkgs; [ 108 | # We always want to keep small trust roots 109 | cacert 110 | root-nameservers 111 | ]; 112 | systemd.extraConfig = '' 113 | DefaultCPUAccounting=on 114 | DefaultMemoryAccounting=on 115 | DefaultBlockIOAccounting=on 116 | DefaultTasksAccounting=on 117 | DefaultIPAccounting=on 118 | ''; 119 | users = { 120 | mutableUsers = false; 121 | extraUsers = flip mapAttrs vars.userInfo (user: 122 | { uid, description, canRoot, loginMachines, canShareData, sshKeys }: 123 | let 124 | canLogin = if user == "root" then true else any (n: n == config.networking.hostName) loginMachines; 125 | in (if user == "root" then { } else { 126 | inherit uid description; 127 | createHome = canLogin; 128 | home = "/home/${user}"; 129 | group = if canLogin then "users" else "nogroup"; 130 | extraGroups = [ ] 131 | ++ optional canRoot "wheel" 132 | ++ optional canShareData "share"; 133 | useDefaultShell = canLogin; 134 | }) // { 135 | hashedPassword = null; 136 | passwordFile = if canLogin then "/conf/pw/${user}" else null; 137 | openssh.authorizedKeys.keys = if canLogin then sshKeys else [ ]; 138 | }); 139 | extraGroups = { 140 | share.gid = 1001; 141 | }; 142 | }; 143 | time.timeZone = mkDefault calculated.myTimeZone; 144 | } 145 | -------------------------------------------------------------------------------- /common/sub/calculated.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | with lib; 3 | let 4 | vars = (import ../../customization/vars.nix { inherit lib; }); 5 | host = config.networking.hostName; 6 | in 7 | rec { 8 | isRemote = name: any (n: n == name) vars.remotes; 9 | dc = name: let 10 | dcs = flip filterAttrs vars.netMaps (dc: { internalMachineMap, ... }: 11 | internalMachineMap ? "${name}"); 12 | in if isRemote name then "remote" else head (attrNames dcs); 13 | vpnIp4 = name: "${vars.vpn.subnet4}${toString vars.vpn.idMap.${name}}"; 14 | vpnIp6 = name: "${vars.vpn.subnet6}${toString vars.vpn.idMap.${name}}"; 15 | vpnGwIp4 = name: "${vars.vpn.remote4}${toString vars.vpn.idMap.${name}}"; 16 | vpnGwIp6 = name: "${vars.vpn.remote6}${toString vars.vpn.idMap.${name}}"; 17 | internalIp4Net = name: lan: let ndc = dc name; net = vars.netMaps.${ndc}; in 18 | "${net.priv4}${toString vars.internalVlanMap.${lan}}.0/24"; 19 | internalIp4 = name: lan: let ndc = dc name; net = vars.netMaps.${ndc}; in 20 | "${net.priv4}${toString vars.internalVlanMap.${lan}}.${toString net.internalMachineMap.${name}.id}"; 21 | bmcIp4 = name: let ndc = dc name; net = vars.netMaps.${ndc}; in 22 | "${net.priv4}${toString vars.internalVlanMap."mlan"}.1${toString net.internalMachineMap.${name}.id}"; 23 | publicIp4 = name: let ndc = dc name; net = vars.netMaps.${ndc}; in 24 | if !(net ? "pub4MachineMap" && net.pub4MachineMap ? "${name}") then null 25 | else "${net.pub4}${toString net.pub4MachineMap.${name}}"; 26 | publicIp6 = name: let ndc = dc name; net = vars.netMaps.${ndc}; in 27 | if !(net ? "pub6MachineMap" && net.pub6MachineMap ? "${name}") then null 28 | else "${net.pub6}${toString net.pub6MachineMap.${name}}"; 29 | gatewayIp4 = name: lan: let ndc = dc name; net = vars.netMaps.${ndc}; in 30 | "${net.priv4}${toString vars.internalVlanMap.${lan}}.1"; 31 | gatewayIp6 = name: lan: let ndc = dc name; net = vars.netMaps.${ndc}; in 32 | "${net.priv6}${toString vars.internalVlanMap.${lan}}::1"; 33 | domain = name: "${dc name}.${vars.domain}"; 34 | dnsIp4 = lan: map (flip internalIp4 lan) myNetMap.dnsServers; 35 | ntpIp4 = lan: flip map myNetMap.ntpServers ({ server, weight }: { 36 | server = internalIp4 server lan; 37 | inherit weight; 38 | }); 39 | 40 | iAmRemote = isRemote host; 41 | myDc = dc host; 42 | myDomain = domain host; 43 | myVpnIp4 = vpnIp4 host; 44 | myVpnIp6 = vpnIp6 host; 45 | myInternalIp4 = internalIp4 host (head myNetData.vlans); 46 | myPublicIp4 = publicIp4 host; 47 | myPublicIp6 = publicIp6 host; 48 | myGatewaysIp4 = map (gatewayIp4 host) myNetData.vlans; 49 | myGatewayIp4 = head myGatewaysIp4; 50 | myGatewaysIp6 = map (gatewayIp6 host) myNetData.vlans; 51 | myGatewayIp6 = head myGatewaysIp6; 52 | myNasIp4s = flip map myNetMap.nasIds 53 | (n: "${myNetMap.priv4}${toString vars.internalVlanMap."dlan"}.${toString n}"); 54 | myInternalIp4Net = internalIp4Net host (head myNetData.vlans); 55 | myNetMap = vars.netMaps.${myDc}; 56 | myNetData = myNetMap.internalMachineMap.${host}; 57 | iAmGateway = !iAmRemote && any (n: host == n) myNetMap.gateways; 58 | iAmOnlyGateway = iAmGateway && length (myNetMap.gateways) == 1; 59 | myTimeZone = if iAmRemote then "UTC" else myNetMap.timeZone; 60 | 61 | myDnsServers = 62 | if iAmRemote then 63 | vars.pubDnsServers 64 | else if any (n: host == n) myNetMap.dnsServers then 65 | myNetMap.pubDnsServers 66 | else 67 | dnsIp4 (head myNetData.vlans); 68 | 69 | myNtpServers = 70 | if iAmRemote then 71 | vars.pubNtpServers 72 | else if any (n: host == n.server) myNetMap.ntpServers then 73 | myNetMap.pubNtpServers 74 | else 75 | ntpIp4 (head myNetData.vlans); 76 | 77 | myCeph = { 78 | mons = myNetMap.ceph.mons; 79 | monIps = map (s: internalIp4 s "slan") myNetMap.ceph.mons; 80 | fsId = myNetMap.ceph.fsId; 81 | osds = myNetMap.ceph.osds.${host}; 82 | }; 83 | myConsul = { 84 | servers = myNetMap.consul.servers; 85 | serverIps = map (s: internalIp4 s "slan") myNetMap.consul.servers; 86 | }; 87 | myMongodb = { 88 | servers = myNetMap.mongodb.servers; 89 | serverIps = map vpnIp4 myNetMap.mongodb.servers; 90 | }; 91 | myZookeeper = { 92 | servers = myNetMap.zookeeper.servers; 93 | serverIps = map vpnIp4 (attrNames myNetMap.zookeeper.servers); 94 | url = "zk://${concatStringsSep "," serverIps}"; 95 | }; 96 | } 97 | -------------------------------------------------------------------------------- /nas/samba.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | let 4 | inherit (lib) 5 | concatMapStrings 6 | concatStringsSep 7 | flip 8 | length 9 | mkAfter 10 | mkIf 11 | mkMerge 12 | optionalAttrs 13 | optionalString 14 | optionals; 15 | 16 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 17 | net = calculated.myNasIps; 18 | 19 | clustered = length calculated.myNetMap.nases >= 2; 20 | 21 | vfsObjects = optionals clustered "fileid"; 22 | 23 | vfsString = optionalString (vfsObjects != [ ]) 24 | ("vfs objects = " + concatStringsSep " " vfsObjects); 25 | in 26 | { 27 | imports = [ 28 | ../core/ctdbd.nix 29 | ]; 30 | 31 | myCtdbd.enable = clustered; 32 | 33 | networking.firewall.extraCommands = mkMerge [ 34 | (flip concatMapStrings [ "mlan" "dlan" ] (n: '' 35 | # Samba ports 36 | ip46tables -A INPUT -i ${n} -p tcp --dport 135 -j ACCEPT 37 | ip46tables -A INPUT -i ${n} -p tcp --dport 139 -j ACCEPT 38 | ip46tables -A INPUT -i ${n} -p tcp --dport 445 -j ACCEPT 39 | ip46tables -A INPUT -i ${n} -p udp --dport 137 -j ACCEPT 40 | ip46tables -A INPUT -i ${n} -p udp --dport 138 -j ACCEPT 41 | '')) 42 | (mkIf clustered (mkAfter ( 43 | flip concatMapStrings calculated.myNetMap.nases (n: '' 44 | ipset add ctdb "${calculated.vpnIp4 n}" 45 | '') 46 | ))) 47 | ]; 48 | 49 | environment.systemPackages = with pkgs; [ 50 | samba_full 51 | ]; 52 | 53 | environment.etc = mkIf clustered { 54 | "ctdb/public_addresses".text = 55 | flip concatMapStrings calculated.myNasIp4s (n: '' 56 | ${n}/24 dlan 57 | ''); 58 | 59 | "ctdb/nodes".text = 60 | flip concatMapStrings calculated.myNetMap.nases (n: '' 61 | ${calculated.vpnIp4 n} 62 | ''); 63 | }; 64 | 65 | services.samba = { 66 | enable = true; 67 | extraConfig = '' 68 | workgroup = ${calculated.myDomain} 69 | server string = %h 70 | security = user 71 | map to guest = Bad User 72 | load printers = no 73 | guest account = nobody 74 | invalid users = root 75 | logging = systemd 76 | log level = 1 77 | max log size = 5000 78 | passdb backend = tdbsam 79 | local master = no 80 | preferred master = yes 81 | dns proxy = no 82 | store dos attributes = yes 83 | map hidden = no 84 | map system = no 85 | map archive = no 86 | nt acl support = yes 87 | inherit acls = yes 88 | map acl inherit = yes 89 | encrypt passwords = yes 90 | client plaintext auth = no 91 | idmap config * : range = 100000-100100 92 | 93 | # Clustered storage setup 94 | netbios name = ${calculated.myDomain} 95 | ${optionalString clustered '' 96 | clustering = yes 97 | idmap config * : backend = tdb2 98 | fileid : algorithm = fsid 99 | ''} 100 | 101 | # Performance 102 | socket options = TCP_NODELAY SO_SNDBUF=131072 SO_RCVBUF=131072 103 | use sendfile = yes 104 | min receivefile size = 16384 105 | aio read size = 16384 106 | aio write size = 16384 107 | 108 | [Private] 109 | path = /ceph/share/private/%u 110 | guest ok = no 111 | public = no 112 | writable = yes 113 | printable = no 114 | create mask = 0600 115 | force create mode = 0600 116 | directory mask = 0700 117 | force directory mode = 0700 118 | force group = share 119 | ${vfsString} 120 | 121 | [Public] 122 | path = /ceph/share/public 123 | guest ok = no 124 | writable = yes 125 | printable = no 126 | create mask = 0660 127 | force create mode = 0660 128 | directory mask = 0770 129 | force directory mode = 0770 130 | force group = share 131 | force user = nobody 132 | ${vfsString} 133 | 134 | [Read Only] 135 | path = /ceph/share/ro 136 | guest ok = no 137 | writable = yes 138 | printable = no 139 | create mask = 0640 140 | force create mode = 0640 141 | directory mask = 0750 142 | force directory mode = 0750 143 | force group = share 144 | ${vfsString} 145 | 146 | [pub] 147 | path = /ceph/www-pub 148 | guest ok = yes 149 | guest only = yes 150 | writable = no 151 | printable = no 152 | ${vfsString} 153 | ''; 154 | }; 155 | 156 | systemd.services.samba-smbd.after = [ 157 | "ctdbd.service" 158 | ]; 159 | 160 | systemd.services.samba-nmbd.after = [ 161 | "ctdbd.service" 162 | ]; 163 | } 164 | -------------------------------------------------------------------------------- /common/consul.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | with lib; 4 | let 5 | vars = (import ../customization/vars.nix { inherit lib; }); 6 | calculated = (import ./sub/calculated.nix { inherit config lib; }); 7 | domain = "consul.${vars.domain}"; 8 | 9 | isServer = flip any calculated.myConsul.serverIps 10 | (ip: ip == calculated.myInternalIp4); 11 | isAclMaster = vars.consulAclDc == calculated.myDc && isServer; 12 | 13 | certName = "${config.networking.hostName}.${calculated.myDc}.${domain}"; 14 | in 15 | { 16 | myDns.forwardZones."${domain}" = [ 17 | { 18 | server = "127.0.0.1"; 19 | port = 8600; 20 | } 21 | ]; 22 | 23 | environment.etc."consul.d/systemd-failed.json".text = builtins.toJSON { 24 | check = { 25 | id = "systemd-failed"; 26 | name = "Systemd Failed Units"; 27 | args = [ (pkgs.writeScript "consul-check-systemd-failed" '' 28 | #! ${pkgs.stdenv.shell} -e 29 | OUT="$(${config.systemd.package}/bin/systemctl --failed)" 30 | echo "$OUT" 31 | if echo "$OUT" | ${pkgs.gnugrep}/bin/grep -q '0 loaded units listed'; then 32 | exit 0 33 | fi 34 | exit 1 # Warning (We don't want services to fail because of this) 35 | '') ]; 36 | interval = "60s"; 37 | }; 38 | }; 39 | 40 | environment.etc."consul.d/systemd-starting.json".text = builtins.toJSON { 41 | check = { 42 | id = "systemd-starting"; 43 | name = "Systemd Starting Units"; 44 | args = [ (pkgs.writeScript "consul-check-systemd-starting" '' 45 | #! ${pkgs.stdenv.shell} -e 46 | touch /dev/shm/systemd-starting-jobs 47 | PREVIOUS="$(cat /dev/shm/systemd-starting-jobs)" 48 | 49 | OUT="$(${config.systemd.package}/bin/systemctl list-jobs)" 50 | PARSED="$(echo "$OUT" | tail -n +2 | head -n -2)" 51 | echo "$PARSED" | tee /dev/shm/systemd-starting-jobs 52 | 53 | if [ -z "$PARSED" ] || [ "$PARSED" != "$PREVIOUS" ]; then 54 | exit 0 55 | fi 56 | exit 1 # Warning (We don't want services to fail because of this) 57 | '') ]; 58 | interval = "120s"; 59 | }; 60 | }; 61 | 62 | networking.firewall = { 63 | extraCommands = '' 64 | # Allow consul to communicate with other consuls 65 | ip46tables -A INPUT -p tcp --dport 8300:8302 -j ACCEPT 66 | ip46tables -A INPUT -p udp --dport 8301:8302 -j ACCEPT 67 | ip46tables -A OUTPUT -m owner --uid-owner consul -p tcp --dport 8300:8302 -j ACCEPT 68 | ip46tables -A OUTPUT -m owner --uid-owner consul -p udp --dport 8301:8302 -j ACCEPT 69 | 70 | # Allow consul to interact with itself 71 | ip46tables -A OUTPUT -m owner --uid-owner consul -o lo -p udp --dport 8600 -j ACCEPT 72 | ip46tables -A OUTPUT -m owner --uid-owner consul -o lo -p tcp --dport 8600 -j ACCEPT 73 | ip46tables -A OUTPUT -m owner --uid-owner consul -o lo -p tcp --dport 8400 -j ACCEPT 74 | '' + optionalString (config.services.dnsmasq.enable) '' 75 | # Allow dnsmasq to query consul 76 | ip46tables -A OUTPUT -m owner --uid-owner dnsmasq -o lo -p udp --dport 8600 -j ACCEPT 77 | ip46tables -A OUTPUT -m owner --uid-owner dnsmasq -o lo -p tcp --dport 8600 -j ACCEPT 78 | '' + optionalString (config.services.unbound.enable) '' 79 | # Allow unbound to query consul 80 | ip46tables -A OUTPUT -m owner --uid-owner unbound -o lo -p udp --dport 8600 -j ACCEPT 81 | ip46tables -A OUTPUT -m owner --uid-owner unbound -o lo -p tcp --dport 8600 -j ACCEPT 82 | ''; 83 | }; 84 | services.consul = { 85 | enable = true; 86 | extraConfig = { 87 | acl_datacenter = vars.consulAclDc; 88 | acl_default_policy = "deny"; 89 | acl_down_policy = "deny"; 90 | acl_token = "anonymous"; 91 | advertise_addr = calculated.myInternalIp4; 92 | bind_addr = calculated.myInternalIp4; 93 | ca_file = "/conf/consul/ca.crt"; 94 | cert_file = "/conf/consul/${certName}.crt"; 95 | datacenter = calculated.myDc; 96 | disable_remote_exec = true; 97 | enable_script_checks = true; 98 | domain = "${domain}"; 99 | key_file = "/conf/consul/${certName}.key"; 100 | retry_join = flip filter calculated.myConsul.serverIps 101 | (ip: ip != calculated.myInternalIp4); 102 | server = isServer; 103 | tls_min_version = "tls12"; 104 | verify_incoming = true; 105 | verify_outgoing = true; 106 | verify_server_hostname = true; 107 | } // (if ! isServer then { } else { 108 | bootstrap_expect = length calculated.myConsul.serverIps; 109 | }); 110 | dropPrivileges = true; 111 | extraConfigFiles = [ ] 112 | ++ optional isAclMaster "/conf/consul/acl-master-1.json"; 113 | forceIpv4 = true; 114 | }; 115 | 116 | systemd.services.consul = { 117 | path = [ pkgs.acl ]; 118 | preStart = '' 119 | setfacl -m u:consul:r /conf/consul/${certName}.key 120 | '' + optionalString isAclMaster '' 121 | setfacl -m u:consul:r /conf/consul/acl-master-1.json 122 | ''; 123 | }; 124 | } 125 | -------------------------------------------------------------------------------- /core/ceph.osd.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, lib, utils, ... }: 2 | let 3 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 4 | stateDir = n: "/var/lib/ceph/osd/ceph-${toString n}"; 5 | 6 | osdScript = pkgs.writeScript "osd-script" '' 7 | #! ${pkgs.stdenv.shell} -e 8 | export PATH="${pkgs.gnugrep}/bin:${pkgs.gnused}/bin:${pkgs.coreutils}/bin:${pkgs.util-linux_full}/bin" 9 | mkdir -p /var/lib/ceph 10 | exec 3>/var/lib/ceph/osds.lock 11 | flock 3 || exit 1 12 | env -0 | grep -zv '^PATH' | sed -z "s,.*,export '\0'; ," | tr '\0' ' ' >>/var/lib/ceph/osds 13 | echo -en "\0" >>/var/lib/ceph/osds 14 | ''; 15 | 16 | in 17 | with lib; 18 | { 19 | imports = [ 20 | ../common/ceph.nix 21 | ]; 22 | 23 | boot.supportedFilesystems = [ "btrfs" "zfs" ]; 24 | 25 | services.udev.packages = [ (pkgs.writeTextFile { 26 | name = "osd-udev-rules"; 27 | destination = "/etc/udev/rules.d/91-osd.rules"; 28 | text = '' 29 | ENV{ID_PART_ENTRY_TYPE}=="6a8d2ac7-1dd2-11b2-99a6-080020736631", RUN+="${osdScript}" 30 | ''; 31 | })]; 32 | 33 | systemd.services."ceph-osd-loader" = { 34 | wantedBy = [ "multi-user.target" ]; 35 | after = [ "network.target" "local-fs.target" ]; 36 | unitConfig.RequiresMountsFor = "/var/lib/ceph/osd/by-user"; 37 | path = with pkgs; [ coreutils gawk gnused gnugrep e2fsprogs util-linux_full config.systemd.package btrfs-progs zfs inotify-tools ]; 38 | script = '' 39 | lock () { 40 | exec 3>/var/lib/ceph/osds.lock 41 | flock 3 42 | } 43 | unlock () { 44 | EXTRA_EVENTS=$((EXTRA_EVENTS + 1)) 45 | exec 3>&- 46 | } 47 | process_event () { 48 | if [ "$ACTION" = "remove" ]; then 49 | if [ "$ID_FS_TYPE" = "zfs_member" ]; then 50 | DIR="$(cat /proc/mounts | grep "^$ID_FS_LABEL" | awk '{print $2}')" 51 | elif [ "$ID_FS_TYPE" = "btrfs" ]; then 52 | DIR="$(cat /proc/mounts | grep "^$DEVNAME" | awk '{print $2}')" 53 | fi 54 | [ -z "$DIR" ] && return 0 55 | NUM="$(echo "$DIR" | sed 's,.*ceph-osd\(.*\),\1,g')" 56 | systemctl stop -s 9 "ceph-osd@$NUM" || true 57 | umount -f "$DIR" || true 58 | return 0 59 | fi 60 | 61 | # Check for already existing 62 | if cat /proc/mounts | awk '{print $1}' | grep -q "\($DEVNAME\|$ID_FS_LABEL\)"; then 63 | echo "$DEVNAME is already mounted" 64 | return 0 65 | fi 66 | 67 | # Find a free user / mountpoint 68 | ALL="$(seq 0 47 | awk '{print "ceph-osd"$0}')" 69 | ALLOC="$(cat /proc/mounts | awk '{print $2}' | grep '^/var/lib/ceph/osd/by-user' | xargs basename -a 2>/dev/null)" || true 70 | UNUSED="$(echo -e "$ALL\n$ALLOC" | sed '/^$/d' | sort -V | uniq -u)" 71 | 72 | SELECTED="$(echo "$UNUSED" | head -n 1)" 73 | DIR="/var/lib/ceph/osd/by-user/$SELECTED" 74 | NUM="$(echo "$SELECTED" | sed 's,ceph-osd\(.*\),\1,')" 75 | 76 | systemctl stop "ceph-osd@$NUM" || true 77 | 78 | # Mount the partition 79 | mkdir -p "$DIR" 80 | if [ "$ID_FS_TYPE" = "zfs_member" ]; then 81 | umount "$DIR" || true 82 | zpool export "$ID_FS_LABEL" || true 83 | zpool import -f "$ID_FS_UUID" || return 0 84 | mount -t zfs "$ID_FS_LABEL" "$DIR" || return 0 85 | elif [ "$ID_FS_TYPE" = "btrfs" ]; then 86 | umount "$DIR" || true 87 | mount -t btrfs -o defaults,noatime,user_subvol_rm_allowed,compress=lzo,space_cache "UUID=$ID_FS_UUID" "$DIR" || return 0 88 | else 89 | echo "Failed to determine the partition type on $DEVNAME" >&2 90 | return 0 91 | fi 92 | 93 | # Correctly own the osd 94 | chown "$SELECTED:ceph" "$DIR" "$DIR"/* 95 | chmod 0700 "$DIR" 96 | 97 | # Start the daemon 98 | systemctl start --no-block "ceph-osd@$NUM" 99 | } 100 | 101 | EXTRA_EVENTS=0 102 | touch /var/lib/ceph/osds.lock 103 | inotifywait -m /var/lib/ceph/osds.lock -e CLOSE_WRITE 2>&1 | while read e; do 104 | if [ "$EXTRA_EVENTS" -gt "0" ]; then 105 | EXTRA_EVENTS=$(($EXTRA_EVENTS - 1)) 106 | continue 107 | fi 108 | lock 109 | if [ -f /var/lib/ceph/osds ]; then 110 | cat /var/lib/ceph/osds | while read -d ''$'\0' E; do 111 | (eval "$E"; process_event) 112 | done 113 | truncate -s 0 /var/lib/ceph/osds 114 | fi 115 | unlock 116 | done 117 | ''; 118 | }; 119 | 120 | systemd.services."ceph-osd@" = { 121 | after = [ "network.target" ]; 122 | 123 | restartTriggers = [ config.environment.etc."ceph/ceph.conf".source ]; 124 | 125 | path = [ config.cephPackage ]; 126 | 127 | serviceConfig = { 128 | Type = "simple"; 129 | User = "ceph-osd%i"; 130 | Group = "ceph"; 131 | PermissionsStartOnly = true; 132 | Restart = "always"; 133 | UMask = "000"; 134 | }; 135 | 136 | preStart = '' 137 | mkdir -p -m 0775 /var/run/ceph 138 | chown ceph-mon:ceph /var/run/ceph 139 | mkdir -p -m 0770 /var/log/ceph 140 | chown ceph-mon:ceph /var/log/ceph 141 | mkdir -p /var/lib/ceph/osd 142 | chown ceph-mon:ceph /var/lib/ceph/osd 143 | chmod 0770 /var/lib/ceph/osd 144 | ''; 145 | 146 | script = '' 147 | ID="$(cat "/var/lib/ceph/osd/by-user/$USER/whoami")" 148 | #rm -f "/var/lib/ceph/osd/ceph-osd$ID" 149 | #ln -s "/var/lib/ceph/osd/by-user/$USER" "/var/lib/ceph/osd/ceph-osd$ID" 150 | exec ceph-osd -i "$ID" --osd-data="/var/lib/ceph/osd/by-user/$USER" --osd-journal="/var/lib/ceph/osd/by-user/$USER/journal" -d 151 | ''; 152 | 153 | postStart = '' 154 | ceph osd crush move "osd.$(cat "/var/lib/ceph/osd/by-user/$USER/whoami")" host="${config.networking.hostName}" 155 | ''; 156 | }; 157 | } 158 | -------------------------------------------------------------------------------- /common/wireguard.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | 3 | with lib; 4 | let 5 | vars = (import ../customization/vars.nix { inherit lib; }); 6 | wgConfig = (import ../customization/wireguard.nix { }); 7 | calculated = (import ../common/sub/calculated.nix { inherit config lib; }); 8 | 9 | # Make sure this is only used for wireguard and nothing else 10 | ports = { 11 | "${vars.domain}" = 656; 12 | "gw.${vars.domain}" = 657; 13 | }; 14 | port = name: ports."${name}"; 15 | 16 | # Files needed to build the configuration 17 | secretDir = "/conf/wireguard"; 18 | keyFile = name: "${secretDir}/${name}.key"; 19 | pskFile = name: "${secretDir}/${vars.domain}.psk"; 20 | 21 | haveGatewayInterface = calculated.iAmRemote || calculated.iAmGateway; 22 | 23 | confFileIn = name: let 24 | isGateway = host: "gw" == head (splitString "." host); 25 | hosts = flip filterAttrs wgConfig.hosts (host: { ... }: 26 | if host == config.networking.hostName || host == "gw.${calculated.myDc}" then 27 | false 28 | else if isGateway name then 29 | calculated.isRemote host || isGateway host 30 | else 31 | ! isGateway host 32 | ); 33 | in pkgs.writeText "wg.${name}.conf.in" ('' 34 | [Interface] 35 | PrivateKey = @KEY@ 36 | ListenPort = ${toString (port name)} 37 | '' + concatStrings (flip mapAttrsToList hosts (host: { publicKey, endpoint ? null }: let 38 | netMap = vars.netMaps."${elemAt (splitString "." host) 1}"; 39 | sendKeepalive = endpoint' != null && isGateway host && (calculated.iAmRemote || 40 | !(calculated.myNetMap ? pub4) 41 | ); 42 | vlans = vars.netMaps."${calculated.dc host}".internalMachineMap."${host}".vlans; 43 | vlans' = listToAttrs (map (vlan: nameValuePair (vlan) (true)) vlans); 44 | matchingVlans = filter (vlan: vlans' ? "${vlan}") calculated.myNetData.vlans; 45 | matchingVlan = if calculated.iAmRemote || matchingVlans == [ ] then head vlans else head matchingVlans; 46 | endpoint' = 47 | if endpoint != null then 48 | endpoint 49 | else if ! isGateway name then 50 | if calculated.isRemote host then 51 | "${vars.vpn.remote4}${toString vars.vpn.idMap."${host}"}:${toString (port name)}" 52 | else 53 | "${calculated.internalIp4 host matchingVlan}:${toString (port name)}" 54 | else 55 | null; 56 | in '' 57 | 58 | [Peer] 59 | PublicKey = ${publicKey} 60 | PresharedKey = @PSK@ 61 | '' + optionalString (!isGateway name) '' 62 | AllowedIPs = ${calculated.vpnIp4 host}/32 63 | AllowedIPs = ${calculated.vpnIp6 host}/128 64 | '' + optionalString (isGateway name && !isGateway host) '' 65 | AllowedIPs = ${calculated.vpnGwIp4 host}/32 66 | AllowedIPs = ${calculated.vpnGwIp6 host}/128 67 | '' + optionalString (isGateway name && isGateway host) '' 68 | AllowedIPs = ${netMap.priv4}0.0/16 69 | '' + optionalString sendKeepalive '' 70 | PersistentKeepalive = 20 71 | '' + optionalString (endpoint' != null) '' 72 | Endpoint = ${endpoint'} 73 | ''))); 74 | 75 | confFile = name: "/dev/shm/wg/${name}.conf"; 76 | 77 | wgBuilder = name: pkgs.writeScript "wg.${name}.conf-builder" '' 78 | #! ${pkgs.stdenv.shell} -e 79 | 80 | cleanup() { 81 | ret=$? 82 | rm -f "$TMP" 83 | trap - EXIT 84 | exit $ret 85 | } 86 | trap cleanup EXIT ERR INT QUIT PIPE TERM 87 | TMP="$(mktemp -p "/dev/shm")" 88 | chmod 0600 "$TMP" 89 | cat "${confFileIn name}" >"$TMP" 90 | 91 | if ! test -e "${keyFile name}" || ! wg pubkey <"${keyFile name}" >/dev/null 2>&1; then 92 | exit 2 93 | fi 94 | export KEY="$(cat "${keyFile name}")" 95 | awk -i inplace ' 96 | { 97 | gsub(/@KEY@/, ENVIRON["KEY"]); 98 | print; 99 | } 100 | ' "$TMP" 101 | 102 | if test -e "${pskFile name}"; then 103 | export PSK="$(cat "${pskFile name}")" 104 | awk -i inplace ' 105 | { 106 | gsub(/@PSK@/, ENVIRON["PSK"]); 107 | print; 108 | } 109 | ' "$TMP" 110 | else 111 | sed -i '/@PSK@/d' "$TMP" 112 | fi 113 | 114 | mkdir -p "$(dirname "${confFile name}")" 115 | chown root:root "$(dirname "${confFile name}")" 116 | chmod 0700 "$(dirname "${confFile name}")" 117 | mv "$TMP" "${confFile name}" 118 | ''; 119 | 120 | interfaceConfig = name: nameValuePair "${name}.vpn" { 121 | port = port name; 122 | configFile = confFile name; 123 | }; 124 | 125 | confService = name: nameValuePair "build-wg.${name}" { 126 | serviceConfig = { 127 | Type = "oneshot"; 128 | Restart = "no"; 129 | RemainAfterExit = true; 130 | ExecStart = wgBuilder name; 131 | ExecReload = wgBuilder name; 132 | }; 133 | 134 | restartTriggers = [ 135 | (confFileIn name) 136 | ]; 137 | 138 | requiredBy = [ 139 | "wg-config-${name}.vpn.service" 140 | ]; 141 | before = [ 142 | "wg-config-${name}.vpn.service" 143 | ]; 144 | requires = [ 145 | "time-syncd.target" 146 | ]; 147 | after = [ 148 | "time-syncd.target" 149 | ]; 150 | 151 | path = [ 152 | pkgs.coreutils 153 | pkgs.gawk 154 | pkgs.gnused 155 | pkgs.wireguard 156 | ]; 157 | }; 158 | 159 | remoteNets = 160 | if calculated.iAmRemote then 161 | vars.netMaps 162 | else 163 | flip filterAttrs vars.netMaps 164 | (n: { priv4, ... }: priv4 != calculated.myNetMap.priv4); 165 | 166 | haveMultipleGateways = !calculated.iAmRemote && length calculated.myNetMap.gateways >= 2; 167 | 168 | extraRoutes = mapAttrsToList (n: { priv4, ... }: "${priv4}0.0/16") remoteNets; 169 | 170 | myId = vars.vpn.idMap.${config.networking.hostName}; 171 | in 172 | { 173 | myNatIfs = mkIf calculated.iAmGateway [ 174 | "gw.${vars.domain}.vpn" 175 | ]; 176 | 177 | networking = { 178 | interfaces = mkMerge [ 179 | ({ 180 | "${vars.domain}.vpn" = { 181 | ip4 = optionals (vars.vpn ? subnet4) [ 182 | { address = "${vars.vpn.subnet4}${toString myId}"; prefixLength = 24; } 183 | ]; 184 | ip6 = optionals (vars.vpn ? subnet6) [ 185 | { address = "${vars.vpn.subnet6}${toString myId}"; prefixLength = 64; } 186 | ]; 187 | }; 188 | }) 189 | (mkIf haveGatewayInterface { 190 | "gw.${vars.domain}.vpn" = if ! calculated.iAmRemote then { } else { 191 | ip4 = optionals (vars.vpn ? remote4) [ 192 | { address = "${vars.vpn.remote4}${toString myId}"; prefixLength = 24; } 193 | ]; 194 | ip6 = optionals (vars.vpn ? remote6) [ 195 | { address = "${vars.vpn.remote6}${toString myId}"; prefixLength = 64; } 196 | ]; 197 | }; 198 | }) 199 | ]; 200 | }; 201 | 202 | networking.wgs = listToAttrs ([ 203 | (interfaceConfig vars.domain) 204 | ] ++ optionals haveGatewayInterface [ 205 | (interfaceConfig "gw.${vars.domain}") 206 | ]); 207 | 208 | networking.firewall.extraCommands = flip concatMapStrings (attrValues ports) (port: '' 209 | ip46tables -A INPUT -p udp --dport ${toString port} -j ACCEPT 210 | ip46tables -A OUTPUT -p udp --dport ${toString port} -j ACCEPT 211 | ''); 212 | 213 | services.keepalived.syncGroups.gateway = mkIf haveMultipleGateways { 214 | notifyMaster = flip concatMapStrings extraRoutes (n: '' 215 | ip route del "${n}" || true 216 | ip route add "${n}" dev "gw.${vars.domain}.vpn" src "${calculated.myInternalIp4}" 217 | '') + '' 218 | ip route del "${vars.vpn.remote4}0/24" || true 219 | ip route add "${vars.vpn.remote4}0/24" dev "gw.${vars.domain}.vpn" \ 220 | src "${calculated.myInternalIp4}" 221 | ''; 222 | notifyBackup = flip concatMapStrings extraRoutes (n: '' 223 | ip route del "${n}" || true 224 | ip route add "${n}" via "${calculated.myGatewayIp4}" \ 225 | src "${calculated.myInternalIp4}" 226 | '') + '' 227 | ip route del "${vars.vpn.remote4}0/24" || true 228 | ip route add "${vars.vpn.remote4}0/24" via "${calculated.myGatewayIp4}" \ 229 | src "${calculated.myInternalIp4}" 230 | ''; 231 | notifyFault = flip concatMapStrings extraRoutes (n: '' 232 | ip route del "${n}" || true 233 | ip route add "${n}" via "${calculated.myGatewayIp4}" 234 | '') + '' 235 | ip route del "${vars.vpn.remote4}0/24" || true 236 | ip route add "${vars.vpn.remote4}0/24" via "${calculated.myGatewayIp4}" \ 237 | src "${calculated.myInternalIp4}" 238 | ''; 239 | notifyStop = flip concatMapStrings extraRoutes (n: '' 240 | ip route del "${n}" || true 241 | ip route add "${n}" via "${calculated.myGatewayIp4}" 242 | '') + '' 243 | ip route del "${vars.vpn.remote4}0/24" || true 244 | ip route add "${vars.vpn.remote4}0/24" via "${calculated.myGatewayIp4}" \ 245 | src "${calculated.myInternalIp4}" 246 | ''; 247 | }; 248 | 249 | systemd.services = mkMerge [ 250 | (listToAttrs ([ 251 | (confService vars.domain) 252 | ] ++ optionals haveGatewayInterface [ 253 | (confService "gw.${vars.domain}") 254 | ])) 255 | (mkIf (haveGatewayInterface && !haveMultipleGateways) { 256 | "network-link-up-gw.${vars.domain}.vpn" = { 257 | postStart = flip concatMapStrings extraRoutes (n: '' 258 | ip route add "${n}" dev "gw.${vars.domain}.vpn" \ 259 | ${optionalString calculated.iAmGateway "src ${calculated.myInternalIp4}"} 260 | ''); 261 | }; 262 | }) 263 | (mkIf (calculated.iAmGateway && !haveMultipleGateways) { 264 | "network-link-up-gw.${vars.domain}.vpn" = let 265 | dependency = "network-addresses-${head calculated.myNetData.vlans}.service"; 266 | in { 267 | requires = [ dependency ]; 268 | after = [ dependency ]; 269 | postStart = '' 270 | ip route add "${vars.vpn.remote4}0/24" dev "gw.${vars.domain}.vpn" \ 271 | src "${calculated.myInternalIp4}" 272 | ''; 273 | }; 274 | }) 275 | ]; 276 | } 277 | -------------------------------------------------------------------------------- /common/sub/base-keepalived-module.nix: -------------------------------------------------------------------------------- 1 | { config, lib, pkgs, ... }: 2 | with lib; 3 | let 4 | 5 | cfg = config.services.keepalived; 6 | 7 | vrrpInstance = name: config: '' 8 | vrrp_instance ${name} { 9 | state ${config.state} 10 | ${optionalString (!config.preempt) "nopreempt"} 11 | ${optionalString (config.preemptDelay != null) "preempt_delay ${toString config.preemptDelay}"} 12 | interface ${config.interface} 13 | ${optionalString (config.trackInterfaces != [ ]) '' 14 | track_interface { 15 | ${concatMapStrings (i: " ${i}\n") config.trackInterfaces} 16 | } 17 | ''} 18 | virtual_router_id ${toString config.virtualRouterId} 19 | priority ${toString config.priority} 20 | advert_int ${toString config.advertInt} 21 | garp_master_delay ${toString config.garpMasterDelay} 22 | authentication { 23 | auth_type ${config.authType} 24 | auth_pass ${config.authPass} 25 | } 26 | ${optionalString (config.virtualIpAddresses != [ ]) '' 27 | virtual_ipaddress { 28 | ${flip concatMapStrings config.virtualIpAddresses (i: 29 | " ${i.ip}" + 30 | (optionalString (i.broadcast != null) " brd ${i.broadcast}") + 31 | (optionalString (i.device != null) " dev ${i.device}") + 32 | (optionalString (i.scope != null) " scope ${i.scope}") + 33 | (optionalString (i.label != null) " label ${i.label}") + 34 | "\n" 35 | )} 36 | } 37 | ''} 38 | } 39 | ''; 40 | 41 | notifyScript = lines: pkgs.writeScript "notify-script" '' 42 | #! ${pkgs.stdenv.shell} 43 | set -e 44 | set -o pipefail 45 | 46 | ${lines} 47 | ''; 48 | 49 | vrrpSyncGroup = name: config: '' 50 | vrrp_sync_group ${name} { 51 | group { 52 | ${concatMapStrings (n: " ${n}\n") config.group} 53 | } 54 | ${optionalString (config.notifyMaster != null) "notify_master \"${notifyScript config.notifyMaster}\""} 55 | ${optionalString (config.notifyBackup != null) "notify_backup \"${notifyScript config.notifyBackup}\""} 56 | ${optionalString (config.notifyFault != null) "notify_fault \"${notifyScript config.notifyFault}\""} 57 | ${optionalString (config.notifyStop != null) "notify_stop \"${notifyScript config.notifyStop}\""} 58 | } 59 | ''; 60 | 61 | configFile = pkgs.writeText "keepalived.conf" '' 62 | global_defs { 63 | router_id ${cfg.routerId} 64 | script_user root 65 | enable_script_security 66 | } 67 | ${concatStrings (mapAttrsToList vrrpSyncGroup cfg.syncGroups)} 68 | ${concatStrings (mapAttrsToList vrrpInstance cfg.instances)} 69 | ''; 70 | 71 | interfaces = (attrNames (fold (n: m: m // { ${n.interface} = null; }) {} (attrValues cfg.instances))); 72 | 73 | ips = concatLists (flip mapAttrsToList cfg.instances (n: d: 74 | map (v: { inherit (v) ip; device = if v.device == null then d.interface else v.device; }) d.virtualIpAddresses)); 75 | in 76 | { 77 | options = { 78 | 79 | services.keepalived = { 80 | 81 | enable = mkOption { 82 | type = types.bool; 83 | default = false; 84 | description = '' 85 | Enable the keepalive daemon. 86 | ''; 87 | }; 88 | 89 | routerId = mkOption { 90 | type = types.str; 91 | default = config.networking.hostName; 92 | description = '' 93 | The label of the node used when sending email. 94 | ''; 95 | }; 96 | 97 | syncGroups = mkOption { 98 | type = types.attrsOf types.optionSet; 99 | default = { }; 100 | description = '' 101 | Defines a vrrp_sync_group block. 102 | ''; 103 | options = { 104 | 105 | group = mkOption { 106 | type = types.listOf types.str; 107 | default = [ ]; 108 | description = '' 109 | Add an entry for each vrrp_instance you want to be a part of the sync group. 110 | ''; 111 | }; 112 | 113 | notifyMaster = mkOption { 114 | type = types.nullOr types.lines; 115 | default = null; 116 | description = '' 117 | Add a notify_master script to run when this node becomes the master. 118 | ''; 119 | }; 120 | 121 | notifyBackup = mkOption { 122 | type = types.nullOr types.lines; 123 | default = null; 124 | description = '' 125 | Add a notify_backup script to run when this node becomes the backup. 126 | ''; 127 | }; 128 | 129 | notifyFault = mkOption { 130 | type = types.nullOr types.lines; 131 | default = null; 132 | description = '' 133 | Add a notify_fault script to run when a node faults. 134 | ''; 135 | }; 136 | 137 | notifyStop = mkOption { 138 | type = types.nullOr types.lines; 139 | default = null; 140 | description = '' 141 | Add a notify_stop script to run when a node faults. 142 | ''; 143 | }; 144 | 145 | }; 146 | }; 147 | 148 | instances = mkOption { 149 | type = types.attrsOf types.optionSet; 150 | default = [ ]; 151 | description = '' 152 | Defines a vrrp_instance block. 153 | ''; 154 | options = { 155 | 156 | state = mkOption { 157 | type = types.addCheck types.str (n: elem n [ "MASTER" "BACKUP" ]); 158 | default = "BACKUP"; 159 | description = '' 160 | The state the node should start in. 161 | ''; 162 | }; 163 | 164 | preempt = mkOption { 165 | type = types.bool; 166 | default = true; 167 | description = '' 168 | Whether the state of a node should be preemptible for master elections. 169 | ''; 170 | }; 171 | 172 | preemptDelay = mkOption { 173 | type = types.nullOr (types.addCheck types.int (n: n > 0)); 174 | default = null; 175 | description = '' 176 | The number of seconds until we decide to preempt the master election when no other nodes are found. 177 | null means not to set the value and use the default. 178 | ''; 179 | }; 180 | 181 | interface = mkOption { 182 | type = types.str; 183 | description = '' 184 | The interface used for sending vrrp packets as well as the default interface assigned to ips. 185 | ''; 186 | }; 187 | 188 | trackInterfaces = mkOption { 189 | type = types.listOf types.str; 190 | default = [ ]; 191 | description = '' 192 | Other interfaces to track to determine the FAULT state of the instance. 193 | Any interfaces in the virtual assignment should be here. 194 | ''; 195 | }; 196 | 197 | virtualRouterId = mkOption { 198 | type = types.addCheck types.int (n: n > 0 && n < 256); 199 | description = '' 200 | A unique identifier for this instance that is consistent among routers 201 | ''; 202 | }; 203 | 204 | priority = mkOption { 205 | type = types.addCheck types.int (n: n > 0 && n < 256); 206 | description = '' 207 | The priority of this node having this instance assigned. 208 | This should be unique between nodes also using this instance. 209 | ''; 210 | }; 211 | 212 | advertInt = mkOption { 213 | type = types.addCheck types.int (n: n > 0); 214 | default = 1; 215 | description = '' 216 | The interval of time between probes for instance advertisement. 217 | ''; 218 | }; 219 | 220 | garpMasterDelay = mkOption { 221 | type = types.addCheck types.int (n: n > 0); 222 | default = 10; 223 | description = '' 224 | The delay for gratuitous ARP after transition to MASTER. 225 | ''; 226 | }; 227 | 228 | authType = mkOption { 229 | type = types.addCheck types.str (n: elem n [ "PASS" "AH" ]); 230 | default = "PASS"; 231 | description = '' 232 | The type of authorization to perform when receiving a vrrp packet. 233 | ''; 234 | }; 235 | 236 | authPass = mkOption { 237 | type = types.str; 238 | description = '' 239 | The password to send for vrrp communication authentication. 240 | NOTE: This is world readable currently as nix has no way to support secrets. 241 | Please avoid using this for security, use it more as a sanity check. 242 | It is advisable to create a secure network for communication. 243 | ''; 244 | }; 245 | 246 | virtualIpAddresses = mkOption { 247 | type = types.listOf types.optionSet; 248 | default = [ ]; 249 | options = { 250 | 251 | ip = mkOption { 252 | type = types.str; 253 | description = '' 254 | The ip address and prefixLength to be assigned to the interface. 255 | The format should look like "/" 256 | ''; 257 | }; 258 | 259 | broadcast = mkOption { 260 | type = types.nullOr types.str; 261 | default = null; 262 | description = '' 263 | The broadcast address to assign. 264 | ''; 265 | }; 266 | 267 | device = mkOption { 268 | type = types.nullOr types.str; 269 | default = null; 270 | description = '' 271 | The device to which the ip should be assigned 272 | ''; 273 | }; 274 | 275 | scope = mkOption { 276 | type = types.nullOr types.str; 277 | default = null; 278 | description = '' 279 | The scope of the ip address, mostly used for ipv6. 280 | ''; 281 | }; 282 | 283 | label = mkOption { 284 | type = types.nullOr types.str; 285 | default = null; 286 | description = '' 287 | The label to apply to the ip address. 288 | ''; 289 | }; 290 | 291 | }; 292 | }; 293 | 294 | }; 295 | }; 296 | 297 | extraConfig = mkOption { 298 | type = types.lines; 299 | default = ""; 300 | description = '' 301 | Extra lines to add to the configuration file. 302 | ''; 303 | }; 304 | 305 | }; 306 | 307 | }; 308 | 309 | config = mkIf cfg.enable { 310 | 311 | assertions = concatLists (flip mapAttrsToList cfg.syncGroups (name: g: 312 | flip map g.group (i: { 313 | assertion = cfg.instances ? "${i}"; 314 | message = "VRRP Sync Group ${name} contains a non-existant instance ${i}"; 315 | }) 316 | )); 317 | 318 | networking.firewall.extraCommands = flip concatMapStrings interfaces (n: '' 319 | # Allow other keepalived's to talk to this one 320 | iptables -A nixos-fw -i ${n} -d 224.0.0.0/8 -j nixos-fw-accept 321 | ip46tables -A nixos-fw -i ${n} -p vrrp -j nixos-fw-accept 322 | ''); 323 | 324 | systemd.services.keepalived = { 325 | wantedBy = [ "multi-user.target" ]; 326 | after = [ "network.target" ]; 327 | 328 | path = [ pkgs.iproute ]; 329 | 330 | preStart = concatStrings (flip mapAttrsToList cfg.syncGroups (_: config: '' 331 | ${optionalString (config.notifyBackup != null) (notifyScript config.notifyBackup)} 332 | '')); 333 | 334 | postStop = flip concatMapStrings ips ({ ip, device }: '' 335 | if ip addr show dev "${device}" | grep -q "${ip}"; then 336 | echo "Have to remove an extra ip from ${device}: ${ip}" 337 | if ! ip addr del "${ip}" dev "${device}"; then 338 | echo "Failed to remove ${ip} from ${device}" 339 | fi 340 | fi 341 | ''); 342 | 343 | serviceConfig.ExecStart = "${pkgs.keepalived}/bin/keepalived -P --release-vips -D -n -f ${configFile}"; 344 | }; 345 | 346 | }; 347 | } 348 | -------------------------------------------------------------------------------- /customization/vars.nix: -------------------------------------------------------------------------------- 1 | { lib, ... }: 2 | with lib; 3 | rec { 4 | gateway = { 5 | dhcpRange = { 6 | lower = 200; 7 | upper = 254; 8 | }; 9 | }; 10 | 11 | internalVlanMap = { 12 | lan = 0; # Must Exist For Compatability 13 | mlan = 1; # Must Exist 14 | slan = 2; # Must Exist 15 | dlan = 3; 16 | ulan = 4; 17 | tlan = 5; 18 | }; 19 | 20 | portMap = { 21 | }; 22 | 23 | vpn = { 24 | # Assumes a prefix of /24 25 | subnet4 = "192.168.17."; 26 | remote4 = "192.168.18."; 27 | 28 | # Assumes a prefix of /64 29 | subnet6 = "fdd5:7d0c:d804::"; 30 | remote6 = "fdd8:ce9d:52a5::"; 31 | 32 | idMap = { 33 | #prodigy = 2; 34 | atomic = 3; 35 | legend = 4; 36 | nevada = 5; 37 | atlas = 6; 38 | ferrari = 7; 39 | newton = 8; 40 | page = 9; 41 | quest = 10; 42 | elite = 11; 43 | #lotus = 12; 44 | exodus = 13; 45 | delta = 14; 46 | lake = 15; 47 | jupiter = 16; 48 | }; 49 | }; 50 | 51 | domain = "wak.io"; 52 | 53 | consulAclDc = "fmt-1"; 54 | 55 | userInfo = { 56 | root = { 57 | uid = 0; 58 | description = "root"; 59 | canRoot = true; 60 | loginMachines = [ ]; # This field is irrelevant since root can always login 61 | canShareData = false; 62 | sshKeys = [ 63 | "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDTgCTxC4LUon2NLsLpbIfM68NHO2rGmnRKA+vqE7AXxQ00wryhPJPngQl6c/nwHbm8v9Oi7IPAyuPXAFWoulO6PIxni1RH4k3o040vNIs32mzQY8234GMACYBevaT6nh5qaogDLOaqygLzT7EHdIr1iPS2+iYbN7OhNNpppg5z5ZuWPFXIMWfPMMfqA4R7T2fcSL5TqYHvfAwkh0XDq6knygn/iDIfOa2Qz05ySEB3kK5v4p1iG33vwOTjNrdIf+NCT2BETMaJn1P/pTUUyKTGSInlXz6xkVt+f2Xf/4I01MXcl5i5YF3Vo6UljdQqx57671RznDCcqxo9X++PkawLbZp6sKv5e4uGeS0aGjUbqjnFHK6owsqnDNlPbzzrUDbY7JbpHmNBKpxBcuhJ0+7qNZJwlBcbBaCKABB9KGKWKvIzdnSWXDORx/pM+5JZi6KY7ErmaqESqKSU/p4fX5psxszceV0+3VjttGsSdBnBC/3Grf7t7OYUvRpN6pnn3FQItJY4GzjWkWTwyYN2i24DfrK+qAbNeKjN6p8qUpawCdDcuWCgcuiJRpoa8vm0rjPz4ptlJq0+FG1GhQZnDuWSjEi+zWkfx7BmygK5BaKqQ5DEq/51Y9f7sCf4YzQrg2OqRvmm1l74ZMFEIUOwbKHTsdFCPUt14RcRDb8NgffX4Q== desktop-nano" 64 | "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDYmR1aAHqYm7RXSHZ/RvAsSYqd9f2arExfjaDt1/3vCy3ohdqc1oBNATiz/MindEr+dRgCaJmU09alQ+E2ufYpaAr8VCThupEXvS4YmTXVOdwTEk5Dl40ht1PvMH4s8yxst4K/VFj9fvTjdkpFiG+s/iS/kbpYBDK7/FsVXy8uJ/CSzSLK5UUeKkuVbCrVXFnR3t9hlh9Wslavn2aPH4MrM0pjftiv5/srC7hYwlUA+IDazA9DxSkjHnd44pnev9Ge8ViF79D99BCeuFfuTvN6FnQmr0KJZJxhnoIjqflyzSV3fFFu18z4pnOds7Hs+pTfvzyrDr5+0Bgbb/c6TvBAg3M+N4CquH4FkYg4m1QJcOHt4d6I953DoycGACXPa0he1K6ub0Qt7VAniitdf6kQASSH9OpZQ0QuO3L6xbBFrrA8ZyXkwAR4EiW5A0xejn7vvWefrRUDUP2g8rb1tPsmpIwGtbccMID1cT6ikhfL67FZaZ2oGcChd+3FfFW1PegnErfu04HDOuZ5jx2ByIrlCF0qTdCP9k7zmYJcRTEGGyfC+s37Zd0uRT3aKcLBjT4ZT88o4K/5AAikoz0zZ3TYOxLwgejjWDAEfnpu5lZNzGJR3zaC1PvnJ4H8pQ6JHUdAPBoDAHMwHqpmBFQTL6EanSHLblrGhz4lUUtb+uJncw== keyring" 65 | "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCg+tT2T3ZFcDFjk2QoqypiKRfhdJsG7OjVNyW/OwSZ411VuXYlF+CDLwaLMX1qC3nxM8UYwoiAWDMCTRB767ktzT4br8ZJYTzJ72XClTKZtEHVe2jZpDFIJFik8HEj1g1aTHm/u5h5gDaZAxjcKf9rqe+pB/K2afYZb+s7lzdL0s+lLvbvmPM7pREK8eo/OCbvisBg6u828HLn5tyB8/qPg0DfGQmfVbypwuwujLwhffIBW364qiuZn0nyvqPzaczqahFnkYjbzsC4NPaA/PkuM2uinqYUeW/W0GbeA1DVFcHpMEfBD748HMKAsKZdX5O5DN0bQdSIjJMnKPwNlfg9rnw87Iz5op1prlQwN4XY7OTXYDBq6X4/sh5GVB7QRw9NKclSuYE8EBGhF7a8FNaGaBXaN2B6G8VtLMbboJfQKQi678gwp/4ZL/yS61LHTTdDSAcsQd8WthnFW1GpRQ+HT0uRwY5xJmy/hxVnTzosLpkbr6AcWCw+9uDUaRXobtbqlCPuCzQdvOc7OYecc7xs+LQbbJlA2EhxYxW8Cnt9n2DNH4M/DLDKCP1QgWhenCdaVbBQfqNpx1Z5y6/JUR3A928YVb/qqC738utYLBwgr/K2CdFdQppiu46nXJTEb64pHx4qzrwvlrxQ6u2EGReLi/WQEmmg0RK74ndeMtvhmw== laptop-nano" 66 | ]; 67 | }; 68 | william = { 69 | uid = 1000; 70 | description = "William A. Kennington III"; 71 | canRoot = true; 72 | loginMachines = [ "legend" "lake" ]; 73 | canShareData = true; 74 | inherit (userInfo.root) sshKeys; 75 | }; 76 | bill = { 77 | uid = 1001; 78 | description = "William A. Kennington Jr"; 79 | canRoot = false; 80 | loginMachines = [ ]; 81 | canShareData = true; 82 | sshKeys = [ ]; 83 | }; 84 | linda = { 85 | uid = 1002; 86 | description = "Linda D. Kennington"; 87 | canRoot = false; 88 | loginMachines = [ ]; 89 | canShareData = true; 90 | sshKeys = [ ]; 91 | }; 92 | ryan = { 93 | uid = 1003; 94 | description = "Ryan C. Kennington"; 95 | canRoot = false; 96 | loginMachines = [ ]; 97 | canShareData = true; 98 | sshKeys = [ ]; 99 | }; 100 | sumit = { 101 | uid = 1004; 102 | description = "Sumit R. Punjabi"; 103 | canRoot = false; 104 | loginMachines = [ ]; 105 | canShareData = true; 106 | sshKeys = [ ]; 107 | }; 108 | }; 109 | 110 | remotes = [ 111 | "nixos" 112 | "lake" 113 | ]; 114 | 115 | pubDnsServers = [ 116 | "8.8.8.8" 117 | "8.8.4.4" 118 | ]; 119 | 120 | pubNtpServers = [ 121 | { server = "0.pool.ntp.org"; weight = "1"; } 122 | { server = "1.pool.ntp.org"; weight = "1"; } 123 | { server = "2.pool.ntp.org"; weight = "1"; } 124 | { server = "3.pool.ntp.org"; weight = "1"; } 125 | ]; 126 | 127 | sshHostKeys = { 128 | "atlas" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDEm1t3sNOBfXF7lupeNvZ50M5hT0DYiOiIAx0f+ZLmN"; 129 | "atomic" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGYJVmfOh/DiRlZr3c29QRJY92oBXBRD9H4RLkeEcdQk"; 130 | "elite" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHhgo8hnoGvRJN+kIVBMLc+WheSnRn1MWGXwKVdMnYOn"; 131 | "exodus" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPriEVGsGu7DnUVa/aPFj+BccNP8KUmM9836My9YYemG"; 132 | "ferrari" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF5Q2HxW5RGFr9yi+UrrdfFh5oR7b6DdWbkYdoWoERM8"; 133 | "nevada" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDG8rNK9ZRJl8brPEytBF7sCh2FBejt+V5u3TB3BwiG4"; 134 | "newton" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIALFIS2Hu2bLEHzxcXpNa9JTRBwt/h1S7yjMdHK1FE4f"; 135 | "page" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBv3UkLvr3OcB4fNXJlpNVDnAFgK1Sfgn8wyXoL+EiiS"; 136 | "quest" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKooOFrPAREfP00RT5SM7EI0Y4bOKG07zu+o2vCNEeyJ"; 137 | "legend" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKmQ0vQjgxQ2Id4H6pTPyTbe6HvOfAP7NOabwgv8k0nh"; 138 | "delta" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMXpH6rNL18r6ZRMrCogCTDPr7rsOH4FtK4lzzSs+wPv"; 139 | "jupiter" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINUBBh+gm8di5eJX/vEB6bqa2itoAs4Z+ubyYLFuEHPm"; 140 | "lake" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHQdyqGMXwjnUJEwdeh9Qf1np/T3sjNmcxY7mgXEUvEH"; 141 | }; 142 | 143 | # netMaps currently assumes /16 ipv4 and /60 ipv6 allocations 144 | # ip processing in nix is hard :/ 145 | netMaps = { 146 | "abe-p" = { 147 | priv4 = "10.0."; 148 | priv6 = "fda4:941a:81b5:000"; 149 | 150 | pubDnsServers = [ 151 | "8.8.8.8" 152 | "8.8.4.4" 153 | ]; 154 | 155 | pubNtpServers = [ 156 | { server = "clock.nyc.he.net"; weight = "5"; } 157 | { server = "0.us.pool.ntp.org"; weight = "1"; } 158 | { server = "1.us.pool.ntp.org"; weight = "1"; } 159 | { server = "2.us.pool.ntp.org"; weight = "1"; } 160 | ]; 161 | 162 | timeZone = "America/New_York"; 163 | 164 | gateways = [ 165 | "atomic" 166 | ]; 167 | 168 | dhcpServers = [ 169 | "atomic" 170 | ]; 171 | 172 | dnsServers = [ 173 | "atomic" 174 | ]; 175 | 176 | ntpServers = [ 177 | { server = "atomic"; weight = "1"; } 178 | ]; 179 | 180 | consul = { 181 | servers = [ "atomic" ]; 182 | }; 183 | 184 | internalMachineMap = { 185 | atomic = { 186 | id = 30; 187 | vlans = [ "slan" "mlan" "dlan" "ulan" "tlan" ]; 188 | bmcMac = "00:25:90:bd:e4:83"; 189 | }; 190 | elite = { 191 | id = 31; 192 | vlans = [ "slan" "dlan" ]; 193 | bmcMac = "0C:C4:7A:C8:FA:B6"; 194 | }; 195 | }; 196 | 197 | nasIds = [ 8 9 ]; 198 | 199 | nases = [ 200 | "elite" 201 | ]; 202 | }; 203 | 204 | "fmt-1" = { 205 | pub4 = "65.19.134."; 206 | pub4Gateway = "65.19.134.241"; 207 | pub4PrefixLength = 28; 208 | 209 | pub6 = "2001:470:1:572::"; 210 | pub6Gateway = "2001:470:1:572::1"; 211 | pub6PrefixLength = 64; 212 | 213 | priv4 = "10.2."; 214 | priv6 = "fda4:941a:81b5:200"; 215 | 216 | pubDnsServers = [ 217 | "8.8.8.8" 218 | "8.8.4.4" 219 | ]; 220 | 221 | pubNtpServers = [ 222 | { server = "clock.fmt.he.net"; weight = "5"; } 223 | { server = "clock.sjc.he.net"; weight = "5"; } 224 | { server = "clepsydra.dec.com"; weight = "2"; } 225 | { server = "clock.via.net"; weight = "1"; } 226 | { server = "tock.gpsclock.com"; weight = "1"; } 227 | ]; 228 | 229 | timeZone = "America/Los_Angeles"; 230 | 231 | gateways = [ 232 | "page" 233 | "quest" 234 | ]; 235 | 236 | dhcpServers = [ 237 | "page" 238 | "quest" 239 | ]; 240 | 241 | dnsServers = [ 242 | "newton" 243 | "page" 244 | "quest" 245 | ]; 246 | 247 | ntpServers = [ 248 | { server = "newton"; weight = "1"; } 249 | { server = "page"; weight = "1"; } 250 | { server = "quest"; weight = "1"; } 251 | ]; 252 | 253 | ceph = { 254 | fsId = "82f46d16-7a12-436a-b18a-f612136c4062"; 255 | mons = [ "newton" "page" "quest" ]; 256 | }; 257 | 258 | consul = { 259 | servers = [ "newton" "page" "quest" ]; 260 | }; 261 | 262 | pub6MachineMap = { 263 | outbound = 2; 264 | newton = 3; 265 | page = 4; 266 | quest = 5; 267 | lb1 = 6; 268 | lb2 = 7; 269 | lb3 = 8; 270 | }; 271 | 272 | pub4MachineMap = { 273 | newton = 242; 274 | page = 243; 275 | quest = 244; 276 | lb1 = 245; 277 | lb2 = 246; 278 | lb3 = 247; 279 | outbound = 254; 280 | }; 281 | 282 | loadBalancerMap = { 283 | lb1 = "newton"; 284 | lb2 = "page"; 285 | lb3 = "quest"; 286 | }; 287 | 288 | vrrpMap = { 289 | lan-4 = 1; 290 | lan-6 = 2; 291 | mlan-4 = 3; 292 | mlan-6 = 4; 293 | slan-4 = 5; 294 | slan-6 = 6; 295 | dlan-4 = 7; 296 | dlan-6 = 8; 297 | ulan-4 = 9; 298 | ulan-6 = 10; 299 | tlan-4 = 11; 300 | tlan-6 = 12; 301 | lb1-4 = 20; 302 | lb1-6 = 21; 303 | lb2-4 = 22; 304 | lb2-6 = 23; 305 | lb3-4 = 24; 306 | lb3-6 = 25; 307 | wan-4 = 254; 308 | wan-6 = 255; 309 | }; 310 | 311 | internalMachineMap = { 312 | sw1g1 = { 313 | id = 11; 314 | vlans = [ "mlan" ]; 315 | bmcMac = "00:12:83:36:DC:00"; 316 | }; 317 | sw10g1 = { 318 | id = 21; 319 | vlans = [ "mlan" ]; 320 | bmcMac = "84:C7:27:80:2B:EB"; 321 | }; 322 | newton = { 323 | id = 31; 324 | vlans = [ "slan" "mlan" "tlan" ]; 325 | bmcMac = "BC:5F:F4:FE:7C:E1"; 326 | }; 327 | page = { 328 | id = 32; 329 | vlans = [ "slan" "mlan" "tlan" ]; 330 | bmcMac = "BC:5F:F4:FE:7D:6D"; 331 | }; 332 | quest = { 333 | id = 33; 334 | vlans = [ "slan" "mlan" "tlan" ]; 335 | bmcMac = "BC:5F:F4:FE:7C:FF"; 336 | }; 337 | delta = { 338 | id = 34; 339 | vlans = [ "slan" ]; 340 | bmcMac = "00:25:90:7C:A1:AE"; 341 | }; 342 | ferrari = { 343 | id = 35; 344 | vlans = [ "slan" ]; 345 | bmcMac = "00:25:90:9d:df:e6"; 346 | }; 347 | atlas = { 348 | id = 36; 349 | vlans = [ "slan" ]; 350 | bmcMac = "0C:C4:7A:AE:6D:80"; 351 | }; 352 | nevada = { 353 | id = 37; 354 | vlans = [ "slan" ]; 355 | bmcMac = "00:C0:A8:12:34:56"; 356 | }; 357 | athena = { 358 | id = 38; 359 | vlans = [ "slan" ]; 360 | bmcMac = "BC:5F:F4:C9:A0:70"; 361 | }; 362 | }; 363 | }; 364 | 365 | "nyc-1" = { 366 | pub4 = "192.81.218."; 367 | pub4Gateway = "192.81.218.1"; 368 | pub4PrefixLength = 24; 369 | 370 | pub6 = "2604:a880:400:d0::"; 371 | pub6Gateway = "2604:a880:400:d0::1"; 372 | pub6PrefixLength = 64; 373 | 374 | priv4 = "10.100."; 375 | priv6 = "fd32:d318:c6fa:e153"; 376 | 377 | pubDnsServers = [ 378 | "8.8.8.8" 379 | "8.8.4.4" 380 | ]; 381 | 382 | pubNtpServers = [ 383 | { server = "clock.nyc.he.net"; weight = "5"; } 384 | { server = "clock.fmt.he.net"; weight = "2"; } 385 | { server = "clock.sjc.he.net"; weight = "2"; } 386 | { server = "0.us.pool.ntp.org"; weight = "1"; } 387 | { server = "1.us.pool.ntp.org"; weight = "1"; } 388 | { server = "2.us.pool.ntp.org"; weight = "1"; } 389 | ]; 390 | 391 | timeZone = "America/New_York"; 392 | 393 | gateways = [ 394 | "jupiter" 395 | ]; 396 | 397 | dhcpServers = [ 398 | "jupiter" 399 | ]; 400 | 401 | dnsServers = [ 402 | "jupiter" 403 | ]; 404 | 405 | ntpServers = [ 406 | { server = "jupiter"; weight = "1"; } 407 | ]; 408 | 409 | pub6MachineMap = { 410 | jupiter = "828:6001"; 411 | }; 412 | 413 | pub4MachineMap = { 414 | jupiter = 29; 415 | }; 416 | 417 | internalMachineMap = { 418 | jupiter = { 419 | id = 31; 420 | vlans = [ "slan" "mlan" "tlan" ]; 421 | }; 422 | }; 423 | }; 424 | 425 | "mtv-w" = { 426 | priv4 = "10.1."; 427 | priv6 = "fda4:941a:81b5:100"; 428 | 429 | pubDnsServers = [ 430 | "8.8.8.8" 431 | "8.8.4.4" 432 | ]; 433 | 434 | pubNtpServers = [ 435 | { server = "clock.fmt.he.net"; weight = "5"; } 436 | { server = "clock.sjc.he.net"; weight = "5"; } 437 | { server = "clepsydra.dec.com"; weight = "2"; } 438 | { server = "clock.via.net"; weight = "1"; } 439 | { server = "tock.gpsclock.com"; weight = "1"; } 440 | ]; 441 | 442 | timeZone = "America/Los_Angeles"; 443 | 444 | gateways = [ 445 | "exodus" 446 | ]; 447 | 448 | dhcpServers = [ 449 | "exodus" 450 | ]; 451 | 452 | dnsServers = [ 453 | "exodus" 454 | ]; 455 | 456 | nases = [ 457 | "exodus" 458 | ]; 459 | 460 | nasIds = [ 461 | 8 462 | 9 463 | ]; 464 | 465 | consul = { 466 | servers = [ "exodus" ]; 467 | }; 468 | 469 | 470 | ntpServers = [ 471 | { server = "exodus"; weight = "1"; } 472 | ]; 473 | 474 | # Cannot use 1 as this is reserved for the default gateway 475 | internalMachineMap = { 476 | exodus = { 477 | id = 30; 478 | vlans = [ "slan" "mlan" "dlan" "ulan" "tlan" ]; 479 | bmcMac = "0c:c4:7a:dd:00:42"; 480 | }; 481 | legend = { 482 | id = 33; 483 | vlans = [ "slan" "mlan" "dlan" ]; 484 | }; 485 | eagle = { 486 | id = 34; 487 | vlans = [ "dlan" ]; 488 | bmcMac = "0c:c4:7a:dd:6e:be"; 489 | }; 490 | }; 491 | }; 492 | 493 | }; 494 | } 495 | --------------------------------------------------------------------------------