├── python ├── README.md ├── nix_cache │ ├── __init__.py │ └── cli.py ├── .pyrightconfig.json ├── nix_csi │ ├── __init__.py │ ├── kubernetes.py │ ├── cli.py │ ├── copytocache.py │ ├── identityservicer.py │ ├── subprocessing.py │ └── service.py ├── pyproject.toml ├── default.nix └── nix_timegc │ └── cli.py ├── keys └── .gitignore ├── pkgs ├── csi-proto-python │ ├── README.md │ ├── pyproject.toml │ └── default.nix ├── python-jsonpath.nix ├── kr8s.nix └── default.nix ├── lib └── default.nix ├── .gitignore ├── deploy ├── .envrc ├── reset ├── kubenix ├── namespace.nix ├── default.nix ├── storageclass.nix ├── csidriver.nix ├── options.nix ├── undeploy.nix ├── ctest.nix ├── rbac.nix ├── config.nix ├── cache.nix └── daemonset.nix ├── flake.nix ├── TODO.md ├── README.md ├── guests ├── nixNG.nix ├── test.nix └── ctest.nix ├── container ├── csi.nix ├── cache.nix └── default.nix ├── demo ├── pod.nix └── nixos.nix ├── flake.lock ├── tmp └── watch.py └── default.nix /python/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /keys/.gitignore: -------------------------------------------------------------------------------- 1 | *.pub 2 | -------------------------------------------------------------------------------- /python/nix_cache/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkgs/csi-proto-python/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/default.nix: -------------------------------------------------------------------------------- 1 | self: lib: { 2 | } 3 | -------------------------------------------------------------------------------- /python/.pyrightconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "pythonVersion": "3.13" 3 | } -------------------------------------------------------------------------------- /python/nix_csi/__init__.py: -------------------------------------------------------------------------------- 1 | from .identityservicer import IdentityServicer 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | result 2 | __pycache__ 3 | image 4 | .aider* 5 | cache-public 6 | cache-secret 7 | id_ed25519 8 | id_ed25519.pub 9 | -------------------------------------------------------------------------------- /deploy: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env fish 2 | 3 | nix run --file . imageToContainerd || return 1 4 | nix run --file . kubenixEval.deploymentScript --argstr local "" -- --yes --prune $argv || return 1 5 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | repoenvpath=$(nix build --impure --no-link --print-out-paths --file ./default.nix repoenv)/bin 4 | PATH_add $repoenvpath 5 | 6 | source_env_if_exists ./.privrc 7 | -------------------------------------------------------------------------------- /reset: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env fish 2 | 3 | kubectl delete statefulset nix-cache --wait 4 | kubectl delete pvc nix-store-nix-cache-0 5 | kubectl delete deployment ctest --wait 6 | kubectl delete daemonset nix-csi-node --wait 7 | sudo rm -rf /var/lib/nix-csi 8 | -------------------------------------------------------------------------------- /kubenix/namespace.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | let 3 | cfg = config.nix-csi; 4 | namespace = cfg.namespace; 5 | in 6 | { 7 | config = lib.mkIf cfg.enable { 8 | kubernetes.resources.none.Namespace.${namespace} = { }; 9 | }; 10 | } 11 | -------------------------------------------------------------------------------- /kubenix/default.nix: -------------------------------------------------------------------------------- 1 | { ... }: 2 | { 3 | imports = [ 4 | ./options.nix 5 | ./namespace.nix 6 | ./daemonset.nix 7 | ./csidriver.nix 8 | ./storageclass.nix 9 | ./config.nix 10 | ./cache.nix 11 | ./rbac.nix 12 | ./ctest.nix 13 | ./undeploy.nix 14 | ]; 15 | } 16 | -------------------------------------------------------------------------------- /kubenix/storageclass.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | let 3 | cfg = config.nix-csi; 4 | in 5 | { 6 | config = lib.mkIf cfg.enable { 7 | kubernetes.resources.none.StorageClass.nix-csi = { 8 | provisioner = "nix.csi.store"; 9 | reclaimPolicy = "Delete"; 10 | volumeBindingMode = "WaitForFirstConsumer"; 11 | allowVolumeExpansion = false; 12 | }; 13 | }; 14 | } 15 | -------------------------------------------------------------------------------- /pkgs/csi-proto-python/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "csi" 7 | version = "0.1.0" 8 | description = "CSI driver spec for Python" 9 | authors = [ 10 | { name="Carl Andersson", email="carl@postspace.net" } 11 | ] 12 | readme = "README.md" 13 | requires-python = ">=3.7" 14 | license = { text = "MIT" } 15 | dependencies = [] 16 | -------------------------------------------------------------------------------- /kubenix/csidriver.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | let 3 | cfg = config.nix-csi; 4 | in 5 | { 6 | config = lib.mkIf cfg.enable { 7 | kubernetes.resources.none.CSIDriver."nix.csi.store" = { 8 | spec = { 9 | attachRequired = false; 10 | podInfoOnMount = false; 11 | volumeLifecycleModes = [ "Ephemeral" ]; 12 | fsGroupPolicy = "File"; 13 | requiresRepublish = false; 14 | storageCapacity = false; 15 | }; 16 | }; 17 | }; 18 | } 19 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | inputs = { 3 | nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; 4 | flake-compatish = { 5 | url = "github:lillecarl/flake-compatish"; 6 | flake = false; 7 | }; 8 | easykubenix = { 9 | url = "github:lillecarl/easykubenix"; 10 | flake = false; 11 | }; 12 | dinix = { 13 | url = "github:lillecarl/dinix"; 14 | flake = false; 15 | }; 16 | nix2container = { 17 | url = "github:nlewo/nix2container"; 18 | flake = false; 19 | }; 20 | }; 21 | outputs = inputs: { }; 22 | } 23 | -------------------------------------------------------------------------------- /python/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "nix-csi" 7 | version = "0.2.2" 8 | description = "CSI driver populating mounts with a /nix view" 9 | authors = [{ name = "lillecarl", email = "github@lillecarl.com" }] 10 | readme = "README.md" 11 | requires-python = ">=3.12" 12 | license = { text = "MIT" } 13 | dependencies = [] 14 | 15 | [project.scripts] 16 | nix-csi = "nix_csi.cli:main" 17 | nix-cache = "nix_cache.cli:main" 18 | nix-timegc = "nix_timegc.cli:main" 19 | 20 | [tool.hatch.build.targets.wheel] 21 | packages = ["nix_csi", "nix_cache", "nix_timegc"] 22 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # TODO 2 | 3 | ## Better SSH key management 4 | * Split server and client configuration 5 | * SSH certificates? 6 | 7 | ## Support Nix signing 8 | - Add support for specifying trusted public keys 9 | * Implement signing all paths in the cache (when?) 10 | 11 | ## Improve GC 12 | * Copy entire CSI stores to cache on an interval (Keep paths alive) 13 | * Rewrite timegc to query dead paths before querying DB for regtime 14 | 15 | ## Building 16 | * Wrap distributed building in a nicer "package" 17 | * Better substitution configuration 18 | * Implement speed factor 19 | 20 | ## Controller 21 | * Rename cache to controller, integrate Kopf for additional future features. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nix-csi 2 | 3 | Mount /nix into Kubernetes pods using the CSI Ephemeral Volume feature. Volumes 4 | share lifetime with Pods and are embedded into the Podspec. 5 | 6 | ## Deploying nix-csi 7 | 8 | Stick your pubkeys in ./keys and they will be imported into the module system 9 | then run the following command and you'll have nix-csi deployed. 10 | ```bash 11 | nix run --file . kubenixEval.deploymentScript -- --yes --prune 12 | ``` 13 | 14 | If you'd rather mangle YAML yourself you can use 15 | ```bash 16 | nix build --file . easykubenix.manifestYAMLFile 17 | ``` 18 | and stuff the result into 19 | 20 | ## Deploying workloads 21 | 22 | TODO, but essentially stick a storePath in volumeAttributes like [this](https://github.com/Lillecarl/hetzkube/blob/4ed76ec77bfb104d1c2307b1ba178efa61dd34e2/kubenix/modules/cheapam.nix#L113) 23 | 24 | -------------------------------------------------------------------------------- /pkgs/python-jsonpath.nix: -------------------------------------------------------------------------------- 1 | { 2 | lib, 3 | buildPythonPackage, 4 | fetchFromGitHub, 5 | # build-system 6 | hatchling, 7 | hatch-vcs, 8 | }: 9 | buildPythonPackage rec { 10 | pname = "python-jsonpath"; 11 | version = "2.0.1"; 12 | pyproject = true; 13 | 14 | src = fetchFromGitHub { 15 | owner = "jg-rp"; 16 | repo = "python-jsonpath"; 17 | tag = "v${version}"; 18 | hash = "sha256-PkoZs6b/dtb9u1308D6LQF6kg39DslJufI/QpKMkZiQ="; 19 | }; 20 | 21 | build-system = [ 22 | hatchling 23 | hatch-vcs 24 | ]; 25 | 26 | dependencies = [ ]; 27 | 28 | meta = with lib; { 29 | description = "A flexible JSONPath engine for Python with JSON Pointer and JSON Patch"; 30 | homepage = "https://github.com/hephex/asyncache"; 31 | license = licenses.mit; 32 | maintainers = with maintainers; [ lillecarl ]; 33 | }; 34 | } 35 | -------------------------------------------------------------------------------- /python/nix_csi/kubernetes.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import kr8s 4 | 5 | 6 | async def get_builder_ips(namespace: str) -> list[str]: 7 | candidate_nodes = [] 8 | nodes = kr8s.asyncio.get("nodes") 9 | # Get all builder tagged nodes 10 | async for node in nodes: 11 | try: 12 | node.metadata["labels"]["nix.csi/builder"] 13 | candidate_nodes.append(node.name) 14 | except KeyError: 15 | pass 16 | 17 | builder_ips = [] 18 | pods = kr8s.asyncio.get( 19 | "pods", namespace=namespace, label_selector={"app": "nix-csi-node"} 20 | ) 21 | async for pod in pods: 22 | try: 23 | nodeName = pod.spec["nodeName"] 24 | if nodeName in candidate_nodes: 25 | builder_ips.append(pod.status["podIP"]) 26 | except KeyError: 27 | pass 28 | 29 | return builder_ips 30 | -------------------------------------------------------------------------------- /python/nix_csi/cli.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | import argparse 4 | from . import service 5 | 6 | 7 | def parse_args(): 8 | parser = argparse.ArgumentParser(description="nix CSI driver") 9 | parser.add_argument( 10 | "--loglevel", 11 | default="INFO", 12 | choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], 13 | help="Set the logging level (default: INFO)", 14 | ) 15 | return parser.parse_args() 16 | 17 | 18 | async def async_main(): 19 | args = parse_args() 20 | logging.basicConfig( 21 | level=logging.WARN, 22 | format="%(asctime)s %(levelname)s [%(name)s] %(message)s", 23 | ) 24 | logger = logging.getLogger("nix-csi") 25 | loglevel_str = logging.getLevelName(logger.getEffectiveLevel()) 26 | logger.info(f"Current log level: {loglevel_str}") 27 | 28 | logging.getLogger("nix-csi").setLevel(getattr(logging, args.loglevel)) 29 | 30 | await service.serve() 31 | 32 | 33 | def main(): 34 | asyncio.run(async_main()) 35 | 36 | 37 | if __name__ == "__main__": 38 | main() 39 | -------------------------------------------------------------------------------- /pkgs/kr8s.nix: -------------------------------------------------------------------------------- 1 | { 2 | lib, 3 | buildPythonPackage, 4 | fetchFromGitHub, 5 | # dependencies 6 | cachetools, 7 | cryptography, 8 | exceptiongroup, 9 | packaging, 10 | pyyaml, 11 | python-jsonpath, 12 | anyio, 13 | httpx, 14 | httpx-ws, 15 | python-box, 16 | # build-system 17 | hatchling, 18 | hatch-vcs, 19 | }: 20 | buildPythonPackage rec { 21 | pname = "kr8s"; 22 | version = "0.20.13"; 23 | pyproject = true; 24 | 25 | src = fetchFromGitHub { 26 | owner = "kr8s-org"; 27 | repo = "kr8s"; 28 | tag = "v${version}"; 29 | hash = "sha256-9fo18ririQwBzxuPp8+oH20URv0nvXCkv0eIUL4xrZ8="; 30 | }; 31 | 32 | build-system = [ 33 | hatchling 34 | hatch-vcs 35 | ]; 36 | 37 | dependencies = [ 38 | cachetools 39 | cryptography 40 | exceptiongroup 41 | packaging 42 | pyyaml 43 | python-jsonpath 44 | anyio 45 | httpx 46 | httpx-ws 47 | python-box 48 | ]; 49 | 50 | pythonImportsCheck = [ "kr8s" ]; 51 | 52 | meta = with lib; { 53 | description = "A Python client library for Kubernetes"; 54 | homepage = "https://github.com/kr8s-org/kr8s"; 55 | license = licenses.mit; 56 | maintainers = with maintainers; [ lillecarl ]; 57 | }; 58 | } 59 | -------------------------------------------------------------------------------- /python/default.nix: -------------------------------------------------------------------------------- 1 | { 2 | lib, # recursive update 3 | buildPythonApplication, # Builder 4 | hatchling, # Build system 5 | coreutils, # ln 6 | csi-proto-python, # CSI GRPC bindings 7 | gitMinimal, # Lix requires Git since it doesn't use libgit2 8 | kr8s, # Kubernetes API 9 | lix, # We need a Nix implementation.... :) 10 | nix_init_db, # Import from one nix DB to another 11 | openssh, # Copying to cache 12 | rsync, # hardlinking 13 | util-linuxMinimal, # mount, umount 14 | }: 15 | let 16 | pyproject = builtins.fromTOML (builtins.readFile ./pyproject.toml); 17 | python = buildPythonApplication { 18 | pname = pyproject.project.name; 19 | version = pyproject.project.version; 20 | src = ./.; 21 | pyproject = true; 22 | build-system = [ hatchling ]; 23 | dependencies = [ 24 | coreutils 25 | csi-proto-python 26 | gitMinimal 27 | kr8s 28 | lix 29 | nix_init_db 30 | openssh 31 | rsync 32 | util-linuxMinimal 33 | ]; 34 | }; 35 | in 36 | { 37 | csi = lib.recursiveUpdate python { meta.mainProgram = "nix-csi"; }; 38 | cache = lib.recursiveUpdate python { meta.mainProgram = "nix-cache"; }; 39 | timegc = lib.recursiveUpdate python { meta.mainProgram = "nix-timegc"; }; 40 | } 41 | -------------------------------------------------------------------------------- /guests/nixNG.nix: -------------------------------------------------------------------------------- 1 | let 2 | flake-compatish = builtins.fetchTree { 3 | type = "github"; 4 | owner = "lillecarl"; 5 | repo = "flake-compatish"; 6 | ref = "main"; 7 | }; 8 | nixpkgs = builtins.fetchTree { 9 | type = "github"; 10 | owner = "NixOS"; 11 | repo = "nixpkgs"; 12 | ref = "nixos-unstable"; 13 | }; 14 | nixng = builtins.fetchTree { 15 | type = "github"; 16 | owner = "nix-community"; 17 | repo = "nixNG"; 18 | ref = "master"; 19 | }; 20 | pkgs = import nixpkgs { }; 21 | flake = (import flake-compatish) nixpkgs; 22 | nglib = import "${nixng}/lib" flake.outputs.lib; 23 | in 24 | { 25 | inherit flake; 26 | nixNG = nglib.makeSystem { 27 | nixpkgs = flake.outputs; 28 | system = builtins.currentSystem; 29 | name = "nixng-nix"; 30 | 31 | config = ( 32 | { pkgs, ... }: 33 | { 34 | dumb-init = { 35 | enable = true; 36 | type.shell = { }; 37 | }; 38 | nix = { 39 | enable = true; 40 | package = pkgs.nixStable; 41 | config = { 42 | experimental-features = [ 43 | "nix-command" 44 | "flakes" 45 | ]; 46 | sandbox = false; 47 | }; 48 | }; 49 | } 50 | ); 51 | }; 52 | } 53 | .nixNG.config.system.build.toplevel 54 | -------------------------------------------------------------------------------- /kubenix/options.nix: -------------------------------------------------------------------------------- 1 | { lib, ... }: 2 | { 3 | options.nix-csi = { 4 | enable = lib.mkEnableOption "nix-csi"; 5 | undeploy = lib.mkOption { 6 | type = lib.types.bool; 7 | default = false; 8 | }; 9 | namespace = lib.mkOption { 10 | description = "Which namespace to deploy cknix resources too"; 11 | type = lib.types.str; 12 | default = "nix-csi"; 13 | }; 14 | authorizedKeys = lib.mkOption { 15 | description = "SSH public keys that can connect to cache and builders"; 16 | type = lib.types.listOf lib.types.str; 17 | default = [ ]; 18 | }; 19 | image = lib.mkOption { 20 | type = lib.types.str; 21 | default = 22 | let 23 | pyproject = builtins.fromTOML (builtins.readFile ../python/pyproject.toml); 24 | version = pyproject.project.version; 25 | image = "quay.io/nix-csi/nix-csi:${version}"; 26 | in 27 | image; 28 | }; 29 | hostMountPath = lib.mkOption { 30 | description = "Where on the host to put cknix store"; 31 | type = lib.types.path; 32 | default = "/var/lib/nix-csi"; 33 | }; 34 | internalServiceName = lib.mkOption { 35 | description = '' 36 | Internal service name used for reaching builder nodes from cache node 37 | ''; 38 | type = lib.types.str; 39 | default = "nix-builders"; 40 | }; 41 | }; 42 | } 43 | -------------------------------------------------------------------------------- /pkgs/default.nix: -------------------------------------------------------------------------------- 1 | self: pkgs: { 2 | # Overlay lib 3 | lib = pkgs.lib.extend (import ../lib); 4 | 5 | # First argument is NIX_STATE_DIR which is where we init the dumped database 6 | nix_init_db = 7 | pkgs.writeScriptBin "nix_init_db" # bash 8 | '' 9 | #! ${pkgs.runtimeShell} 10 | NSD="$1" 11 | shift 12 | export USER nobody 13 | nix-store --option store local --dump-db "$@" | NIX_STATE_DIR="$NSD" nix-store --load-db --option store local 14 | ''; 15 | 16 | nix-csi = self.csi-root.csi; 17 | nix-cache = self.csi-root.cache; 18 | nix-timegc = self.csi-root.timegc; 19 | csi-root = pkgs.python3Packages.callPackage ../python { 20 | inherit (self) csi-proto-python kr8s; 21 | }; 22 | 23 | lix = pkgs.lix.overrideAttrs (oldAttrs: { 24 | patches = (oldAttrs.patches or [ ]) ++ [ 25 | (pkgs.fetchpatch { 26 | url = "https://github.com/Lillecarl/lix/commit/9ac72bbd0c7802ca83a907d1fec135f31aab6d24.patch"; 27 | hash = "sha256-NLyURqjzbyftbjxwOGWW26jcLRtvvE0hdIriiYEnQ4Q="; 28 | }) 29 | ]; 30 | doCheck = false; 31 | doInstallCheck = false; 32 | }); 33 | 34 | csi-proto-python = pkgs.python3Packages.callPackage ./csi-proto-python { }; 35 | python-jsonpath = pkgs.python3Packages.callPackage ./python-jsonpath.nix { }; 36 | kr8s = pkgs.python3Packages.callPackage ./kr8s.nix { inherit (self) python-jsonpath; }; 37 | } 38 | -------------------------------------------------------------------------------- /pkgs/csi-proto-python/default.nix: -------------------------------------------------------------------------------- 1 | # Credits to Claude Sonnet 3.7 2 | { 3 | lib, 4 | buildPythonPackage, 5 | fetchFromGitHub, 6 | grpcio-tools, 7 | grpcio, 8 | grpclib, 9 | protobuf, 10 | mypy-protobuf, 11 | python, 12 | pythonRelaxDepsHook, 13 | }: 14 | let 15 | version = "1.11.0"; 16 | spec = fetchFromGitHub { 17 | owner = "container-storage-interface"; 18 | repo = "spec"; 19 | rev = "v${version}"; 20 | sha256 = "sha256-mDvlHB2vVqJIQO6y2UJlDohzHUbCvzJ9hJc7XFAbFb0="; 21 | }; 22 | in 23 | buildPythonPackage { 24 | inherit version; 25 | pname = "csi-proto-python"; 26 | 27 | src = ./.; 28 | 29 | buildInputs = [ ]; 30 | 31 | nativeBuildInputs = [ 32 | grpclib 33 | mypy-protobuf 34 | grpcio-tools 35 | ]; 36 | 37 | propagatedBuildInputs = [ 38 | grpclib 39 | mypy-protobuf 40 | ]; 41 | 42 | format = "pyproject"; 43 | preBuild = '' 44 | mkdir -p src/csi 45 | protoc \ 46 | --proto_path="${spec}" \ 47 | --python_out="src/csi" \ 48 | --grpclib_python_out="src/csi" \ 49 | --mypy_out="src/csi" \ 50 | csi.proto 51 | 52 | substituteInPlace src/csi/csi_grpc.py \ 53 | --replace-fail "import csi_pb2" "from . import csi_pb2" 54 | ''; 55 | 56 | meta = with lib; { 57 | description = "Python gRPC/protobuf library for Kubernetes CSI spec"; 58 | homepage = "https://github.com/container-storage-interface/spec"; 59 | license = licenses.asl20; 60 | platforms = platforms.all; 61 | }; 62 | } 63 | -------------------------------------------------------------------------------- /kubenix/undeploy.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | let 3 | cfg = config.nix-csi; 4 | in 5 | { 6 | config = lib.mkIf cfg.undeploy { 7 | assertions = [ 8 | { 9 | assertion = !(cfg.enable && cfg.undeploy); 10 | message = "nix-csi.undeploy cannot be true when nix-csi.enable is also true."; 11 | } 12 | { 13 | assertion = !(cfg.undeploy && lib.hasPrefix "/nix" cfg.hostMountPath); 14 | message = "nix-csi.undeploy will not undeploy /nix based mount locations like ${cfg.hostMountPath}"; 15 | } 16 | ]; 17 | 18 | kubernetes.resources.${cfg.namespace}.DaemonSet.nix-csi-cleanup = { 19 | spec.selector.matchLabels.app = "nix-csi-cleanup"; 20 | spec.template = { 21 | metadata.labels.app = "nix-csi-cleanup"; 22 | spec = { 23 | containers = lib.mkNamedList { 24 | cleanup = { 25 | name = "cleanup"; 26 | image = "busybox:latest"; 27 | command = [ 28 | "find" 29 | "/nix" 30 | "-mindepth" 31 | "1" 32 | "-delete" 33 | ]; 34 | 35 | volumeMounts = lib.mkNamedList { 36 | nix-store.mountPath = "/nix"; 37 | }; 38 | securityContext.privileged = true; 39 | }; 40 | }; 41 | 42 | volumes = lib.mkNamedList { 43 | nix-store.hostPath.path = cfg.hostMountPath; 44 | }; 45 | }; 46 | }; 47 | }; 48 | }; 49 | } 50 | -------------------------------------------------------------------------------- /python/nix_csi/copytocache.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from collections import defaultdict 3 | from pathlib import Path 4 | from asyncio import Semaphore, sleep 5 | from .subprocessing import run_captured, run_console 6 | 7 | logger = logging.getLogger("nix-csi") 8 | 9 | # Locks that prevent the same derivation to be uploaded in parallel 10 | copyLock: defaultdict[Path, Semaphore] = defaultdict(Semaphore) 11 | 12 | 13 | async def copyToCache(packagePath: Path): 14 | # Only run one copy per path per time 15 | async with copyLock[packagePath]: 16 | paths = [str(packagePath)] 17 | # Get all paths recursively++ 18 | pathInfoDrv = await run_captured( 19 | "nix", 20 | "path-info", 21 | "--recursive", 22 | "--derivation", 23 | packagePath, 24 | ) 25 | if pathInfoDrv.returncode == 0: 26 | paths += pathInfoDrv.stdout.splitlines() 27 | 28 | # Unique the paths since we're running path-info twice 29 | paths = list(set(paths)) 30 | # Filter derivation files 31 | paths = {p for p in paths if not p.endswith(".drv")} 32 | if len(paths) > 0: 33 | for _ in range(6): 34 | await sleep(5) 35 | nixCopy = await run_captured( 36 | "nix", "copy", "--to", "ssh-ng://nix@nix-cache", *paths 37 | ) 38 | if nixCopy.returncode == 0: 39 | logger.debug(nixCopy.combined) 40 | break 41 | await sleep(5) 42 | -------------------------------------------------------------------------------- /kubenix/ctest.nix: -------------------------------------------------------------------------------- 1 | { 2 | config, 3 | pkgs, 4 | lib, 5 | ... 6 | }: 7 | let 8 | cfg = config.nix-csi; 9 | system = pkgs.stdenv.hostPlatform.system; 10 | in 11 | { 12 | options.nix-csi.ctest = { 13 | enable = lib.mkEnableOption "ctest"; 14 | replicas = lib.mkOption { 15 | type = lib.types.int; 16 | default = 10; 17 | }; 18 | }; 19 | config = lib.mkIf cfg.ctest.enable { 20 | kubernetes.resources.${cfg.namespace}.Deployment.ctest = 21 | let 22 | pkg.${system} = import ../guests/ctest.nix { 23 | inherit pkgs; 24 | }; 25 | in 26 | { 27 | spec = { 28 | replicas = cfg.ctest.replicas; 29 | selector.matchLabels.app = "ctest"; 30 | template = { 31 | metadata.labels.app = "ctest"; 32 | spec = { 33 | containers = [ 34 | { 35 | name = "ctest"; 36 | command = [ pkg.${system}.meta.mainProgram ]; 37 | image = "quay.io/nix-csi/scratch:1.0.1"; 38 | volumeMounts = lib.mkNamedList { 39 | nix-csi.mountPath = "/nix"; 40 | }; 41 | } 42 | ]; 43 | volumes = lib.mkNamedList { 44 | nix-csi.csi = { 45 | driver = "nix.csi.store"; 46 | readOnly = false; 47 | volumeAttributes.${system} = pkg.${system}; 48 | }; 49 | }; 50 | }; 51 | }; 52 | }; 53 | }; 54 | }; 55 | } 56 | -------------------------------------------------------------------------------- /container/csi.nix: -------------------------------------------------------------------------------- 1 | { 2 | config, 3 | pkgs, 4 | lib, 5 | ... 6 | }: 7 | { 8 | config = { 9 | # Umbrella service for CSI 10 | services.csi = { 11 | type = "scripted"; 12 | options = [ "starts-rwfs" ]; 13 | command = 14 | pkgs.writeScriptBin "csi" # bash 15 | '' 16 | #! ${pkgs.runtimeShell} 17 | mkdir --parents /run 18 | mkdir --parents /var/log 19 | ''; 20 | depends-on = [ 21 | "csi-daemon" 22 | "csi-logger" 23 | "openssh" 24 | ]; 25 | }; 26 | services.csi-daemon = { 27 | command = "${lib.getExe pkgs.nix-csi} --loglevel DEBUG"; 28 | log-type = "file"; 29 | logfile = "/var/log/csi-daemon.log"; 30 | depends-on = [ 31 | "shared-setup" 32 | "csi-gc" 33 | "nix-daemon" 34 | ]; 35 | }; 36 | services.csi-logger = { 37 | command = "${lib.getExe' pkgs.coreutils "tail"} --follow /var/log/csi-daemon.log /var/log/dinit.log"; 38 | options = [ "shares-console" ]; 39 | depends-on = [ "csi-daemon" ]; 40 | }; 41 | services.csi-gc = { 42 | type = "scripted"; 43 | command = 44 | pkgs.writeScriptBin "csi-gc" # bash 45 | '' 46 | #! ${pkgs.runtimeShell} 47 | # Fix gcroots for /nix/var/result 48 | nix build --out-link /nix/var/result /nix/var/result 49 | # Collect old shit 50 | ${lib.getExe pkgs.nix-timegc} 3600 51 | ''; 52 | log-type = "file"; 53 | logfile = "/var/log/csi-gc.log"; 54 | depends-on = [ 55 | "nix-daemon" 56 | "shared-setup" 57 | ]; 58 | }; 59 | }; 60 | } 61 | -------------------------------------------------------------------------------- /kubenix/rbac.nix: -------------------------------------------------------------------------------- 1 | { config, lib, ... }: 2 | let 3 | cfg = config.nix-csi; 4 | in 5 | { 6 | config = lib.mkIf cfg.enable { 7 | kubernetes.resources.${cfg.namespace} = { 8 | ServiceAccount.nix-csi = { }; 9 | 10 | ClusterRole.nix-csi = { 11 | rules = [ 12 | # Cache maintains up2date /etc/nix/machines 13 | { 14 | apiGroups = [ "" ]; 15 | resources = [ 16 | "nodes" 17 | "pods" 18 | ]; 19 | verbs = [ 20 | "get" 21 | "list" 22 | "watch" 23 | ]; 24 | } 25 | # ssh secret, CRUD 26 | { 27 | apiGroups = [ "" ]; 28 | resources = [ 29 | "secrets" 30 | ]; 31 | verbs = [ 32 | "get" 33 | "list" 34 | "create" 35 | "patch" 36 | ]; 37 | } 38 | # Read authorized-keys 39 | { 40 | apiGroups = [ "" ]; 41 | resources = [ 42 | "configmaps" 43 | ]; 44 | verbs = [ 45 | "get" 46 | "list" 47 | ]; 48 | } 49 | ]; 50 | }; 51 | 52 | # Binds the Role to the ServiceAccount. 53 | ClusterRoleBinding.nix-csi = { 54 | subjects = lib.mkNamedList { 55 | nix-csi = { 56 | kind = "ServiceAccount"; 57 | namespace = cfg.namespace; 58 | }; 59 | }; 60 | roleRef = { 61 | kind = "ClusterRole"; 62 | name = "nix-csi"; 63 | apiGroup = "rbac.authorization.k8s.io"; 64 | }; 65 | }; 66 | }; 67 | }; 68 | } 69 | -------------------------------------------------------------------------------- /python/nix_csi/identityservicer.py: -------------------------------------------------------------------------------- 1 | from csi import csi_grpc, csi_pb2 2 | from google.protobuf.wrappers_pb2 import BoolValue 3 | from importlib import metadata 4 | 5 | CSI_PLUGIN_NAME = "nix.csi.store" 6 | CSI_VENDOR_VERSION = metadata.version("nix-csi") 7 | 8 | 9 | class IdentityServicer(csi_grpc.IdentityBase): 10 | async def GetPluginInfo(self, stream): 11 | request: csi_pb2.GetPluginInfoRequest | None = await stream.recv_message() 12 | if request is None: 13 | raise ValueError("GetPluginInfoRequest is None") 14 | reply = csi_pb2.GetPluginInfoResponse( 15 | name=CSI_PLUGIN_NAME, vendor_version=CSI_VENDOR_VERSION 16 | ) 17 | await stream.send_message(reply) 18 | 19 | async def GetPluginCapabilities(self, stream): 20 | request: ( 21 | csi_pb2.GetPluginCapabilitiesRequest | None 22 | ) = await stream.recv_message() 23 | if request is None: 24 | raise ValueError("GetPluginCapabilitiesRequest is None") 25 | reply = csi_pb2.GetPluginCapabilitiesResponse( 26 | capabilities=[ 27 | csi_pb2.PluginCapability( 28 | service=csi_pb2.PluginCapability.Service( 29 | type=csi_pb2.PluginCapability.Service.CONTROLLER_SERVICE 30 | ) 31 | ), 32 | ] 33 | ) 34 | await stream.send_message(reply) 35 | 36 | async def Probe(self, stream): 37 | request: csi_pb2.ProbeRequest | None = await stream.recv_message() 38 | if request is None: 39 | raise ValueError("ProbeRequest is None") 40 | reply = csi_pb2.ProbeResponse(ready=BoolValue(value=True)) 41 | await stream.send_message(reply) 42 | -------------------------------------------------------------------------------- /guests/test.nix: -------------------------------------------------------------------------------- 1 | let 2 | nixpkgs = builtins.fetchTree { 3 | type = "github"; 4 | owner = "NixOS"; 5 | repo = "nixpkgs"; 6 | ref = "nixos-unstable"; 7 | }; 8 | dinixSrc = builtins.fetchTree { 9 | type = "github"; 10 | owner = "lillecarl"; 11 | repo = "dinix"; 12 | ref = "main"; 13 | }; 14 | pkgs = import nixpkgs { }; 15 | lib = pkgs.lib; 16 | folderPaths = [ 17 | "/tmp" 18 | "/var/tmp" 19 | "/var/log" 20 | "/var/lib" 21 | "/var/run" 22 | "/etc/ssl/certs" 23 | ]; 24 | dinixEval = ( 25 | import dinixSrc { 26 | inherit pkgs; 27 | modules = [ 28 | { 29 | config = { 30 | services.boot.depends-on = [ "setup" ]; 31 | services.setup = { 32 | type = "scripted"; 33 | command = lib.getExe ( 34 | pkgs.writeScriptBin "init" '' 35 | #! ${lib.getExe' pkgs.execline "execlineb"} 36 | export PATH "/nix/var/result/bin" 37 | foreground { echo "Initializing" } 38 | ${lib.concatMapStringsSep "\n" (folder: "foreground { mkdir --parents ${folder} }") folderPaths} 39 | foreground { ln --symbolic ${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-bundle.crt } 40 | foreground { ln --symbolic ${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt } 41 | '' 42 | ); 43 | }; 44 | }; 45 | } 46 | ]; 47 | } 48 | ); 49 | in 50 | pkgs.buildEnv { 51 | name = "containerEnv"; 52 | paths = with pkgs; [ 53 | curl 54 | coreutils 55 | fishMinimal 56 | execline 57 | lix 58 | gitMinimal 59 | ncdu 60 | dinixEval.config.containerWrapper 61 | ]; 62 | } 63 | -------------------------------------------------------------------------------- /container/cache.nix: -------------------------------------------------------------------------------- 1 | { 2 | config, 3 | pkgs, 4 | lib, 5 | ... 6 | }: 7 | { 8 | config = { 9 | # Umbrella service for cache 10 | services.cache = { 11 | type = "scripted"; 12 | options = [ "starts-rwfs" ]; 13 | command = 14 | pkgs.writeScriptBin "csi" # bash 15 | '' 16 | #! ${pkgs.runtimeShell} 17 | mkdir --parents /run 18 | mkdir --parents /var/log 19 | ''; 20 | depends-on = [ 21 | "cache-daemon" 22 | "cache-logger" 23 | "cache-gc" 24 | "openssh" 25 | ]; 26 | }; 27 | services.cache-daemon = { 28 | command = "${lib.getExe pkgs.nix-cache} --loglevel DEBUG"; 29 | log-type = "file"; 30 | logfile = "/var/log/cache-daemon.log"; 31 | depends-on = [ "shared-setup" ]; 32 | depends-ms = [ "nix-daemon" ]; 33 | }; 34 | services.cache-logger = { 35 | command = "${lib.getExe' pkgs.coreutils "tail"} --follow /var/log/cache-daemon.log /var/log/dinit.log"; 36 | options = [ "shares-console" ]; 37 | depends-on = [ "cache-daemon" ]; 38 | }; 39 | # Make OpenSSH depend on cache-daemon so it can create the secret that'll 40 | # be mounted into /etc/ssh-mount when it's crashed once. 41 | # services.openssh.waits-for = [ "cache-daemon" ]; 42 | services.cache-gc = { 43 | type = "scripted"; 44 | command = 45 | pkgs.writeScriptBin "cache-gc" # bash 46 | '' 47 | #! ${pkgs.runtimeShell} 48 | # Fix gcroots for /nix/var/result 49 | nix build --out-link /nix/var/result /nix/var/result 50 | # Collect old shit 51 | ${lib.getExe pkgs.nix-timegc} 86400 52 | ''; 53 | log-type = "file"; 54 | logfile = "/var/log/cache-gc.log"; 55 | depends-on = [ 56 | "nix-daemon" 57 | "shared-setup" 58 | ]; 59 | }; 60 | }; 61 | } 62 | -------------------------------------------------------------------------------- /demo/pod.nix: -------------------------------------------------------------------------------- 1 | { 2 | pkgs ? import { }, 3 | }: 4 | let 5 | sysMap = { 6 | "x86_64-linux" = "aarch64-linux"; 7 | "aarch64-linux" = "x86_64-linux"; 8 | }; 9 | pkgsCrossish = import pkgs.path { system = sysMap.${builtins.currentSystem}; }; 10 | 11 | # You can use flakes, npins, niv, fetchTree, fetchFromGitHub or whatever. 12 | easykubenix = builtins.fetchTree { 13 | type = "github"; 14 | owner = "lillecarl"; 15 | repo = "easykubenix"; 16 | }; 17 | ekn = import easykubenix { 18 | inherit pkgs; 19 | modules = [ 20 | { 21 | kluctl = { 22 | discriminator = "demodeploy"; # Used for kluctl pruning (removing resources not in generated manifests) 23 | pushManifest = { 24 | enable = true; # Push manifest (which depends on pkgs.hello) before deploying 25 | to = "ssh://root@192.168.88.20"; # Shouldn't be root but here we are currently, maybe shouldn't be a module option either? 26 | }; 27 | }; 28 | kubernetes.resources.none.Pod.hello.spec = { 29 | containers = { 30 | _namedlist = true; # This is a meta thing to use attrsets instead of lists 31 | hello = { 32 | image = "quay.io/nix-csi/scratch:1.0.1"; # 1.0.1 sets PATH to /nix/var/result/bin 33 | command = [ "hello" ]; 34 | volumeMounts = { 35 | _namedlist = true; 36 | nix.mountPath = "/nix"; 37 | }; 38 | }; 39 | }; 40 | volumes = { 41 | _namedlist = true; 42 | nix.csi = { 43 | driver = "nix.csi.store"; 44 | volumeAttributes.${pkgs.stdenv.hostPlatform.system} = pkgs.hello; # this is stringified into a storepath, 45 | volumeAttributes.${pkgsCrossish.stdenv.hostPlatform.system} = pkgsCrossish.hello; # this is stringified into a storepath, 46 | # Now the manifest depends on pkgs.hello so when we push it we bring pkgs.hello and nix-csi can fetch it. 47 | }; 48 | }; 49 | }; 50 | } 51 | ]; 52 | }; 53 | in 54 | ekn 55 | -------------------------------------------------------------------------------- /guests/ctest.nix: -------------------------------------------------------------------------------- 1 | { 2 | pkgs ? import { }, 3 | }: 4 | let 5 | ctest = pkgs.stdenv.mkDerivation { 6 | pname = "big-binary"; 7 | version = "0.1"; 8 | 9 | src = 10 | pkgs.writeText "main.c" # c 11 | '' 12 | #include 13 | #include 14 | #include // Required for signal handling 15 | #include // Required for bool type 16 | 17 | // ${toString builtins.currentTime} 18 | 19 | #define ARRAY_SIZE 100 * 1024 * 1024 20 | 21 | static char big_array[ARRAY_SIZE] = {1}; 22 | 23 | // Volatile flag to ensure visibility across threads/signal handlers 24 | volatile bool shutdown_requested = false; 25 | 26 | // Signal handler function 27 | void handle_shutdown_signal(int signum) { 28 | printf("\nReceived signal %d. Initiating graceful shutdown...\n", signum); 29 | shutdown_requested = true; 30 | } 31 | 32 | int main() { 33 | // Register signal handlers for SIGINT and SIGTERM 34 | signal(SIGINT, handle_shutdown_signal); 35 | signal(SIGTERM, handle_shutdown_signal); 36 | 37 | printf("Binary size is large due to a static array!\n"); 38 | 39 | // Use a volatile accumulator to prevent the loop from being optimized away. 40 | // This forces a read of every element in the array. 41 | volatile long long sum = 0; 42 | for (size_t i = 0; i < ARRAY_SIZE; ++i) { 43 | sum += big_array[i]; 44 | } 45 | 46 | printf("Finished array sum. Sum: %lld\n", sum); 47 | 48 | // Loop to keep the process alive, checking for shutdown requests 49 | while (!shutdown_requested) { 50 | printf("Application running...\n"); 51 | sleep(1); 52 | } 53 | 54 | printf("Graceful shutdown complete. Exiting.\n"); 55 | return 0; 56 | } 57 | ''; 58 | 59 | # Skip the unpack phase for a single source file. 60 | dontUnpack = true; 61 | 62 | dontStrip = true; 63 | NIX_CFLAGS_COMPILE = "-O0"; 64 | 65 | buildPhase = '' 66 | runHook preBuild 67 | $CC $NIX_CFLAGS_COMPILE -o big-binary $src 68 | runHook postBuild 69 | ''; 70 | 71 | installPhase = '' 72 | runHook preInstall 73 | mkdir -p $out/bin 74 | cp big-binary $out/bin/ 75 | runHook postInstall 76 | ''; 77 | meta.mainProgram = "big-binary"; 78 | }; 79 | in 80 | pkgs.buildEnv { 81 | name = "ctestenv"; 82 | paths = [ 83 | ctest 84 | pkgs.fishMinimal 85 | pkgs.lix 86 | ]; 87 | meta.mainProgram = ctest.meta.mainProgram; 88 | } 89 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "dinix": { 4 | "flake": false, 5 | "locked": { 6 | "lastModified": 1763918467, 7 | "narHash": "sha256-U/su/TftKztIq8eaFNmd6Tdm1sR/G6sqbCaOvH4ixtE=", 8 | "owner": "lillecarl", 9 | "repo": "dinix", 10 | "rev": "837b5f8e2e9c9bf4219b3499c61b3ee3fa1beec3", 11 | "type": "github" 12 | }, 13 | "original": { 14 | "owner": "lillecarl", 15 | "repo": "dinix", 16 | "type": "github" 17 | } 18 | }, 19 | "easykubenix": { 20 | "flake": false, 21 | "locked": { 22 | "lastModified": 1763943450, 23 | "narHash": "sha256-V88fdoF+TYqLgtfrdM19VdGfubohfhGXqb3BUE/H3zA=", 24 | "owner": "lillecarl", 25 | "repo": "easykubenix", 26 | "rev": "31d4df88afecf586c441c833fba5667a9413180e", 27 | "type": "github" 28 | }, 29 | "original": { 30 | "owner": "lillecarl", 31 | "repo": "easykubenix", 32 | "type": "github" 33 | } 34 | }, 35 | "flake-compatish": { 36 | "flake": false, 37 | "locked": { 38 | "lastModified": 1761756986, 39 | "narHash": "sha256-tVpAaOnxCCd3QE4wgPiIeaGQOxjH/rgKLwrt6cjfeZQ=", 40 | "owner": "lillecarl", 41 | "repo": "flake-compatish", 42 | "rev": "cdca78f6ff83848a0f39e53b72fe4c82ba4fd787", 43 | "type": "github" 44 | }, 45 | "original": { 46 | "owner": "lillecarl", 47 | "repo": "flake-compatish", 48 | "type": "github" 49 | } 50 | }, 51 | "nix2container": { 52 | "flake": false, 53 | "locked": { 54 | "lastModified": 1761716996, 55 | "narHash": "sha256-vdOuy2pid2/DasUgb08lDOswdPJkN5qjXfBYItVy/R4=", 56 | "owner": "nlewo", 57 | "repo": "nix2container", 58 | "rev": "e5496ab66e9de9e3f67dc06f692dfbc471b6316e", 59 | "type": "github" 60 | }, 61 | "original": { 62 | "owner": "nlewo", 63 | "repo": "nix2container", 64 | "type": "github" 65 | } 66 | }, 67 | "nixpkgs": { 68 | "locked": { 69 | "lastModified": 1763678758, 70 | "narHash": "sha256-+hBiJ+kG5IoffUOdlANKFflTT5nO3FrrR2CA3178Y5s=", 71 | "owner": "NixOS", 72 | "repo": "nixpkgs", 73 | "rev": "117cc7f94e8072499b0a7aa4c52084fa4e11cc9b", 74 | "type": "github" 75 | }, 76 | "original": { 77 | "owner": "NixOS", 78 | "ref": "nixos-unstable", 79 | "repo": "nixpkgs", 80 | "type": "github" 81 | } 82 | }, 83 | "root": { 84 | "inputs": { 85 | "dinix": "dinix", 86 | "easykubenix": "easykubenix", 87 | "flake-compatish": "flake-compatish", 88 | "nix2container": "nix2container", 89 | "nixpkgs": "nixpkgs" 90 | } 91 | } 92 | }, 93 | "root": "root", 94 | "version": 7 95 | } 96 | -------------------------------------------------------------------------------- /python/nix_csi/subprocessing.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import shlex 3 | import asyncio 4 | import time 5 | from typing import NamedTuple 6 | 7 | from grpclib import GRPCError 8 | from grpclib.const import Status 9 | 10 | logger = logging.getLogger("nix-csi") 11 | 12 | 13 | class SubprocessResult(NamedTuple): 14 | returncode: int 15 | stdout: str 16 | stderr: str 17 | combined: str 18 | elapsed: float 19 | 20 | 21 | async def try_captured(*args): 22 | result = await run_captured(*args) 23 | if result.returncode != 0: 24 | raise GRPCError( 25 | Status.INTERNAL, 26 | f"{shlex.join([str(arg) for arg in args[:5]])}... failed: {result.returncode=}", 27 | result.combined 28 | ) 29 | return result 30 | 31 | 32 | async def try_console(*args, log_level: int = logging.DEBUG): 33 | result = await run_console(*args, log_level=log_level) 34 | if result.returncode != 0: 35 | raise GRPCError( 36 | Status.INTERNAL, 37 | f"{shlex.join([str(arg) for arg in args[:5]])}... failed: {result.returncode=}", 38 | result.combined 39 | ) 40 | return result 41 | 42 | 43 | # Run async subprocess, capture output and returncode 44 | async def run_captured(*args): 45 | return await run_console(*args, log_level=logging.NOTSET) 46 | 47 | 48 | # Run async subprocess, forward output to console and return returncode 49 | async def run_console(*args, log_level: int = logging.DEBUG): 50 | start_time = time.perf_counter() 51 | log_command(*args, log_level=log_level) 52 | proc = await asyncio.create_subprocess_exec( 53 | *[str(arg) for arg in args], 54 | stdout=asyncio.subprocess.PIPE, 55 | stderr=asyncio.subprocess.PIPE, 56 | ) 57 | 58 | stdout_data = [] 59 | stderr_data = [] 60 | combined_data = [] 61 | 62 | async def stream_output(stream, buffer): 63 | async for line in stream: 64 | decoded = line.decode().strip() 65 | buffer.append(decoded) 66 | combined_data.append(decoded) 67 | logger.log(log_level, decoded) 68 | 69 | await asyncio.gather( 70 | stream_output(proc.stdout, stdout_data), 71 | stream_output(proc.stderr, stderr_data), 72 | proc.wait(), 73 | ) 74 | elapsed_time = time.perf_counter() - start_time 75 | if elapsed_time > 5: 76 | logger.info( 77 | f"Comamnd executed in {elapsed_time} seconds: {shlex.join([str(arg) for arg in args[:5]])}" 78 | ) 79 | 80 | assert proc.returncode is not None 81 | return SubprocessResult( 82 | proc.returncode, 83 | "\n".join(stdout_data).strip(), 84 | "\n".join(stderr_data).strip(), 85 | "\n".join(combined_data).strip(), 86 | elapsed_time, 87 | ) 88 | 89 | 90 | def log_command(*args, log_level: int): 91 | logger.log( 92 | log_level, 93 | f"Running command: {shlex.join([str(arg) for arg in args])}", 94 | ) 95 | -------------------------------------------------------------------------------- /demo/nixos.nix: -------------------------------------------------------------------------------- 1 | { 2 | pkgs ? import { }, 3 | }: 4 | let 5 | lib = pkgs.lib; 6 | # You can use flakes, npins, niv, fetchTree, fetchFromGitHub or whatever. 7 | easykubenix = builtins.fetchTree { 8 | type = "github"; 9 | owner = "lillecarl"; 10 | repo = "easykubenix"; 11 | }; 12 | nixos = import "${pkgs.path}/nixos/lib/eval-config.nix" { 13 | inherit pkgs; 14 | modules = [ 15 | ( 16 | { 17 | config, 18 | pkgs, 19 | lib, 20 | ... 21 | }: 22 | { 23 | boot.isContainer = true; 24 | boot.specialFileSystems = lib.mkForce { }; 25 | boot.nixStoreMountOpts = lib.mkForce [ ]; 26 | services.journald.console = "/dev/stderr"; 27 | networking.resolvconf.enable = false; 28 | environment.etc.hostname.enable = lib.mkForce false; 29 | environment.etc.hosts.enable = lib.mkForce false; 30 | system.stateVersion = "25.05"; 31 | } 32 | ) 33 | ]; 34 | }; 35 | ekn = import easykubenix { 36 | inherit pkgs; 37 | modules = [ 38 | { 39 | kluctl = { 40 | discriminator = "demodeploy"; # Used for kluctl pruning (removing resources not in generated manifests) 41 | pushManifest = { 42 | enable = true; # Push manifest (which depends on pkgs.hello) before deploying 43 | to = "ssh://root@192.168.88.20"; # Shouldn't be root but here we are currently, maybe shouldn't be a module option either? 44 | failCachePush = true; 45 | }; 46 | }; 47 | kubernetes.resources.none.Pod.nixos.spec = { 48 | automountServiceAccountToken = false; 49 | containers = { 50 | _namedlist = true; # This is a meta thing to use attrsets instead of lists 51 | hello = { 52 | image = "quay.io/nix-csi/scratch:1.0.1"; # 1.0.1 sets PATH to /nix/var/result/bin 53 | command = [ 54 | "/nix/var/result/init" 55 | "--system" 56 | "--log-level=debug" 57 | "--log-target=console" 58 | ]; 59 | volumeMounts = { 60 | _namedlist = true; 61 | nix.mountPath = "/nix"; 62 | nix.readOnly = true; 63 | run.mountPath = "/run"; 64 | tmp.mountPath = "/tmp"; 65 | cgroup.mountPath = "/sys/fs/cgroup"; 66 | }; 67 | env = { 68 | _namedlist = true; 69 | container.value = "1"; 70 | }; 71 | }; 72 | }; 73 | volumes = { 74 | _namedlist = true; 75 | run.emptyDir.medium = "Memory"; 76 | tmp.emptyDir.medium = "Memory"; 77 | cgroup.hostPath.path = "/sys/fs/cgroup"; 78 | nix.csi = { 79 | driver = "nix.csi.store"; 80 | volumeAttributes.${pkgs.stdenv.hostPlatform.system} = pkgs.buildEnv { 81 | name = "initenv"; 82 | paths = [ 83 | pkgs.fish 84 | pkgs.bash 85 | nixos.config.system.build.toplevel 86 | ]; 87 | }; 88 | }; 89 | }; 90 | }; 91 | } 92 | ]; 93 | }; 94 | in 95 | ekn // { inherit nixos; } 96 | -------------------------------------------------------------------------------- /python/nix_timegc/cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import argparse 4 | import sqlite3 5 | import subprocess 6 | import os 7 | import sys 8 | from sqlite3 import Connection 9 | from datetime import datetime, timedelta 10 | from pathlib import Path 11 | 12 | 13 | def get_db_uri(db_path: Path) -> str: 14 | """Determines the correct SQLite connection URI based on user privileges.""" 15 | if os.geteuid() == 0: 16 | return f"file:{db_path}?mode=rwc&immutable=0" 17 | else: 18 | db_uri = f"file:{db_path}?mode=ro" 19 | try: 20 | with sqlite3.connect(db_uri, uri=True) as testConn: 21 | testConn.execute("SELECT 1 FROM ValidPaths LIMIT 1;") 22 | except sqlite3.OperationalError: 23 | db_uri = f"file:{db_path}?mode=ro&immutable=1" 24 | return db_uri 25 | 26 | 27 | def get_old_paths(conn: Connection, seconds: int) -> list[str]: 28 | """Get store paths older than specified seconds from the Nix database.""" 29 | cutoff_time = int((datetime.now() - timedelta(seconds=seconds)).timestamp()) 30 | cursor = conn.cursor() 31 | cursor.execute( 32 | """ 33 | SELECT path 34 | FROM ValidPaths 35 | WHERE registrationTime < ? 36 | """, 37 | (cutoff_time,), 38 | ) 39 | return [row[0] for row in cursor.fetchall()] 40 | 41 | 42 | def delete_paths(paths: list[str], dry_run: bool = False) -> None: 43 | """Attempt to delete specified store paths.""" 44 | if not paths: 45 | print("No old paths to delete.") 46 | return 47 | 48 | action = "Would delete" if dry_run else "Attempting to delete" 49 | print(f"{action} {len(paths)} paths...") 50 | 51 | if dry_run: 52 | for path in paths: 53 | print(path) 54 | return 55 | 56 | # Let nix-store handle checks for live GC roots. 57 | # check=False allows the command to continue even if some paths are live. 58 | result = subprocess.run( 59 | ["nix-store", "--delete", *paths], 60 | check=False, 61 | capture_output=True, 62 | text=True, 63 | ) 64 | 65 | if result.returncode != 0: 66 | print( 67 | "Deletion command finished with errors (some paths may still be live).", 68 | file=sys.stderr, 69 | ) 70 | print(f"Stderr:\n{result.stderr}", file=sys.stderr) 71 | else: 72 | print(f"Successfully processed {len(paths)} paths for deletion.") 73 | 74 | 75 | def main() -> None: 76 | parser = argparse.ArgumentParser( 77 | description="Delete Nix store paths older than a specified time.", 78 | formatter_class=argparse.RawDescriptionHelpFormatter, 79 | ) 80 | parser.add_argument( 81 | "seconds", type=int, help="Attempt to delete paths older than this many seconds" 82 | ) 83 | parser.add_argument( 84 | "--dry-run", 85 | action="store_true", 86 | help="Print paths that would be deleted without deleting them.", 87 | ) 88 | args = parser.parse_args() 89 | 90 | try: 91 | db_path = Path(os.environ.get("NIX_STATE_DIR", "/nix/var/nix")) / "db/db.sqlite" 92 | if not db_path.exists(): 93 | raise Exception(f"Nix database not found: {db_path}") 94 | 95 | is_dry_run = args.dry_run or os.geteuid() != 0 96 | 97 | with sqlite3.connect(get_db_uri(db_path), uri=True) as conn: 98 | old_paths = get_old_paths(conn, args.seconds) 99 | print( 100 | f"Found {len(old_paths)} paths older than {args.seconds} seconds." 101 | ) 102 | 103 | delete_paths(old_paths, is_dry_run) 104 | 105 | except Exception as e: 106 | print(f"Error: {e}", file=sys.stderr) 107 | sys.exit(1) 108 | 109 | 110 | if __name__ == "__main__": 111 | main() 112 | -------------------------------------------------------------------------------- /kubenix/config.nix: -------------------------------------------------------------------------------- 1 | { 2 | config, 3 | pkgs, 4 | lib, 5 | ... 6 | }: 7 | let 8 | cfg = config.nix-csi; 9 | 10 | defaultSystemFeatures = [ 11 | "nixos-test" 12 | "benchmark" 13 | "big-parallel" 14 | # "kvm" 15 | ]; 16 | 17 | semanticConfType = 18 | with lib.types; 19 | let 20 | confAtom = 21 | nullOr (oneOf [ 22 | bool 23 | int 24 | float 25 | str 26 | path 27 | package 28 | ]) 29 | // { 30 | description = "Nix config atom (null, bool, int, float, str, path or package)"; 31 | }; 32 | in 33 | attrsOf (either confAtom (listOf confAtom)); 34 | 35 | nixSubmodule = 36 | with lib; 37 | types.submodule ( 38 | { config, ... }: 39 | { 40 | options = { 41 | nixConf = mkOption { 42 | type = types.package; 43 | internal = true; 44 | }; 45 | 46 | checkConfig = mkOption { 47 | type = types.bool; 48 | default = true; 49 | }; 50 | 51 | checkAllErrors = mkOption { 52 | type = types.bool; 53 | default = true; 54 | }; 55 | 56 | extraOptions = mkOption { 57 | type = types.lines; 58 | default = ""; 59 | }; 60 | 61 | settings = mkOption { 62 | type = types.submodule { 63 | freeformType = semanticConfType; 64 | options = { }; 65 | }; 66 | default = { }; 67 | }; 68 | }; 69 | config = { 70 | settings = { 71 | trusted-public-keys = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ]; 72 | trusted-users = [ "root" ]; 73 | substituters = mkAfter [ "https://cache.nixos.org/" ]; 74 | system-features = defaultSystemFeatures; 75 | }; 76 | nixConf = 77 | (pkgs.formats.nixConf { 78 | inherit (config) 79 | checkAllErrors 80 | checkConfig 81 | extraOptions 82 | ; 83 | package = pkgs.lix.out; 84 | inherit (pkgs.lix) version; 85 | }).generate 86 | "nix.conf" 87 | config.settings; 88 | }; 89 | } 90 | ); 91 | in 92 | { 93 | options.nix-csi.nixNodeConfig = lib.mkOption { 94 | type = nixSubmodule; 95 | }; 96 | options.nix-csi.nixCacheConfig = lib.mkOption { 97 | type = nixSubmodule; 98 | }; 99 | 100 | config = lib.mkIf cfg.enable { 101 | nix-csi = 102 | let 103 | sharedSettings = { 104 | store = "daemon"; 105 | allowed-users = [ "*" ]; 106 | trusted-users = [ 107 | "root" 108 | "nix" 109 | ]; 110 | auto-allocate-uids = true; 111 | experimental-features = [ 112 | "nix-command" 113 | "flakes" 114 | "auto-allocate-uids" 115 | ]; 116 | builders-use-substitutes = true; 117 | warn-dirty = false; 118 | }; 119 | in 120 | { 121 | nixNodeConfig.settings = sharedSettings // { 122 | substituters = [ 123 | "ssh-ng://nix@nix-cache?trusted=1&priority=20" 124 | ]; 125 | }; 126 | nixCacheConfig.settings = sharedSettings // { 127 | max-jobs = lib.mkDefault 0; 128 | }; 129 | }; 130 | kubernetes.resources.${cfg.namespace} = { 131 | ConfigMap.nix-csi-config.data = { 132 | "nix.conf" = builtins.readFile (cfg.nixNodeConfig.nixConf); 133 | }; 134 | ConfigMap.nix-cache-config.data = { 135 | "nix.conf" = builtins.readFile (cfg.nixCacheConfig.nixConf); 136 | }; 137 | }; 138 | }; 139 | } 140 | -------------------------------------------------------------------------------- /tmp/watch.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import asyncio 4 | import kr8s 5 | import time 6 | 7 | 8 | async def update_config(namespace: str): 9 | """Fetches IPs and performs the update action.""" 10 | print("Debounce timer finished. Fetching IPs and updating config...") 11 | try: 12 | # 1. Get nodes with the specific label directly from the API 13 | builder_nodes = { 14 | node.name 15 | async for node in kr8s.asyncio.get( 16 | "nodes", label_selector="nix.csi/builder" 17 | ) 18 | } 19 | 20 | if not builder_nodes: 21 | print("No builder nodes found.") 22 | return 23 | 24 | # 2. Get relevant pods and filter them 25 | builder_ips = [ 26 | pod.status.podIP 27 | async for pod in kr8s.asyncio.get( 28 | "pods", namespace=namespace, label_selector={"app": "nix-csi-node"} 29 | ) 30 | if pod.spec.nodeName in builder_nodes and pod.status.podIP 31 | ] 32 | 33 | print(f"Discovered builder IPs: {builder_ips}") 34 | # TODO: Render configuration file and restart service here 35 | except kr8s.NotFoundError: 36 | print("Resources not found, skipping update.") 37 | except Exception as e: 38 | print(f"An error occurred during update: {e}") 39 | 40 | 41 | async def main(): 42 | namespace = "nix-csi" 43 | label_selector = {"app": "nix-csi-node"} 44 | debounce_task = None 45 | first_event_time = None 46 | debounce_delay = 5 # seconds 47 | max_debounce_wait = 60 # seconds 48 | 49 | print("Performing initial IP fetch on startup...") 50 | await update_config(namespace) 51 | 52 | while True: 53 | try: 54 | async for event in kr8s.asyncio.watch( 55 | "pods", namespace=namespace, label_selector=label_selector 56 | ): 57 | if event[0] not in ["ADDED", "DELETED"]: 58 | continue 59 | 60 | now = time.monotonic() 61 | 62 | if first_event_time is None: 63 | first_event_time = now 64 | 65 | if debounce_task: 66 | debounce_task.cancel() 67 | 68 | # Force execution if max wait time is exceeded 69 | if now - first_event_time >= max_debounce_wait: 70 | print( 71 | f"Max debounce time of {max_debounce_wait}s reached. Forcing update." 72 | ) 73 | await update_config(namespace) 74 | # Reset state for the next series of events 75 | first_event_time = None 76 | debounce_task = None 77 | continue 78 | 79 | # Otherwise, schedule the normal debounced update 80 | print( 81 | f"Pod event '{event[0]}' detected for {event[1].name}. Resetting debounce timer." 82 | ) 83 | 84 | async def debounced_run(): 85 | nonlocal first_event_time, debounce_task 86 | try: 87 | await asyncio.sleep(debounce_delay) 88 | await update_config(namespace) 89 | # Reset state after a successful debounced run 90 | first_event_time = None 91 | debounce_task = None 92 | except asyncio.CancelledError: 93 | # This is expected if another event arrives 94 | pass 95 | 96 | debounce_task = asyncio.create_task(debounced_run()) 97 | 98 | except asyncio.CancelledError: 99 | print("Main task cancelled, shutting down.") 100 | break 101 | except Exception as e: 102 | print(f"Kubernetes API error: {e}. Retrying in 15 seconds...") 103 | # Reset state on error to avoid stale timers after reconnect 104 | if debounce_task: 105 | debounce_task.cancel() 106 | debounce_task = None 107 | first_event_time = None 108 | await asyncio.sleep(15) 109 | 110 | 111 | if __name__ == "__main__": 112 | try: 113 | asyncio.run(main()) 114 | except KeyboardInterrupt: 115 | print("Interrupted by user.") 116 | -------------------------------------------------------------------------------- /kubenix/cache.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, lib, ... }: 2 | let 3 | cfg = config.nix-csi; 4 | nsRes = config.kubernetes.resources.${cfg.namespace}; 5 | in 6 | { 7 | options.nix-csi.cache = { 8 | storageClassName = lib.mkOption { 9 | type = lib.types.nullOr lib.types.str; 10 | default = null; 11 | }; 12 | loadBalancerPort = lib.mkOption { 13 | type = lib.types.int; 14 | default = 2222; 15 | }; 16 | }; 17 | config = 18 | let 19 | labels = { 20 | "app.kubernetes.io/name" = "cache"; 21 | "app.kubernetes.io/part-of" = "nix-csi"; 22 | }; 23 | in 24 | lib.mkIf cfg.enable { 25 | kubernetes.resources.${cfg.namespace} = { 26 | ConfigMap.authorized-keys.data.authorized_keys = lib.concatLines cfg.authorizedKeys; 27 | StatefulSet.nix-cache = { 28 | spec = { 29 | serviceName = "nix-cache"; 30 | replicas = 1; 31 | selector.matchLabels = labels; 32 | template = { 33 | metadata.labels = labels; 34 | metadata.annotations = { 35 | "kubectl.kubernetes.io/default-container" = "nix-cache"; 36 | configHash = lib.hashAttrs nsRes.ConfigMap.nix-cache-config; 37 | }; 38 | spec = { 39 | serviceAccountName = "nix-csi"; 40 | initContainers = lib.mkNumberedList { 41 | "1" = { 42 | name = "initcopy"; 43 | image = cfg.image; 44 | command = [ "initcopy" ]; 45 | volumeMounts = lib.mkNamedList { 46 | nix-store.mountPath = "/nix-volume"; 47 | nix-config.mountPath = "/etc/nix"; 48 | }; 49 | }; 50 | }; 51 | containers = lib.mkNamedList { 52 | cache = { 53 | command = [ 54 | "dinit" 55 | "--log-file" 56 | "/var/log/dinit.log" 57 | "--quiet" 58 | "cache" 59 | ]; 60 | image = "quay.io/nix-csi/scratch:1.0.1"; 61 | env = lib.mkNamedList { 62 | HOME.value = "/nix/var/nix-csi/root"; 63 | KUBE_NAMESPACE.valueFrom.fieldRef.fieldPath = "metadata.namespace"; 64 | BUILDERS_SERVICE_NAME.value = cfg.internalServiceName; 65 | }; 66 | ports = lib.mkNamedList { 67 | ssh.containerPort = 22; 68 | http.containerPort = 80; 69 | }; 70 | volumeMounts = lib.mkNamedList { 71 | nix-config.mountPath = "/etc/nix-mount"; 72 | ssh.mountPath = "/etc/ssh-mount"; 73 | nix-store = { 74 | mountPath = "/nix"; 75 | subPath = "nix"; 76 | }; 77 | }; 78 | }; 79 | }; 80 | volumes = lib.mkNamedList { 81 | nix-config.configMap.name = "nix-cache-config"; 82 | ssh.secret = { 83 | secretName = "ssh"; 84 | defaultMode = 384; 85 | optional = true; 86 | }; 87 | }; 88 | }; 89 | }; 90 | volumeClaimTemplates = lib.mkNumberedList { 91 | "1" = { 92 | metadata.name = "nix-store"; 93 | spec = { 94 | accessModes = [ "ReadWriteOnce" ]; 95 | resources.requests.storage = "10Gi"; 96 | inherit (cfg.cache) storageClassName; 97 | }; 98 | }; 99 | }; 100 | }; 101 | }; 102 | 103 | Service.nix-cache = { 104 | spec = { 105 | selector = labels; 106 | ports = lib.mkNamedList { 107 | ssh = { 108 | port = 22; 109 | targetPort = "ssh"; 110 | }; 111 | }; 112 | type = "ClusterIP"; 113 | }; 114 | }; 115 | Service.nix-cache-lb = { 116 | spec = { 117 | selector = labels; 118 | ports = lib.mkNamedList { 119 | ssh = { 120 | port = cfg.cache.loadBalancerPort; 121 | targetPort = "ssh"; 122 | }; 123 | }; 124 | type = "LoadBalancer"; 125 | }; 126 | }; 127 | }; 128 | }; 129 | } 130 | -------------------------------------------------------------------------------- /kubenix/daemonset.nix: -------------------------------------------------------------------------------- 1 | { config, pkgs, lib, ... }: 2 | let 3 | cfg = config.nix-csi; 4 | nsRes = config.kubernetes.resources.${cfg.namespace}; 5 | in 6 | { 7 | config = 8 | let 9 | labels = { 10 | "app.kubernetes.io/name" = "csi"; 11 | "app.kubernetes.io/part-of" = "nix-csi"; 12 | }; 13 | in 14 | lib.mkIf cfg.enable { 15 | kubernetes.resources.${cfg.namespace} = { 16 | DaemonSet.nix-node = { 17 | spec = { 18 | updateStrategy = { 19 | type = "RollingUpdate"; 20 | rollingUpdate.maxUnavailable = 1; 21 | }; 22 | selector.matchLabels = labels; 23 | template = { 24 | metadata.labels = labels; 25 | metadata.annotations = { 26 | "kubectl.kubernetes.io/default-container" = "nix-node"; 27 | configHash = lib.hashAttrs nsRes.ConfigMap.nix-cache-config; 28 | }; 29 | spec = { 30 | serviceAccountName = "nix-csi"; 31 | subdomain = cfg.internalServiceName; 32 | initContainers = lib.mkNumberedList { 33 | "1" = { 34 | name = "initcopy"; 35 | image = cfg.image; 36 | command = [ "initcopy" ]; 37 | volumeMounts = lib.mkNamedList { 38 | nix-store.mountPath = "/nix-volume"; 39 | nix-config.mountPath = "/etc/nix"; 40 | }; 41 | }; 42 | }; 43 | containers = lib.mkNamedList { 44 | nix-node = { 45 | image = "quay.io/nix-csi/scratch:1.0.1"; 46 | command = [ 47 | "dinit" 48 | "--log-file" 49 | "/var/log/dinit.log" 50 | "--quiet" 51 | "csi" 52 | ]; 53 | securityContext.privileged = true; 54 | env = 55 | lib.mkNamedList { 56 | CSI_ENDPOINT.value = "unix:///csi/csi.sock"; 57 | HOME.value = "/nix/var/nix-csi/root"; 58 | KUBE_NAMESPACE.valueFrom.fieldRef.fieldPath = "metadata.namespace"; 59 | KUBE_NODE_NAME.valueFrom.fieldRef.fieldPath = "spec.nodeName"; 60 | KUBE_POD_IP.valueFrom.fieldRef.fieldPath = "status.podIP"; 61 | USER.value = "root"; 62 | } 63 | // lib.optionalAttrs (lib.stringLength (builtins.getEnv "GITHUB_KEY") > 0) { 64 | NIX_CONFIG.value = "access-tokens = github.com=${builtins.getEnv "GITHUB_KEY"}"; 65 | }; 66 | volumeMounts = lib.mkNamedList { 67 | csi-socket.mountPath = "/csi"; 68 | nix-config.mountPath = "/etc/nix-mount"; 69 | registration.mountPath = "/registration"; 70 | ssh.mountPath = "/etc/ssh-mount"; 71 | kubelet = { 72 | mountPath = "/var/lib/kubelet"; 73 | mountPropagation = "Bidirectional"; 74 | }; 75 | nix-store = { 76 | mountPath = "/nix"; 77 | mountPropagation = "Bidirectional"; 78 | subPath = "nix"; 79 | }; 80 | }; 81 | }; 82 | csi-node-driver-registrar = { 83 | image = "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.15.0"; 84 | args = [ 85 | "--v=5" 86 | "--csi-address=/csi/csi.sock" 87 | "--kubelet-registration-path=/var/lib/kubelet/plugins/nix.csi.store/csi.sock" 88 | ]; 89 | env = lib.mkNamedList { 90 | KUBE_NODE_NAME.valueFrom.fieldRef.fieldPath = "spec.nodeName"; 91 | }; 92 | volumeMounts = lib.mkNamedList { 93 | csi-socket.mountPath = "/csi"; 94 | kubelet.mountPath = "/var/lib/kubelet"; 95 | registration.mountPath = "/registration"; 96 | }; 97 | }; 98 | livenessprobe = { 99 | image = "registry.k8s.io/sig-storage/livenessprobe:v2.17.0"; 100 | args = [ "--csi-address=/csi/csi.sock" ]; 101 | volumeMounts = lib.mkNamedList { 102 | csi-socket.mountPath = "/csi"; 103 | registration.mountPath = "/registration"; 104 | }; 105 | }; 106 | }; 107 | volumes = lib.mkNamedList { 108 | nix-config.configMap.name = "nix-csi-config"; 109 | registration.hostPath.path = "/var/lib/kubelet/plugins_registry"; 110 | nix-store.hostPath = { 111 | path = cfg.hostMountPath; 112 | type = "DirectoryOrCreate"; 113 | }; 114 | csi-socket.hostPath = { 115 | path = "/var/lib/kubelet/plugins/nix.csi.store/"; 116 | type = "DirectoryOrCreate"; 117 | }; 118 | kubelet.hostPath = { 119 | path = "/var/lib/kubelet"; 120 | type = "Directory"; 121 | }; 122 | ssh.secret = { 123 | secretName = "ssh"; 124 | defaultMode = 384; 125 | }; 126 | }; 127 | }; 128 | }; 129 | }; 130 | }; 131 | # DNS for pods 132 | Service.${cfg.internalServiceName}.spec = { 133 | clusterIP = "None"; 134 | selector = labels; 135 | ports = lib.mkNamedList { 136 | ssh.port = 22; 137 | }; 138 | }; 139 | }; 140 | }; 141 | } 142 | -------------------------------------------------------------------------------- /container/default.nix: -------------------------------------------------------------------------------- 1 | # You could easily get tempted to create folders that go into container root 2 | # using copyToRoot but it's easy to shoot yourself in the foot with Kubernetes 3 | # mounting it's own shit over those paths making a mess out of your life. 4 | { 5 | pkgs, 6 | dinix, 7 | nix2container, 8 | }: 9 | let 10 | lib = pkgs.lib; 11 | dinixEval = import dinix { 12 | inherit pkgs; 13 | modules = [ 14 | ./csi.nix 15 | ./cache.nix 16 | { 17 | config = { 18 | users = { 19 | enable = true; 20 | users.root = { 21 | shell = pkgs.runtimeShell; 22 | homeDir = "/nix/var/nix-csi/root"; 23 | }; 24 | users.nix = { 25 | uid = 1000; 26 | gid = 1000; 27 | comment = "Nix worker user"; 28 | }; 29 | groups.nix.gid = 1000; 30 | groups.nixbld.gid = 30000; 31 | users.sshd = { 32 | uid = 993; 33 | gid = 992; 34 | comment = "SSH privilege separation user"; 35 | }; 36 | groups.sshd.gid = 992; 37 | }; 38 | env-file.variables = { 39 | PYTHONUNBUFFERED = "1"; # If something ends up print logging 40 | NIXPKGS_ALLOW_UNFREE = "1"; # Allow building anything 41 | }; 42 | services.openssh = { 43 | type = "process"; 44 | command = 45 | pkgs.writeScriptBin "openssh-launcher" # bash 46 | '' 47 | #! ${pkgs.runtimeShell} 48 | for i in $(seq 1 10); do 49 | test -f /etc/ssh/sshd_config && break 50 | sleep 1 51 | done 52 | exec ${lib.getExe' pkgs.openssh "sshd"} -D -f /etc/ssh/sshd_config -e 53 | ''; 54 | depends-on = [ "shared-setup" ]; 55 | log-type = "file"; 56 | logfile = "/var/log/ssh.log"; 57 | }; 58 | services.nix-daemon = { 59 | command = "${lib.getExe pkgs.lix} daemon --store local"; 60 | depends-on = [ "shared-setup" ]; 61 | log-type = "file"; 62 | logfile = "/var/log/nix-daemon.log"; 63 | }; 64 | services.config-reconciler = { 65 | type = "process"; 66 | log-type = "file"; 67 | logfile = "/var/log/config-reconciler.log"; 68 | command = 69 | pkgs.writeScriptBin "config-reconciler" # bash 70 | '' 71 | #! ${pkgs.runtimeShell} 72 | set -euo pipefail 73 | export PATH=${ 74 | lib.makeBinPath [ 75 | pkgs.rsync 76 | pkgs.coreutils 77 | ] 78 | } 79 | while true 80 | do 81 | # Tricking OpenSSH's security policies like a pro! 82 | # 83 | # Exclude authorized_keys from root, we don't want anyone 84 | # logging in as root in our containers. 85 | rsync --archive --mkpath --copy-links --chmod=D700,F600 --exclude='authorized_keys' /etc/ssh-mount/ $HOME/.ssh/ 86 | # Here authorized_keys don't matter since we only check %h 87 | # for authorized_keys in sshd config 88 | rsync --archive --mkpath --copy-links --chmod=D700,F600 --chown=root:root /etc/ssh-mount/ /etc/ssh/ 89 | # Everyone should log in as Nix to build or substitute 90 | rsync --archive --mkpath --copy-links --chmod=D700,F600 --chown=nix:nix /etc/ssh-mount/ /home/nix/.ssh/ 91 | 92 | # Copy mounted Nix config to nix config dir 93 | # (Need RW /etc/nix for writing machines file) 94 | rsync --archive --mkpath --copy-links --chmod=D755,F644 --chown=root:root /etc/nix-mount/ /etc/nix/ 95 | # 60s reconciliation is good enough(TM) 96 | sleep 60 97 | done 98 | ''; 99 | }; 100 | services.shared-setup = { 101 | type = "scripted"; 102 | log-type = "file"; 103 | logfile = "/var/log/shared-setup.log"; 104 | depends-on = [ "config-reconciler" ]; 105 | command = 106 | pkgs.writeScriptBin "shared-setup" # bash 107 | '' 108 | #! ${pkgs.runtimeShell} 109 | set -euo pipefail 110 | set -x 111 | export PATH=${ 112 | lib.makeBinPath ( 113 | with pkgs; 114 | [ 115 | rsync 116 | coreutils 117 | lix 118 | ] 119 | ) 120 | } 121 | mkdir --parents {/tmp,/var/tmp} 122 | chmod -R 1777 {/tmp,/var/tmp} 123 | mkdir --parents {/var/log} 124 | chmod -R 0755 {/var/log} 125 | rsync --archive ${pkgs.dockerTools.binSh}/ / 126 | rsync --archive ${pkgs.dockerTools.caCertificates}/ / 127 | rsync --archive ${pkgs.dockerTools.usrBinEnv}/ / 128 | rsync --archive --mkpath --copy-links --chmod=D700,F600 --exclude='authorized_keys' /etc/ssh-mount/ $HOME/.ssh/ 129 | rsync --archive --mkpath --copy-links --chmod=D700,F600 --chown=root:root /etc/ssh-mount/ /etc/ssh/ 130 | rsync --archive --mkpath --copy-links --chmod=D700,F600 --chown=nix:nix /etc/ssh-mount/ /home/nix/.ssh/ 131 | # Fix gcroots for /nix/var/result. The one created by initCopy 132 | # points to invalid symlinks in the chain 133 | # (auto -> /nix-volume/var/result) rather than 134 | # (auto -> /nix/var/result). The link back to store works 135 | # though so this just fixes gcroots. 136 | nix build --store local --out-link /nix/var/result /nix/var/result 137 | ''; 138 | }; 139 | }; 140 | } 141 | ]; 142 | }; 143 | initcopy = 144 | pkgs.writeScriptBin "initcopy" # bash 145 | '' 146 | #! ${pkgs.runtimeShell} 147 | set -euo pipefail 148 | export PATH=${ 149 | lib.makeBinPath [ 150 | pkgs.rsync 151 | pkgs.lix 152 | ] 153 | }:$PATH 154 | # Copy entire entire container image into volume 155 | nix path-info --store local --all | nix copy --store local --to /nix-volume --no-check-sigs --stdin 156 | # Link /nix/var/result 157 | nix build --store /nix-volume --out-link /nix-volume/nix/var/result ${pathEnv} 158 | ''; 159 | pathEnv = pkgs.buildEnv { 160 | name = "rootEnv"; 161 | paths = with pkgs; [ 162 | dinixEval.config.containerWrapper 163 | bash # Used for build and upload scripts 164 | coreutils 165 | fishMinimal 166 | lix 167 | openssh 168 | util-linuxMinimal 169 | gnugrep 170 | getent 171 | doggo 172 | iputils 173 | curl 174 | ]; 175 | }; 176 | in 177 | nix2container.buildImage { 178 | name = "nix-csi"; 179 | initializeNixDatabase = true; 180 | maxLayers = 120; 181 | config.Env = [ 182 | "PATH=${ 183 | lib.makeBinPath [ 184 | initcopy 185 | pathEnv 186 | ] 187 | }" 188 | ]; 189 | # So we can peek into eval 190 | meta.dinixEval = dinixEval; 191 | } 192 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | let 2 | inputs = 3 | ( 4 | let 5 | lockFile = builtins.readFile ./flake.lock; 6 | lockAttrs = builtins.fromJSON lockFile; 7 | fcLockInfo = lockAttrs.nodes.flake-compatish.locked; 8 | fcSrc = builtins.fetchTree fcLockInfo; 9 | flake-compatish = import fcSrc; 10 | in 11 | flake-compatish ./. 12 | ).inputs; 13 | in 14 | { 15 | pkgs ? import inputs.nixpkgs { }, 16 | local ? null, 17 | }: 18 | let 19 | pkgs' = pkgs.extend (import ./pkgs); 20 | in 21 | let 22 | pkgs = pkgs'; 23 | lib = pkgs.lib; 24 | 25 | dinix = inputs.dinix; 26 | 27 | crossAttrs = { 28 | "x86_64-linux" = "aarch64-linux"; 29 | "aarch64-linux" = "x86_64-linux"; 30 | }; 31 | pkgsCross = import pkgs.path { 32 | system = crossAttrs.${builtins.currentSystem}; 33 | overlays = [ (import ./pkgs) ]; 34 | }; 35 | persys = pkgs: rec { 36 | inherit pkgs lib; 37 | n2c = import inputs.nix2container { 38 | inherit pkgs; 39 | }; 40 | easykubenix = import inputs.easykubenix; 41 | 42 | # kubenix evaluation 43 | kubenixEval = easykubenix { 44 | inherit pkgs; 45 | modules = [ 46 | ./kubenix 47 | { 48 | config.nix-csi.authorizedKeys = lib.pipe (lib.filesystem.listFilesRecursive ./keys) [ 49 | (lib.filter (name: lib.hasSuffix ".pub" name)) 50 | (lib.map (name: builtins.readFile name)) 51 | (lib.map (key: lib.trim key)) 52 | ]; 53 | } 54 | { 55 | config = { 56 | nix-csi = { 57 | enable = true; 58 | } 59 | // lib.optionalAttrs (local != null) { 60 | image = imageRef; 61 | cache.storageClassName = "local-path"; 62 | ctest = { 63 | enable = true; 64 | replicas = 1; 65 | }; 66 | }; 67 | kluctl = { 68 | discriminator = "nix-csi"; 69 | } 70 | // lib.optionalAttrs (local != null) { 71 | preDeployScript = 72 | pkgs.writeScriptBin "preDeployScript" # bash 73 | '' 74 | #! ${pkgs.runtimeShell} 75 | set -euo pipefail 76 | set -x 77 | nix copy --no-check-sigs --to ssh-ng://nix@192.168.88.20 "$1" -v || true 78 | ''; 79 | }; 80 | }; 81 | } 82 | ]; 83 | }; 84 | 85 | # script to build daemonset image 86 | image = import ./container { 87 | inherit pkgs dinix; 88 | inherit (n2c) nix2container; 89 | }; 90 | inherit (image.passthru) dinixEval; 91 | 92 | imageToContainerd = copyToContainerd image; 93 | imageRef = "quay.io/nix-csi/${image.imageRefUnsafe}"; 94 | 95 | copyToContainerd = 96 | image: 97 | pkgs.writeScriptBin "copyToContainerd" # execline 98 | '' 99 | #!${pkgs.execline}/bin/execlineb -P 100 | 101 | # Set up a socket we can write to 102 | backtick -E fifo { mktemp -u ocisocket.XXXXXX } 103 | foreground { mkfifo $fifo } 104 | trap { default { rm ''${fifo} } } 105 | 106 | # Dump image to socket in the background 107 | background { 108 | # Ignore stdout (since containerd requires sudo and we want a clean prompt) 109 | redirfd -w 1 /dev/null 110 | ${lib.getExe n2c.skopeo-nix2container} 111 | --insecure-policy copy 112 | nix:${image} 113 | oci-archive:''${fifo}:${imageRef} 114 | } 115 | export CONTAINERD_ADDRESS /run/containerd/containerd.sock 116 | 117 | foreground { 118 | sudo -E ${lib.getExe' pkgs.containerd "ctr"} 119 | --namespace k8s.io 120 | images import ''${fifo} 121 | } 122 | rm ''${fifo} 123 | ''; 124 | 125 | deploy = 126 | pkgs.writers.writeFishBin "deploy" # fish 127 | '' 128 | # Build container image 129 | nix run --file . imageToContainerd || begin 130 | echo "DaemonSet image failed" 131 | return 1 132 | end 133 | ${lib.getExe kubenixEval.deploymentScript} $argv 134 | ''; 135 | 136 | # simpler than devshell 137 | pypkgs = ( 138 | pypkgs: with pypkgs; [ 139 | pkgs.nix-csi 140 | pkgs.csi-proto-python 141 | pkgs.kr8s 142 | ] 143 | ); 144 | python = pkgs.python3.withPackages pypkgs; 145 | xonsh = pkgs.xonsh.override { 146 | extraPackages = pypkgs; 147 | }; 148 | # env to add to PATH with direnv 149 | repoenv = pkgs.buildEnv { 150 | name = "repoenv"; 151 | paths = [ 152 | python 153 | xonsh 154 | n2c.skopeo-nix2container 155 | pkgs.kluctl 156 | pkgs.buildah 157 | pkgs.step-cli 158 | ]; 159 | }; 160 | }; 161 | in 162 | let 163 | on = persys pkgs; 164 | off = persys pkgsCross; 165 | in 166 | on 167 | // { 168 | inherit off; 169 | 170 | uploadCsi = 171 | let 172 | csiUrl = system: "quay.io/nix-csi/nix-csi:${on.pkgs.nix-csi.version}-${system}"; 173 | csiManifest = "quay.io/nix-csi/nix-csi:${on.pkgs.nix-csi.version}"; 174 | in 175 | pkgs.writeScriptBin "merge" # bash 176 | '' 177 | #! ${pkgs.runtimeShell} 178 | set -euo pipefail 179 | set -x 180 | buildDir=$(mktemp -d ocibuild.XXXXXX) 181 | echo $buildDir 182 | mkdir -p $buildDir 183 | cleanup() { 184 | rm -rf "$buildDir" 185 | } 186 | trap cleanup EXIT 187 | # Build and publish nix-csi image(s) 188 | ${lib.getExe on.image.copyTo} oci-archive:$buildDir/csi-${on.pkgs.stdenv.hostPlatform.system}:${csiUrl on.pkgs.stdenv.hostPlatform.system} 189 | ${lib.getExe off.image.copyTo} oci-archive:$buildDir/csi-${off.pkgs.stdenv.hostPlatform.system}:${csiUrl off.pkgs.stdenv.hostPlatform.system} 190 | podman load --input $buildDir/csi-${on.pkgs.stdenv.hostPlatform.system} 191 | podman load --input $buildDir/csi-${off.pkgs.stdenv.hostPlatform.system} 192 | podman push ${csiUrl on.pkgs.stdenv.hostPlatform.system} 193 | podman push ${csiUrl off.pkgs.stdenv.hostPlatform.system} 194 | podman manifest rm ${csiManifest} &>/dev/null || true 195 | podman manifest create ${csiManifest} 196 | podman manifest add ${csiManifest} ${csiUrl on.pkgs.stdenv.hostPlatform.system} 197 | podman manifest add ${csiManifest} ${csiUrl off.pkgs.stdenv.hostPlatform.system} 198 | podman manifest push ${csiManifest} 199 | ''; 200 | uploadScratch = 201 | let 202 | scratchVersion = "1.0.1"; 203 | scratchUrl = system: "quay.io/nix-csi/scratch:${scratchVersion}-${system}"; 204 | scratchManifest = "quay.io/nix-csi/scratch:${scratchVersion}"; 205 | in 206 | pkgs.writeScriptBin "merge" # bash 207 | '' 208 | #! ${pkgs.runtimeShell} 209 | set -euo pipefail 210 | set -x 211 | # Build and publish scratch image(s) 212 | container=$(buildah from --platform linux/amd64 scratch) 213 | buildah config --env "PATH=/nix/var/result/bin" $container 214 | buildah commit $container ${scratchUrl on.pkgs.stdenv.hostPlatform.system} 215 | buildah push ${scratchUrl on.pkgs.stdenv.hostPlatform.system} 216 | container=$(buildah from --platform linux/arm64 scratch) 217 | buildah config --env "PATH=/nix/var/result/bin" $container 218 | buildah commit $container ${scratchUrl off.pkgs.stdenv.hostPlatform.system} 219 | buildah push ${scratchUrl off.pkgs.stdenv.hostPlatform.system} 220 | buildah manifest rm ${scratchManifest} &>/dev/null || true 221 | buildah manifest create ${scratchManifest} 222 | buildah manifest add ${scratchManifest} ${scratchUrl on.pkgs.stdenv.hostPlatform.system} 223 | buildah manifest add ${scratchManifest} ${scratchUrl off.pkgs.stdenv.hostPlatform.system} 224 | buildah manifest push ${scratchManifest} 225 | ''; 226 | } 227 | -------------------------------------------------------------------------------- /python/nix_cache/cli.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import asyncio 4 | import os 5 | import kr8s 6 | import argparse 7 | import logging 8 | import textwrap 9 | from pathlib import Path 10 | from nix_csi.subprocessing import run_captured 11 | 12 | 13 | async def update_machines(namespace: str): 14 | """Fetches builder pod info and atomically updates the nix machines file.""" 15 | try: 16 | builder_nodes = { 17 | node.name: node 18 | async for node in kr8s.asyncio.get( 19 | "nodes", label_selector="nix.csi/builder" 20 | ) 21 | } 22 | 23 | if not builder_nodes: 24 | logging.info("No builder nodes found, preparing to clear machines file.") 25 | 26 | arch_map = { 27 | "amd64": "x86_64-linux", 28 | "arm64": "aarch64-linux", 29 | } 30 | 31 | builders = [] 32 | async for pod in kr8s.asyncio.get( 33 | "pods", namespace=namespace, label_selector={"app": "nix-csi-node"} 34 | ): 35 | node_name = pod.spec.nodeName 36 | if node := builder_nodes.get(node_name): 37 | k8s_arch = node.metadata.labels.get("kubernetes.io/arch") 38 | if not k8s_arch: 39 | logging.warning( 40 | f"Node '{node_name}' missing 'kubernetes.io/arch' label. Skipping pod '{pod.name}'." 41 | ) 42 | continue 43 | 44 | nix_arch = arch_map.get(k8s_arch) 45 | if not nix_arch: 46 | logging.warning( 47 | f"Unhandled architecture '{k8s_arch}' for node '{node_name}'. Skipping pod '{pod.name}'." 48 | ) 49 | continue 50 | 51 | # Cluster internal DNS search-domain will sort out the full name 52 | builders.append( 53 | f"ssh-ng://{pod.metadata['name']}.{os.environ['BUILDERS_SERVICE_NAME']}?trusted=1 {nix_arch}" 54 | ) 55 | 56 | machines_path = Path("/etc/nix/machines") 57 | temp_path = machines_path.with_suffix(".tmp") 58 | content = "".join(f"{builder}\n" for builder in builders) 59 | temp_path.write_text(content) 60 | temp_path.rename(machines_path) 61 | 62 | logging.info( 63 | f"Atomically updated {machines_path} with {len(builders)} builders." 64 | ) 65 | 66 | except kr8s.NotFoundError: 67 | logging.warning("Resources not found, skipping update.") 68 | except Exception: 69 | logging.exception("An error occurred during update.") 70 | 71 | 72 | async def update_worker(update_event: asyncio.Event, namespace: str): 73 | """Waits for an update signal, debounces, and runs the update.""" 74 | while True: 75 | await update_event.wait() 76 | update_event.clear() 77 | logging.info("Change detected. Debouncing for 5 second before update.") 78 | await asyncio.sleep(5) 79 | await update_machines(namespace) 80 | 81 | 82 | async def watch_pods(update_event: asyncio.Event, namespace: str): 83 | """Watches for pod events and signals for a config update.""" 84 | label_selector = {"app": "nix-csi-node"} 85 | logging.info( 86 | f"Watching for pod events in namespace '{namespace}' with selector '{label_selector}'..." 87 | ) 88 | while True: 89 | try: 90 | async for event_type, obj in kr8s.asyncio.watch( 91 | "pods", namespace=namespace, label_selector=label_selector 92 | ): 93 | if event_type not in ["ADDED", "DELETED"]: 94 | continue 95 | 96 | logging.info( 97 | f"Pod event '{event_type}' for {obj.name}. Triggering update." 98 | ) 99 | update_event.set() 100 | 101 | except asyncio.CancelledError: 102 | logging.info("Pod watcher task cancelled.") 103 | break 104 | except Exception: 105 | logging.exception("Pod watch error. Retrying in 15 seconds...") 106 | await asyncio.sleep(15) 107 | 108 | 109 | async def watch_nodes(update_event: asyncio.Event): 110 | """Watches for node label events and signals for a config update.""" 111 | label_selector = "nix.csi/builder" 112 | logging.info(f"Watching for node events with selector '{label_selector}'...") 113 | while True: 114 | try: 115 | async for event_type, obj in kr8s.asyncio.watch( 116 | "nodes", label_selector=label_selector 117 | ): 118 | if event_type not in ["ADDED", "DELETED"]: 119 | continue 120 | logging.info( 121 | f"Node event '{event_type}' for {obj.name}. Triggering update." 122 | ) 123 | update_event.set() 124 | 125 | except asyncio.CancelledError: 126 | logging.info("Node watcher task cancelled.") 127 | break 128 | except Exception: 129 | logging.exception("Node watch error. Retrying in 15 seconds...") 130 | await asyncio.sleep(15) 131 | 132 | 133 | async def reconcile_secrets(namespace: str, privKey: Path, pubKey: Path): 134 | """Periodically ensures the SSH secret is up-to-date.""" 135 | while True: 136 | try: 137 | logging.info("Reconciling SSH secret...") 138 | authorized_keys_cm = await kr8s.asyncio.objects.ConfigMap.get( 139 | name="authorized-keys", namespace=namespace 140 | ) 141 | keys = [pubKey.read_text().strip()] + str( 142 | authorized_keys_cm.data.get("authorized_keys", "") 143 | ).strip().splitlines() 144 | 145 | stringData = { 146 | "id_ed25519": privKey.read_text(), 147 | "id_ed25519.pub": pubKey.read_text(), 148 | "known_hosts": f"* {pubKey.read_text()}", 149 | "config": textwrap.dedent(""" 150 | Host * 151 | User nix 152 | IdentityFile ~/.ssh/id_ed25519 153 | UserKnownHostsFile ~/.ssh/known_hosts 154 | """), 155 | "authorized_keys": "\n".join(keys) + "\n", 156 | "sshd_config": textwrap.dedent(""" 157 | Port 22 158 | AddressFamily Any 159 | HostKey /etc/ssh/id_ed25519 160 | SyslogFacility DAEMON 161 | SetEnv PATH=/nix/var/result/bin 162 | SetEnv NIXPKGS_ALLOW_UNFREE=1 163 | PermitRootLogin no 164 | PubkeyAuthentication yes 165 | PasswordAuthentication no 166 | ChallengeResponseAuthentication no 167 | UsePAM no 168 | AuthorizedKeysFile %h/.ssh/authorized_keys 169 | StrictModes no 170 | Subsystem sftp internal-sftp 171 | """), 172 | } 173 | 174 | secret_manifest = { 175 | "apiVersion": "v1", 176 | "kind": "Secret", 177 | "metadata": {"name": "ssh", "namespace": namespace}, 178 | "stringData": stringData, 179 | "type": "Opaque", 180 | } 181 | secret = await kr8s.asyncio.objects.Secret(secret_manifest) 182 | 183 | if await secret.exists(): 184 | await secret.patch({"stringData": stringData}) 185 | logging.info("Patched existing SSH secret.") 186 | else: 187 | await secret.create() 188 | logging.info("Created new SSH secret.") 189 | 190 | await asyncio.sleep(60) 191 | 192 | except asyncio.CancelledError: 193 | logging.info("Secret reconciler task cancelled.") 194 | break 195 | except Exception: 196 | logging.exception("Error reconciling secret. Retrying in 15 seconds...") 197 | await asyncio.sleep(15) 198 | 199 | 200 | async def async_main(): 201 | namespace = os.environ["KUBE_NAMESPACE"] 202 | 203 | privKey = Path("/nix/var/nix-csi/root/privkey") 204 | pubKey = privKey.with_suffix(".pub") 205 | if not privKey.exists(): 206 | await run_captured( 207 | "ssh-keygen", "-t", "ed25519", "-f", privKey, "-N", "", "-C", "nix-csi" 208 | ) 209 | 210 | # This event will be used to signal when an update is needed. 211 | update_needed_event = asyncio.Event() 212 | 213 | # Trigger an initial update on startup 214 | update_needed_event.set() 215 | 216 | tasks = [ 217 | asyncio.create_task(update_worker(update_needed_event, namespace)), 218 | asyncio.create_task(watch_pods(update_needed_event, namespace)), 219 | asyncio.create_task(watch_nodes(update_needed_event)), 220 | asyncio.create_task(reconcile_secrets(namespace, privKey, pubKey)), 221 | ] 222 | 223 | await asyncio.gather(*tasks) 224 | 225 | 226 | def parse_args(): 227 | parser = argparse.ArgumentParser(description="nix CSI driver") 228 | parser.add_argument( 229 | "--loglevel", 230 | default="INFO", 231 | choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], 232 | help="Set the logging level (default: INFO)", 233 | ) 234 | return parser.parse_args() 235 | 236 | 237 | def main(): 238 | args = parse_args() 239 | logging.basicConfig( 240 | level=logging.INFO, 241 | format="%(asctime)s %(levelname)s [%(name)s] %(message)s", 242 | ) 243 | logging.getLogger("nix-csi").setLevel(getattr(logging, args.loglevel)) 244 | 245 | try: 246 | asyncio.run(async_main()) 247 | except KeyboardInterrupt: 248 | print("Interrupted by user.") 249 | 250 | 251 | if __name__ == "__main__": 252 | main() 253 | -------------------------------------------------------------------------------- /python/nix_csi/service.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | import os 4 | import shutil 5 | import socket 6 | import math 7 | import subprocess 8 | 9 | from csi import csi_grpc, csi_pb2 10 | from grpclib import GRPCError 11 | from grpclib.const import Status 12 | from grpclib.server import Server 13 | from importlib import metadata 14 | from pathlib import Path 15 | from typing import List 16 | from cachetools import TTLCache 17 | from asyncio import Semaphore, sleep 18 | from collections import defaultdict 19 | from .identityservicer import IdentityServicer 20 | from .copytocache import copyToCache 21 | from .subprocessing import run_captured, run_console, try_captured, try_console 22 | 23 | logger = logging.getLogger("nix-csi") 24 | 25 | CSI_PLUGIN_NAME = "nix.csi.store" 26 | CSI_VENDOR_VERSION = metadata.version("nix-csi") 27 | 28 | MOUNT_ALREADY_MOUNTED = 32 29 | 30 | # Paths we base everything on. 31 | # Remember that these are CSI pod paths not node paths. 32 | NIX_ROOT = Path("/") 33 | CSI_ROOT = NIX_ROOT / "nix/var/nix-csi" 34 | CSI_VOLUMES = CSI_ROOT / "volumes" 35 | CSI_GCROOTS = NIX_ROOT / "nix/var/nix/gcroots/nix-csi" 36 | NAMESPACE = os.environ["KUBE_NAMESPACE"] 37 | 38 | RSYNC_CONCURRENCY = Semaphore(1) 39 | 40 | 41 | def get_kernel_boot_time(stat_file: Path = Path("/proc/stat")) -> int: 42 | """Returns kernel boot time as Unix timestamp.""" 43 | for line in stat_file.read_text().splitlines(): 44 | if line.startswith("btime "): 45 | return int(line.split()[1].strip()) 46 | raise RuntimeError("btime not found in /hoststat") 47 | 48 | 49 | def reboot_cleanup(): 50 | """Cleanup volume trees and gcroots if we have rebooted""" 51 | stat_file = Path("/proc/stat") 52 | state_file = CSI_ROOT / "proc_stat" 53 | state_file.parent.mkdir(parents=True, exist_ok=True) 54 | 55 | needs_cleanup = False 56 | if state_file.exists(): 57 | try: 58 | old_boot = get_kernel_boot_time(state_file) 59 | current_boot = get_kernel_boot_time(stat_file) 60 | needs_cleanup = old_boot != current_boot 61 | except RuntimeError: 62 | # Corrupted state file, treat as needing cleanup 63 | needs_cleanup = True 64 | 65 | shutil.copy2(stat_file, state_file) 66 | 67 | if needs_cleanup: 68 | logger.info("Reboot detected - cleaning volumes and gcroots") 69 | for path in [CSI_VOLUMES, CSI_GCROOTS]: 70 | if path.exists(): 71 | shutil.rmtree(path) 72 | path.mkdir(parents=True, exist_ok=True) 73 | 74 | 75 | async def get_current_system(): 76 | return ( 77 | await try_captured( 78 | "nix", "eval", "--raw", "--impure", "--expr", "builtins.currentSystem" 79 | ) 80 | ).stdout 81 | 82 | 83 | def initialize(): 84 | logger.info("Initializing NodeServicer") 85 | # Clean old volumes on startup 86 | reboot_cleanup() 87 | # Create directories we operate in 88 | CSI_ROOT.mkdir(parents=True, exist_ok=True) 89 | CSI_VOLUMES.mkdir(parents=True, exist_ok=True) 90 | CSI_GCROOTS.mkdir(parents=True, exist_ok=True) 91 | 92 | 93 | class NodeServicer(csi_grpc.NodeBase): 94 | # Cache positive Nix commands 95 | pathInfoCache: TTLCache[Path, List[str]] = TTLCache(math.inf, 60) 96 | volumeLocks: defaultdict[str, Semaphore] = defaultdict(Semaphore) 97 | 98 | def __init__(self, system: str): 99 | self.system = system 100 | 101 | async def NodePublishVolume(self, stream): 102 | request: csi_pb2.NodePublishVolumeRequest | None = await stream.recv_message() 103 | if request is None: 104 | raise ValueError("NodePublishVolumeRequest is None") 105 | 106 | async with self.volumeLocks[request.volume_id]: 107 | targetPath = Path(request.target_path) 108 | storePath = request.volume_context.get(self.system) 109 | packagePath: Path = Path("/nonexistent/path/that/should/never/exist") 110 | gcPath = CSI_GCROOTS / request.volume_id 111 | 112 | if storePath is not None: 113 | packagePath = Path(storePath) 114 | if not packagePath.exists(): 115 | logger.debug(f"{storePath=}") 116 | buildCommand = [ 117 | "nix", 118 | "build", 119 | "--out-link", 120 | gcPath, 121 | packagePath, 122 | ] 123 | 124 | # Fetch storePath from caches 125 | await try_console(*buildCommand) 126 | else: 127 | raise GRPCError( 128 | Status.INVALID_ARGUMENT, 129 | f"Volume doesn't have storePath configured for {self.system}", 130 | ) 131 | 132 | if not packagePath.exists(): 133 | raise GRPCError( 134 | Status.INVALID_ARGUMENT, 135 | "packagePath passed through all steps yet doesn't exist", 136 | ) 137 | 138 | # Root directory for volume. Contains /nix, also contains "workdir" and 139 | # "upperdir" if we're doing overlayfs 140 | volumeRoot = CSI_VOLUMES / request.volume_id 141 | # Capitalized to emphasise they're Nix environment variables 142 | NIX_STATE_DIR = volumeRoot / "nix/var/nix" 143 | # Create NIX_STATE_DIR where database will be initialized 144 | NIX_STATE_DIR.mkdir(parents=True, exist_ok=True) 145 | 146 | # Get closure 147 | paths = [] 148 | pathInfoCacheResult = self.pathInfoCache.get(packagePath) 149 | if pathInfoCacheResult is not None: 150 | paths = pathInfoCacheResult 151 | else: 152 | pathInfo = await try_captured( 153 | "nix", 154 | "path-info", 155 | "--recursive", 156 | packagePath, 157 | ) 158 | paths = pathInfo.stdout.splitlines() 159 | self.pathInfoCache[packagePath] = paths 160 | 161 | try: 162 | # This try block is essentially nix copy into a chroot store with 163 | # extra steps. (Hardlinking instead of dumbcopying) 164 | 165 | # Install CSI gcroots 166 | await try_captured("nix", "build", "--out-link", gcPath, packagePath) 167 | 168 | # Copy closure to substore, rsync saves a lot of implementation 169 | # headache here. --archive keeps all attributes, --hard-links 170 | # hardlinks everything hardlinkable. 171 | async with RSYNC_CONCURRENCY: 172 | await try_captured( 173 | "rsync", 174 | "--one-file-system", 175 | "--recursive", 176 | "--links", 177 | "--hard-links", 178 | "--mkpath", 179 | *paths, 180 | volumeRoot / "nix/store", 181 | ) 182 | 183 | # Create Nix database 184 | # This is a bash script that runs nix-store --dump-db | NIX_STATE_DIR=something nix-store --load-db 185 | await try_captured( 186 | "nix_init_db", 187 | NIX_STATE_DIR, 188 | *paths, 189 | ) 190 | 191 | # install gcroots in container using chroot store this is 192 | # required because the auto roots created for /nix/var/result 193 | # will point to Narnia while this one points into store. 194 | await try_captured( 195 | "nix", 196 | "build", 197 | "--store", 198 | volumeRoot, 199 | "--out-link", 200 | NIX_STATE_DIR / "gcroots/result", 201 | packagePath, 202 | ) 203 | 204 | # install /nix/var/result in container using chroot store 205 | await try_captured( 206 | "nix", 207 | "build", 208 | "--store", 209 | volumeRoot, 210 | "--out-link", 211 | volumeRoot / "nix/var/result", 212 | packagePath, 213 | ) 214 | except Exception as ex: 215 | # Remove gcroots if we failed something else 216 | gcPath.unlink(missing_ok=True) 217 | # Remove what we were working on 218 | shutil.rmtree(volumeRoot, True) 219 | raise ex 220 | 221 | targetPath.mkdir(parents=True, exist_ok=True) 222 | mountCommand = [] 223 | if request.readonly: 224 | # For readonly we use a bind mount, the benefit is that different 225 | # container stores using bindmounts will get the same inodes and 226 | # share page cache with others, reducing memory usage. 227 | mountCommand = [ 228 | "mount", 229 | "--verbose", 230 | "--bind", 231 | "-o", 232 | "ro", 233 | volumeRoot / "nix", 234 | targetPath, 235 | ] 236 | else: 237 | # For readwrite we use an overlayfs mount, the benefit here is that 238 | # it works as CoW even if the underlying filesystem doesn't support 239 | # it, reducing host storage usage. 240 | workdir = volumeRoot / "workdir" 241 | upperdir = volumeRoot / "upperdir" 242 | workdir.mkdir(parents=True, exist_ok=True) 243 | upperdir.mkdir(parents=True, exist_ok=True) 244 | mountCommand = [ 245 | "mount", 246 | "--verbose", 247 | "-t", 248 | "overlay", 249 | "overlay", 250 | "-o", 251 | f"rw,lowerdir={volumeRoot / 'nix'},upperdir={upperdir},workdir={workdir}", 252 | targetPath, 253 | ] 254 | 255 | mount = await run_console(*mountCommand) 256 | if mount.returncode == MOUNT_ALREADY_MOUNTED: 257 | logger.debug(f"Mount target {targetPath} was already mounted") 258 | elif mount.returncode != 0: 259 | raise GRPCError( 260 | Status.INTERNAL, 261 | f"Failed to mount {mount.returncode=} {mount.stderr=}", 262 | ) 263 | 264 | reply = csi_pb2.NodePublishVolumeResponse() 265 | await stream.send_message(reply) 266 | 267 | asyncio.create_task(copyToCache(packagePath)) 268 | 269 | async def NodeUnpublishVolume(self, stream): 270 | request: csi_pb2.NodeUnpublishVolumeRequest | None = await stream.recv_message() 271 | if request is None: 272 | raise ValueError("NodeUnpublishVolumeRequest is None") 273 | 274 | async with self.volumeLocks[request.volume_id]: 275 | errors = [] 276 | targetPath = Path(request.target_path) 277 | 278 | for _ in range(100): 279 | if not os.path.ismount(targetPath): 280 | break 281 | 282 | umount = await run_console( 283 | "umount", "--recursive", "--verbose", targetPath 284 | ) 285 | if umount.returncode != 0: 286 | errors.append( 287 | f"umount failed {umount.returncode=} {umount.stderr=}" 288 | ) 289 | await sleep(0.1) 290 | else: 291 | errors.append("Failed to unmount after many retries") 292 | 293 | gcPath = CSI_GCROOTS / request.volume_id 294 | if gcPath.exists(): 295 | try: 296 | gcPath.unlink() 297 | except Exception as ex: 298 | errors.append(f"gcroot unlink failed: {ex}") 299 | 300 | volume_path = CSI_VOLUMES / request.volume_id 301 | if volume_path.exists(): 302 | try: 303 | shutil.rmtree(volume_path) 304 | except Exception as ex: 305 | errors.append(f"volume cleanup failed: {ex}") 306 | 307 | if errors: 308 | raise GRPCError(Status.INTERNAL, "cleanup failed", "; ".join(errors)) 309 | 310 | reply = csi_pb2.NodeUnpublishVolumeResponse() 311 | await stream.send_message(reply) 312 | 313 | async def NodeGetCapabilities(self, stream): 314 | request: csi_pb2.NodeGetCapabilitiesRequest | None = await stream.recv_message() 315 | if request is None: 316 | raise ValueError("NodeGetCapabilitiesRequest is None") 317 | reply = csi_pb2.NodeGetCapabilitiesResponse(capabilities=[]) 318 | await stream.send_message(reply) 319 | 320 | async def NodeGetInfo(self, stream): 321 | request: csi_pb2.NodeGetInfoRequest | None = await stream.recv_message() 322 | if request is None: 323 | raise ValueError("NodeGetInfoRequest is None") 324 | reply = csi_pb2.NodeGetInfoResponse( 325 | node_id=str(os.environ.get("KUBE_NODE_NAME")), 326 | ) 327 | await stream.send_message(reply) 328 | 329 | async def NodeGetVolumeStats(self, stream): 330 | del stream # typechecker 331 | raise GRPCError(Status.UNIMPLEMENTED, "NodeGetVolumeStats not implemented") 332 | 333 | async def NodeExpandVolume(self, stream): 334 | del stream # typechecker 335 | raise GRPCError(Status.UNIMPLEMENTED, "NodeExpandVolume not implemented") 336 | 337 | async def NodeStageVolume(self, stream): 338 | del stream # typechecker 339 | raise GRPCError(Status.UNIMPLEMENTED, "NodeStageVolume not implemented") 340 | 341 | async def NodeUnstageVolume(self, stream): 342 | del stream # typechecker 343 | raise GRPCError(Status.UNIMPLEMENTED, "NodeUnstageVolume not implemented") 344 | 345 | 346 | async def serve(): 347 | sock_path = "/csi/csi.sock" 348 | Path(sock_path).unlink(missing_ok=True) 349 | 350 | identityServicer = IdentityServicer() 351 | nodeServicer = NodeServicer(await get_current_system()) 352 | initialize() 353 | 354 | server = Server( 355 | [ 356 | identityServicer, 357 | nodeServicer, 358 | ] 359 | ) 360 | 361 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 362 | sock.bind(sock_path) 363 | sock.listen(128) 364 | sock.setblocking(False) 365 | 366 | await server.start(sock=sock) 367 | logger.info(f"CSI driver (grpclib) listening on unix://{sock_path}") 368 | await server.wait_closed() 369 | --------------------------------------------------------------------------------