├── tf ├── provider.tf ├── .gitignore ├── charon-worker-ud.nix ├── .terraform.lock.hcl ├── lambda.tf └── ci.tf ├── keys ├── styx-dev-1.public ├── testsuite.public ├── styx-test-1.public ├── styx-nixcache-test-1.public ├── styx-dev-1.secret ├── testsuite.secret └── README.md ├── params ├── dev-1.signed ├── test-1.signed ├── dev-1.json └── test-1.json ├── .envrc ├── pb ├── gen.go ├── manifest.proto ├── extras.go ├── manifest_meta.proto ├── buildroot.proto ├── manifester.proto ├── params.proto ├── entry.proto ├── narinfo.proto ├── db.proto ├── manifest.pb.go └── manifest_meta.pb.go ├── bin ├── umountall ├── localworker ├── testlocal ├── testumount ├── diffpaths ├── testmount ├── testvm ├── remanifest_missing ├── imagestats ├── testdaemon ├── roundtripmount ├── deploy-styx ├── runvm └── make-styx-include ├── Makefile ├── .gitignore ├── manifester ├── util.go ├── tarball_test.go ├── proto.go └── chunkstore.go ├── shell.nix ├── common ├── chunksize.go ├── zstd.go ├── blocks.go ├── const.go ├── shift │ └── shift.go ├── trunc.go ├── cdig │ ├── cdig_test.go │ └── cdig.go ├── load.go ├── util.go ├── error.go ├── chunkpool.go ├── client │ └── client.go ├── http.go ├── map.go ├── systemd │ └── systemd.go ├── signature.go └── errgroup │ └── errgroup.go ├── tests ├── bare_test.go ├── variable_chunk_test.go ├── repeated_read_test.go ├── small_test.go ├── remanifest_test.go ├── tarball_test.go ├── diff_test.go ├── materialize_test.go ├── chunked_manifest_test.go ├── recompress_test.go ├── vaporize_test.go ├── restart_test.go ├── prefetch_test.go └── gc_test.go ├── erofs ├── slab.go └── slab_image.go ├── vm-base.nix ├── charon.orq ├── ci ├── start.go ├── proto.go ├── tclient.go ├── config │ └── default.nix └── notify.go ├── daemon ├── const.go ├── context.go ├── prefetch.go ├── util.go ├── repair.go ├── recompress.go ├── diff_test.go ├── proto.go ├── stats.go ├── fscache.go ├── catalog.go ├── debug.go └── tarball.go ├── cmd ├── styx │ ├── ssm.go │ ├── cobrautil.go │ ├── tarball.go │ └── internal.go └── charon │ ├── cobrautil.go │ └── main.go ├── vm-testsuite.nix ├── testvm.nix ├── vm-interactive.nix ├── module └── default.nix └── go.mod /tf/provider.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-east-1" 3 | } 4 | -------------------------------------------------------------------------------- /keys/styx-dev-1.public: -------------------------------------------------------------------------------- 1 | styx-dev-1:SCMYzQjLTMMuy/MlovgOX0rRVCYKOj+cYAfQrqzcLu0= -------------------------------------------------------------------------------- /keys/testsuite.public: -------------------------------------------------------------------------------- 1 | testsuite:19gI/OY0vBPjxGLu/rlF2aTsqAgII0Fli9+2pi/EOBQ= -------------------------------------------------------------------------------- /keys/styx-test-1.public: -------------------------------------------------------------------------------- 1 | styx-test-1:bmMrKgN5yF3dGgOI67TZSfLts5IQHwdrOCZ7XHcaN+w= -------------------------------------------------------------------------------- /params/dev-1.signed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dnr/styx/HEAD/params/dev-1.signed -------------------------------------------------------------------------------- /params/test-1.signed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dnr/styx/HEAD/params/test-1.signed -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | use nix 2 | source_env_if_exists .envrc.private 3 | PATH_add bin 4 | PATH_add scripts 5 | -------------------------------------------------------------------------------- /keys/styx-nixcache-test-1.public: -------------------------------------------------------------------------------- 1 | styx-nixcache-test-1:IbJB9NG5antB2WpE+aE5QzmXapT2yLQb8As/FRkbm3Q= 2 | -------------------------------------------------------------------------------- /tf/.gitignore: -------------------------------------------------------------------------------- 1 | .terraform 2 | terraform.tfstate 3 | terraform.tfstate.backup 4 | terraform.tfvars 5 | -------------------------------------------------------------------------------- /pb/gen.go: -------------------------------------------------------------------------------- 1 | package pb 2 | 3 | //go:generate sh -c "protoc --go_out=. --go_opt=paths=source_relative *.proto" 4 | -------------------------------------------------------------------------------- /bin/umountall: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | for d in $(mount -t erofs | cut -d' ' -f3); do sudo ./styx umount $(basename $d); done 3 | -------------------------------------------------------------------------------- /keys/styx-dev-1.secret: -------------------------------------------------------------------------------- 1 | styx-dev-1:5tUpekkQnRrmrEypweLOSAMPdDSsW2BqEvWaszV/17BIIxjNCMtMwy7L8yWi+A5fStFUJgo6P5xgB9CurNwu7Q== -------------------------------------------------------------------------------- /keys/testsuite.secret: -------------------------------------------------------------------------------- 1 | testsuite:Hdj/UnsS1hp/kPey8JBY6pG3leqp8iKtIzGYYWm9f07X2Aj85jS8E+PEYu7+uUXZpOyoCAgjQWWL37amL8Q4FA== -------------------------------------------------------------------------------- /bin/localworker: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | make charon && ./charon worker --temporal_params=keys/temporal-creds-charon.secret --worker --scale_group_name=charon-asg 3 | -------------------------------------------------------------------------------- /bin/testlocal: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -eux 3 | # Be careful, this can lock up the cachefiles subsystem if tests fail in 4 | # particular ways. 5 | go test -v -exec sudo "$@" ./tests 6 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | styx: */*.go */*/*.go 3 | go build ./cmd/styx 4 | 5 | charon: cmd/charon/*.go ci/*.go 6 | go build ./cmd/charon 7 | 8 | gen generate: 9 | go generate ./... 10 | 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.direnv 2 | /.envrc.private 3 | /styx 4 | /charon 5 | /notes 6 | /notes-* 7 | /crap 8 | /keys/*.secret 9 | !/keys/styx-dev-1.secret 10 | /testvm.qcow2 11 | /work 12 | /result 13 | /result-* 14 | -------------------------------------------------------------------------------- /bin/testumount: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | base=/mnt/test 4 | 5 | for dir in $base/*; do 6 | if [[ $(stat -f -c %t $dir) = e0f5e1e2 ]]; then 7 | sudo ./styx umount $(basename $dir) 8 | fi 9 | done 10 | -------------------------------------------------------------------------------- /bin/diffpaths: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | nix-store --dump $1 > /dev/shm/a.nar 3 | nix-store --dump $2 > /dev/shm/b.nar 4 | trap "rm /dev/shm/a.nar /dev/shm/b.nar" EXIT 5 | du -b /dev/shm/{a,b}.nar 6 | zstd -c --single-thread -3 --patch-from /dev/shm/{a,b}.nar | wc -c 7 | -------------------------------------------------------------------------------- /keys/README.md: -------------------------------------------------------------------------------- 1 | 2 | `styx-dev-1`: styx key for local development 3 | `styx-test-1`: styx key for public test environment (on aws) 4 | `testsuite`: key used by test suite 5 | 6 | `styx-nixcache-test-1`: nix cache key for sharing dev/test packages 7 | 8 | -------------------------------------------------------------------------------- /manifester/util.go: -------------------------------------------------------------------------------- 1 | package manifester 2 | 3 | import "io" 4 | 5 | type countWriter struct { 6 | w io.Writer 7 | c int 8 | } 9 | 10 | func (c *countWriter) Write(p []byte) (n int, err error) { 11 | c.c += len(p) 12 | return c.w.Write(p) 13 | } 14 | -------------------------------------------------------------------------------- /bin/testmount: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | base=/mnt/test 4 | upstream=http://cache.nixos.org 5 | 6 | for sph; do 7 | sph=${sph#/nix/store/} 8 | sph=${sph%%/*} 9 | dir=$base/$sph 10 | sudo mkdir -p $dir 11 | sudo ./styx mount $upstream $sph $dir 12 | done 13 | -------------------------------------------------------------------------------- /params/dev-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "manifesterUrl": "http://localhost:7420", 3 | "manifestCacheUrl": "http://localhost:7420", 4 | "chunkReadUrl": "http://localhost:7420", 5 | "chunkDiffUrl": "http://localhost:7420", 6 | "params": { 7 | "chunkShift": 16, 8 | "digestAlgo": "sha256", 9 | "digestBits": 192 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /bin/testvm: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | fstype=ext4 4 | # don't use getopt so we can support -test.* flags 5 | if [[ $1 = -t ]]; then 6 | fstype=$2 7 | shift 2 8 | fi 9 | 10 | set -eux 11 | # Note: test flags need to be in form "-test.run", not just "-run" 12 | nix-build --no-out-link --argstr fstype "$fstype" --argstr testflags "$*" testvm.nix 13 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | { 2 | pkgs ? import { }, 3 | }: 4 | pkgs.mkShell { 5 | buildInputs = with pkgs; [ 6 | awscli2 7 | #brotli.dev # for cbrotli 8 | #erofs-utils 9 | #gcc 10 | go 11 | #gzip 12 | jq 13 | protobuf 14 | protoc-gen-go 15 | skopeo 16 | terraform 17 | #xdelta 18 | #xz 19 | ]; 20 | } 21 | -------------------------------------------------------------------------------- /common/chunksize.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "github.com/dnr/styx/common/shift" 4 | 5 | func DefaultChunkShift(fileSize int64) shift.Shift { 6 | // aim for 64-256 chunks/file 7 | switch { 8 | case fileSize <= 256<<16: // 16 MiB 9 | return 16 10 | case fileSize <= 256<<18: // 64 MiB 11 | return 18 12 | default: 13 | return shift.MaxChunkShift 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /bin/remanifest_missing: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | sudo styx debug --all-images | jq -r ' 4 | .Params.params as {manifest_cache_url: $cacheurl, manifester_url: $requrl} | 5 | .Images | to_entries | .[] | select(.value.Image.mount_state == 2) | 6 | {i: .key, u: .value.Image.upstream} | 7 | @sh "styx internal remanifest --cacheurl \($cacheurl) --requrl \($requrl) --storepath \(.i) --upstream \(.u) --request_if_not" 8 | ' | sh 9 | -------------------------------------------------------------------------------- /tests/bare_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestBare(t *testing.T) { 10 | tb := newTestBase(t) 11 | tb.startAll() 12 | 13 | // bare file package 14 | mp1 := tb.mount("3a7xq2qhxw2r7naqmc53akmx7yvz0mkf-less-is-more.patch") 15 | require.Equal(t, "13jlq14n974nn919530hnx4l46d0p2zyhx4lrd9b1k122dn7w9z5", tb.nixHash(mp1)) 16 | } 17 | -------------------------------------------------------------------------------- /pb/manifest.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package pb; 4 | option go_package = "github.com/dnr/styx/pb"; 5 | 6 | import "entry.proto"; 7 | import "manifest_meta.proto"; 8 | import "params.proto"; 9 | 10 | message Manifest { 11 | GlobalParams params = 1; 12 | repeated Entry entries = 3; 13 | 14 | // build parameters 15 | int32 small_file_cutoff = 2; 16 | 17 | // Metadata on how this was generated 18 | ManifestMeta meta = 10; 19 | } 20 | -------------------------------------------------------------------------------- /params/test-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "manifesterUrl": "https://gfpltjv5viqaqhpsmlbz4atrsm0owuqh.lambda-url.us-east-1.on.aws/", 3 | "chunkDiffUrl": "https://gfpltjv5viqaqhpsmlbz4atrsm0owuqh.lambda-url.us-east-1.on.aws/", 4 | "manifestCacheUrl": "https://styx-1.s3.amazonaws.com/", 5 | "chunkReadUrl": "https://styx-1.s3.amazonaws.com/", 6 | "params": { 7 | "chunkShift": 16, 8 | "digestAlgo": "sha256", 9 | "digestBits": 192 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /erofs/slab.go: -------------------------------------------------------------------------------- 1 | package erofs 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/dnr/styx/common/cdig" 7 | "github.com/dnr/styx/common/shift" 8 | ) 9 | 10 | type ( 11 | SlabLoc struct { 12 | SlabId uint16 13 | Addr uint32 14 | } 15 | 16 | SlabManager interface { 17 | VerifyParams(blockShift shift.Shift) error 18 | AllocateBatch(ctx context.Context, blocks []uint16, digests []cdig.CDig) ([]SlabLoc, error) 19 | SlabInfo(slabId uint16) (tag string, totalBlocks uint32) 20 | } 21 | ) 22 | -------------------------------------------------------------------------------- /common/zstd.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/DataDog/zstd" 7 | ) 8 | 9 | type ZstdCtxPool struct { 10 | p sync.Pool 11 | } 12 | 13 | var globalPool = &ZstdCtxPool{ 14 | p: sync.Pool{New: func() any { return zstd.NewCtx() }}, 15 | } 16 | 17 | func GetZstdCtxPool() *ZstdCtxPool { 18 | return globalPool 19 | } 20 | 21 | func (z *ZstdCtxPool) Get() zstd.Ctx { 22 | return z.p.Get().(zstd.Ctx) 23 | } 24 | 25 | func (z *ZstdCtxPool) Put(c zstd.Ctx) { 26 | z.p.Put(c) 27 | } 28 | -------------------------------------------------------------------------------- /common/blocks.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "github.com/dnr/styx/common/shift" 4 | 5 | func AppendBlocksList(blocks []uint16, size int64, blockShift, chunkShift shift.Shift) []uint16 { 6 | nChunks := chunkShift.Blocks(size) 7 | allButLast := TruncU16(chunkShift.Size() >> blockShift) 8 | for j := 0; j < int(nChunks)-1; j++ { 9 | blocks = append(blocks, allButLast) 10 | } 11 | lastChunkLen := chunkShift.Leftover(size) 12 | blocks = append(blocks, TruncU16(blockShift.Blocks(lastChunkLen))) 13 | return blocks 14 | } 15 | -------------------------------------------------------------------------------- /manifester/tarball_test.go: -------------------------------------------------------------------------------- 1 | package manifester 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestGetSpNameFromUrl(t *testing.T) { 10 | assert.Equal(t, 11 | "nixexprs-nixos-25.11.1056", 12 | getSpNameFromUrl("https://releases.nixos.org/nixos/25.11/nixos-25.11.1056.d9bc5c7dceb3/nixexprs.tar.xz")) 13 | assert.Equal(t, 14 | "nixexprs-nixpkgs-26.05pre910304", 15 | getSpNameFromUrl("https://releases.nixos.org/nixpkgs/nixpkgs-26.05pre910304.f997fa0f94fb/nixexprs.tar.xz")) 16 | } 17 | -------------------------------------------------------------------------------- /common/const.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | var ( 4 | // binary paths (can be overridden by ldflags) 5 | NixBin = "nix" 6 | GzipBin = "gzip" 7 | XzBin = "xz" 8 | FilefragBin = "filefrag" 9 | 10 | // replaced by ldflags 11 | Version = "dev" 12 | ) 13 | 14 | const ( 15 | // Context for signatures 16 | ManifestContext = "styx-manifest-1" 17 | DaemonParamsContext = "styx-daemon-params-1" 18 | ) 19 | 20 | const ( 21 | CTHdr = "Content-Type" 22 | CTJson = "application/json" 23 | CTProto = "application/protobuf" 24 | ) 25 | -------------------------------------------------------------------------------- /pb/extras.go: -------------------------------------------------------------------------------- 1 | package pb 2 | 3 | import "github.com/dnr/styx/common/shift" 4 | 5 | func (e *Entry) FileMode() int { 6 | if e.Executable { 7 | return 0o755 8 | } 9 | return 0o644 10 | } 11 | 12 | func (e *Entry) DigestBytesDef() int { 13 | if e.DigestBytes == 0 { 14 | return 24 15 | } 16 | return int(e.DigestBytes) 17 | } 18 | 19 | func (e *Entry) ChunkShiftDef() shift.Shift { 20 | if e.ChunkShift == 0 { 21 | return shift.DefaultChunkShift 22 | } 23 | return shift.Shift(e.ChunkShift) 24 | } 25 | 26 | func (e *Entry) Chunks() int { 27 | return len(e.Digests) / e.DigestBytesDef() 28 | } 29 | -------------------------------------------------------------------------------- /bin/imagestats: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # use like: 3 | # imagestats /nix/store/j5hf8xgjlpvnlcnx1vskmhxwwwd934hi-my-package 4 | args=() 5 | for i; do args+=(-i "$i"); done 6 | exec 7>&1 7 | sudo styx debug "${args[@]}" | 8 | tee >(jq '.Images.[].Stats' >&7) | 9 | jq -r ' 10 | .Images.[].Manifest.entries.[] | 11 | select(.type == 1 and .size > 224) | 12 | # use this for chunks: 13 | # {path: .path, all: (.size / 65536 | ceil), pres: (.stats_present_chunks // 0)} | 14 | # use this blocks for blocks: 15 | {path: .path, all: (.size / 4096 | ceil), pres: (.stats_present_blocks // 0)} | 16 | "\(.path) \(.pres) / \(.all) = \(.pres/.all)" 17 | ' | sort -s -n -k6 18 | -------------------------------------------------------------------------------- /bin/testdaemon: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eux 3 | 4 | make 5 | 6 | flags="--params=https://styx-1.s3.amazonaws.com/params/test-1 --styx_pubkey=styx-test-1:bmMrKgN5yF3dGgOI67TZSfLts5IQHwdrOCZ7XHcaN+w=" 7 | path=$(dirname $(which modprobe)) 8 | 9 | sudo sh -eux -c " 10 | cat > /run/systemd/system/styx-test.service <> b 27 | } 28 | 29 | func (b Shift) FileChunkSize(totalSize int64, isLast bool) int64 { 30 | if !isLast { 31 | return b.Size() 32 | } 33 | return b.Leftover(totalSize) 34 | } 35 | -------------------------------------------------------------------------------- /vm-base.nix: -------------------------------------------------------------------------------- 1 | { 2 | config, 3 | lib, 4 | pkgs, 5 | ... 6 | }: 7 | { 8 | boot.loader.systemd-boot.enable = true; 9 | boot.loader.efi.canTouchEfiVariables = true; 10 | boot.kernelPackages = pkgs.linuxPackages_latest; 11 | 12 | networking.hostName = "testvm"; 13 | networking.networkmanager.enable = true; 14 | 15 | users.users.test = { 16 | isNormalUser = true; 17 | initialPassword = "test"; 18 | extraGroups = [ "wheel" ]; 19 | }; 20 | 21 | security.sudo.wheelNeedsPassword = false; 22 | 23 | documentation.doc.enable = false; 24 | documentation.info.enable = false; 25 | documentation.man.enable = false; 26 | documentation.nixos.enable = false; 27 | 28 | system.stateVersion = "23.11"; 29 | } 30 | -------------------------------------------------------------------------------- /pb/manifester.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package pb; 4 | option go_package = "github.com/dnr/styx/pb"; 5 | 6 | import "params.proto"; 7 | 8 | // see manifester/proto.go for more details on protocol 9 | message ManifesterChunkDiffReq { 10 | GlobalParams params = 1; // for compatibility check 11 | repeated Req req = 2; 12 | 13 | message Req { 14 | bytes bases = 1; // concatenated digests 15 | bytes reqs = 2; // concatenated digests 16 | 17 | // If set: bases and Reqs each comprise one single file in this compression 18 | // format. Pass each one through this decompressor before diffing. 19 | string expand_before_diff = 3; 20 | } 21 | } 22 | 23 | message Lengths { 24 | repeated int64 length = 1; 25 | } 26 | -------------------------------------------------------------------------------- /charon.orq: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "charon-worker", 4 | "nix-build": "image:-A charon-image", 5 | "stop_old_first": true, 6 | "volumes": [ 7 | { "id": "charon_secrets", "container": "/secrets" } 8 | ], 9 | "argv": [ 10 | "worker", 11 | "--worker", 12 | "--log_axiom", 13 | "--temporal_params=/secrets/temporal-creds-charon.secret", 14 | "--smtp_params=/secrets/smtp-creds-charon.secret", 15 | "--scale_group_name=charon-asg" 16 | ], 17 | "env": { 18 | "AWS_SHARED_CREDENTIALS_FILE": "/secrets/charon-asg-scaler-creds.secret", 19 | "AWS_REGION": "us-east-1", 20 | "AXIOM_DATASET": "styx", 21 | "AXIOM_TOKEN": ^^^CHARON_AXIOM_TOKEN 22 | }, 23 | "host": "dd5" 24 | } 25 | ] 26 | -------------------------------------------------------------------------------- /ci/start.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "go.temporal.io/sdk/client" 8 | ) 9 | 10 | type ( 11 | StartConfig struct { 12 | TemporalParams string 13 | Args CiArgs 14 | } 15 | ) 16 | 17 | func Start(ctx context.Context, cfg StartConfig) error { 18 | c, _, err := getTemporalClient(ctx, cfg.TemporalParams) 19 | if err != nil { 20 | return err 21 | } 22 | 23 | opts := client.StartWorkflowOptions{ 24 | ID: fmt.Sprintf("ci-%s-%s", cfg.Args.StyxRepo.Branch, cfg.Args.Channel), 25 | TaskQueue: taskQueue, 26 | } 27 | run, err := c.ExecuteWorkflow(context.Background(), opts, ci, &cfg.Args) 28 | if err != nil { 29 | return err 30 | } 31 | fmt.Println("workflow id:", run.GetID(), "run id:", run.GetRunID()) 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /common/trunc.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "math" 4 | 5 | func TruncU8[L ~int | ~int16 | ~int32 | ~int64 | ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64](v L) uint8 { 6 | if v < 0 || v > math.MaxUint8 { 7 | panic("overflow") 8 | } 9 | return uint8(v) 10 | } 11 | 12 | func TruncU16[L ~int | ~int32 | ~int64 | ~uint | ~uint16 | ~uint32 | ~uint64](v L) uint16 { 13 | if v < 0 || v > math.MaxUint16 { 14 | panic("overflow") 15 | } 16 | return uint16(v) 17 | } 18 | 19 | func TruncU32[L ~int | ~int64 | ~uint | ~uint32 | ~uint64](v L) uint32 { 20 | if v < 0 || v > math.MaxUint32 { 21 | panic("overflow") 22 | } 23 | return uint32(v) 24 | } 25 | 26 | func TruncU64[L ~int | ~int64 | ~uint | ~uint64](v L) uint64 { 27 | if v < 0 { 28 | panic("overflow") 29 | } 30 | return uint64(v) 31 | } 32 | -------------------------------------------------------------------------------- /common/cdig/cdig_test.go: -------------------------------------------------------------------------------- 1 | package cdig 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestSlice(t *testing.T) { 11 | b := make([]byte, Bytes*4) 12 | for i := range b { 13 | b[i] = byte(i) 14 | } 15 | d0, d1, d2, d3 := FromBytes(b[0:Bytes]), FromBytes(b[Bytes:Bytes*2]), FromBytes(b[Bytes*2:Bytes*3]), FromBytes(b[Bytes*3:]) 16 | require.NotEqual(t, d0, d1) 17 | 18 | // This mostly tests that this aliasing works as expected. 19 | s1 := FromSliceAlias(b) 20 | require.Equal(t, 4, len(s1)) 21 | require.Equal(t, d0, s1[0]) 22 | require.Equal(t, d1, s1[1]) 23 | require.Equal(t, d2, s1[2]) 24 | require.Equal(t, d3, s1[3]) 25 | 26 | s2 := make([]CDig, 4) 27 | copy(s2, s1) 28 | 29 | b2 := ToSliceAlias(s2) 30 | require.True(t, bytes.Equal(b2, b)) 31 | } 32 | -------------------------------------------------------------------------------- /daemon/const.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | const ( 4 | dbFilename = "styx.bolt" 5 | 6 | compactFile = "COMPACT" 7 | 8 | slabPrefix = "_slab_" 9 | slabImagePrefix = "_slabimg_" 10 | manifestSlabPrefix = "_manifests_" 11 | 12 | isManifestPrefix = "M/" 13 | 14 | fakeCacheBind = "localhost:7444" 15 | ) 16 | 17 | var ( 18 | metaBucket = []byte("meta") 19 | chunkBucket = []byte("chunk") 20 | slabBucket = []byte("slab") 21 | imageBucket = []byte("image") 22 | manifestBucket = []byte("manifest") 23 | catalogFBucket = []byte("catalogf") // name + hash -> [sysid] 24 | catalogRBucket = []byte("catalogr") // hash -> name 25 | gcstateBucket = []byte("gcstate") 26 | fakeCacheBucket = []byte("fakecache") 27 | 28 | metaSchema = []byte("schema") 29 | metaParams = []byte("params") 30 | ) 31 | -------------------------------------------------------------------------------- /common/load.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "net/url" 10 | "os" 11 | ) 12 | 13 | func LoadFromFileOrHttpUrl(urlString string) ([]byte, error) { 14 | u, err := url.Parse(urlString) 15 | if err != nil { 16 | return nil, err 17 | } 18 | switch u.Scheme { 19 | case "file": 20 | return os.ReadFile(u.Path) 21 | case "http", "https": 22 | res, err := RetryHttpRequest(context.Background(), http.MethodGet, urlString, "", nil) 23 | if err != nil { 24 | return nil, err 25 | } 26 | defer res.Body.Close() 27 | if res.StatusCode != http.StatusOK { 28 | return nil, fmt.Errorf("http error: %s", res.Status) 29 | } 30 | return io.ReadAll(res.Body) 31 | default: 32 | return nil, errors.New("unknown scheme, must use file or http[s]") 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /common/util.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "strings" 8 | ) 9 | 10 | func ValOrErr[T any](v T, err error) (T, error) { 11 | if err != nil { 12 | var zero T 13 | return zero, err 14 | } 15 | return v, nil 16 | } 17 | 18 | func IsContextError(err error) bool { 19 | return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) 20 | } 21 | 22 | func NormalizeUpstream(u *string) { 23 | if !strings.HasSuffix(*u, "/") { 24 | // upstream should be a url pointing to a directory, so always use trailing-/ form. 25 | // nix drops the / even if it's present in nix.conf, so add it back here. 26 | *u = *u + "/" 27 | } 28 | } 29 | 30 | func ContiguousBytes(in [][]byte) []byte { 31 | if len(in) == 0 { 32 | return nil 33 | } else if len(in) == 1 { 34 | return in[0] // bytes.Join does a copy in this case, otherwise we could just use that 35 | } else { 36 | return bytes.Join(in, nil) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bin/roundtripmount: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eux 4 | 5 | # in runvm, do something like: 6 | # cd /tmp/styxsrc 7 | # systemctl stop styx ; rm -rf /var/cache/styx ; systemctl start styx 8 | # styx manifester --chunklocaldir=/tmp/chunks --styx_signkey=keys/styx-dev-1.secret & 9 | # styx init --params=file://$PWD/params/dev-1.signed --styx_pubkey=styx-dev-1:SCMYzQjLTMMuy/MlovgOX0rRVCYKOj+cYAfQrqzcLu0= 10 | 11 | for a; do 12 | tmp=$(mktemp -p /tmp -d roundtrip.XXXXXXXXX) 13 | j=$(nix path-info $a --json) 14 | if ! echo "$j" | jq -r '.[0].signatures' | grep -q cache.nixos.org; then 15 | echo "$a did not come from cache.nixos.org" 16 | continue 17 | fi 18 | sph=$(echo ${a#/nix/store/} | cut -c-32) 19 | sudo mount -t erofs -o domain_id=styx,fsid=$sph none $tmp 20 | # Comment this out to leave stuff on error 21 | trap "sudo umount $tmp; rmdir $tmp; echo 'ERROR!!!'" EXIT 22 | diff -ur $a $tmp 23 | sudo umount $tmp; rmdir $tmp 24 | trap "" EXIT 25 | done 26 | -------------------------------------------------------------------------------- /cmd/styx/ssm.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | awsconfig "github.com/aws/aws-sdk-go-v2/config" 8 | ssm "github.com/aws/aws-sdk-go-v2/service/ssm" 9 | "github.com/nix-community/go-nix/pkg/narinfo/signature" 10 | ) 11 | 12 | func loadKeysFromSsm(names []string) ([]signature.SecretKey, error) { 13 | awscfg, err := awsconfig.LoadDefaultConfig(context.Background(), awsconfig.WithEC2IMDSRegion()) 14 | if err != nil { 15 | return nil, err 16 | } 17 | ssmcli := ssm.NewFromConfig(awscfg) 18 | decrypt := true 19 | keys := make([]signature.SecretKey, len(names)) 20 | for i, name := range names { 21 | if out, err := ssmcli.GetParameter(context.Background(), &ssm.GetParameterInput{ 22 | Name: &name, 23 | WithDecryption: &decrypt, 24 | }); err != nil { 25 | return nil, err 26 | } else if keys[i], err = signature.LoadSecretKey(strings.TrimSpace(*out.Parameter.Value)); err != nil { 27 | return nil, err 28 | } 29 | } 30 | return keys, nil 31 | } 32 | -------------------------------------------------------------------------------- /tests/variable_chunk_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/dnr/styx/common/shift" 9 | ) 10 | 11 | func TestVariableChunk(t *testing.T) { 12 | tb := newTestBase(t) 13 | // override this to force different chunk sizes 14 | tb.chunkSizer = func(size int64) shift.Shift { 15 | switch { 16 | case size <= 25<<12: // 100 KiB 17 | return 12 18 | case size <= 50<<13: // 400 KiB 19 | return 13 20 | case size <= 50<<17: // 6.4 MiB 21 | return 17 22 | default: 23 | return 20 24 | } 25 | } 26 | tb.startAll() 27 | 28 | // 12 files in the 400-6.4mb bucket, 10 in 100-400, rest < 100 29 | mp1 := tb.mount("kbi7qf642gsxiv51yqank8bnx39w3crd-calf-0.90.3") 30 | require.Equal(t, "1bhyfn2k8w41cx7ddarmjmwscas0946n6gw5mralx9lg0vbbcx6d", tb.nixHash(mp1)) 31 | 32 | // 1 file > 6.4mb 33 | mp2 := tb.mount("d30xd6x3669hg2a6xwjb1r3nb9a99sw2-openblas-0.3.27") 34 | require.Equal(t, "158wdip28fcfvw911s1wx8h132ljzvzqa3m5zgjj4vn0hby9kzzn", tb.nixHash(mp2)) 35 | } 36 | -------------------------------------------------------------------------------- /vm-testsuite.nix: -------------------------------------------------------------------------------- 1 | { 2 | config, 3 | lib, 4 | pkgs, 5 | fstype, 6 | ... 7 | }: 8 | let 9 | styxtest = (import ./. { inherit pkgs; }).styx-test; 10 | runstyxtest = pkgs.writeShellScriptBin "runstyxtest" '' 11 | cd ${styxtest}/bin 12 | if [[ $UID != 0 ]]; then sudo=sudo; fi 13 | $sudo modprobe cachefiles 14 | exec $sudo ./styxtest -test.v "$@" 15 | ''; 16 | in 17 | { 18 | imports = [ 19 | ./vm-base.nix 20 | ./module 21 | ]; 22 | 23 | # test suite needs only kernel options 24 | services.styx.enableKernelOptions = true; 25 | 26 | # set up configurable fs type 27 | assertions = [ 28 | { 29 | assertion = config.virtualisation.diskImage != null; 30 | message = "must use disk image"; 31 | } 32 | ]; 33 | # set fstype of root fs 34 | virtualisation.fileSystems."/".fsType = lib.mkForce fstype; 35 | # ensure btrfs enabled 36 | system.requiredKernelConfig = with config.lib.kernelConfig; [ (isEnabled "BTRFS_FS") ]; 37 | 38 | environment.systemPackages = with pkgs; [ 39 | psmisc # for fuser 40 | runstyxtest 41 | ]; 42 | } 43 | -------------------------------------------------------------------------------- /daemon/context.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import "context" 4 | 5 | type ( 6 | allocateContext struct { 7 | sph Sph 8 | forManifest bool 9 | } 10 | 11 | mountContext struct { 12 | imageSize int64 13 | isBare bool 14 | imageData []byte 15 | } 16 | 17 | daemonCtxKey int 18 | ) 19 | 20 | var ( 21 | allocateCtxKey any = daemonCtxKey(1) 22 | mountCtxKey any = daemonCtxKey(2) 23 | ) 24 | 25 | func withAllocateCtx(ctx context.Context, sph Sph, forManifest bool) context.Context { 26 | return context.WithValue(ctx, allocateCtxKey, allocateContext{sph: sph, forManifest: forManifest}) 27 | } 28 | 29 | func fromAllocateCtx(ctx context.Context) (Sph, bool, bool) { 30 | actx, ok := ctx.Value(allocateCtxKey).(allocateContext) 31 | return actx.sph, actx.forManifest, ok 32 | } 33 | 34 | func withMountContext(ctx context.Context, mctx *mountContext) context.Context { 35 | return context.WithValue(ctx, mountCtxKey, mctx) 36 | } 37 | 38 | func fromMountCtx(ctx context.Context) (*mountContext, bool) { 39 | mctx, ok := ctx.Value(mountCtxKey).(*mountContext) 40 | return mctx, ok 41 | } 42 | -------------------------------------------------------------------------------- /tests/repeated_read_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/dnr/styx/common/shift" 10 | "github.com/dnr/styx/daemon" 11 | ) 12 | 13 | func TestRepeatedRead(t *testing.T) { 14 | tb := newTestBase(t) 15 | // override this so our calculation works out 16 | tb.chunkSizer = func(int64) shift.Shift { return 16 } 17 | tb.startAll() 18 | 19 | mp1 := tb.mount("d30xd6x3669hg2a6xwjb1r3nb9a99sw2-openblas-0.3.27") 20 | bigFile := filepath.Join(mp1, "lib/libopenblasp-r0.3.27.so") 21 | require.Equal(t, "0q7zclw8sxfq5mvx0lf3clmqw31z9biq4adihcwh2hk6f39lia3w", tb.nixHash(bigFile)) 22 | 23 | // reimplement some logic to figure out how many extra reads are expected (currently 1) 24 | size := 27720680 25 | extra := 0 26 | reqSize := daemon.InitOpSize 27 | for size > 0 { 28 | size -= reqSize << 16 29 | extra += (reqSize - 1) / daemon.MaxOpSize 30 | reqSize = min(reqSize*2, daemon.MaxDiffOps*daemon.MaxOpSize) 31 | } 32 | require.Positive(t, extra, "file isn't big enough to have extra reads with current params") 33 | 34 | d1 := tb.debug() 35 | require.EqualValues(t, extra, d1.Stats.ExtraReqs) 36 | } 37 | -------------------------------------------------------------------------------- /common/error.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | ) 9 | 10 | type NotFoundable interface { 11 | IsNotFound() bool 12 | } 13 | 14 | func IsNotFound(err error) bool { 15 | var nf NotFoundable 16 | return errors.As(err, &nf) && nf.IsNotFound() 17 | } 18 | 19 | type HttpError struct { 20 | code int 21 | body string 22 | } 23 | 24 | var _ NotFoundable = (*HttpError)(nil) 25 | 26 | // note: this does not close res.Body, caller should close it 27 | func HttpErrorFromRes(res *http.Response) HttpError { 28 | body, _ := io.ReadAll(io.LimitReader(res.Body, 1024)) 29 | io.Copy(io.Discard, res.Body) 30 | return NewHttpError(res.StatusCode, string(body)) 31 | } 32 | 33 | func NewHttpError(code int, body string) HttpError { return HttpError{code: code, body: body} } 34 | 35 | func (e HttpError) Error() string { 36 | if len(e.body) == 0 { 37 | return fmt.Sprintf("http status %d", e.code) 38 | } 39 | return fmt.Sprintf("http status %d: %q", e.code, e.body) 40 | } 41 | func (e HttpError) Code() int { return e.code } 42 | func (e HttpError) Body() string { return e.body } 43 | func (e HttpError) IsNotFound() bool { 44 | return e.code == http.StatusNotFound || e.code == http.StatusForbidden 45 | } 46 | -------------------------------------------------------------------------------- /tests/small_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/dnr/styx/daemon" 10 | ) 11 | 12 | func TestSmallImage(t *testing.T) { 13 | tb := newTestBase(t) 14 | tb.startAll() 15 | 16 | // 144K package 17 | mp1 := tb.mount("qa22bifihaxyvn6q2a6w9m0nklqrk9wh-opusfile-0.12") 18 | d1 := tb.debug(daemon.DebugReq{IncludeSlabs: true}) 19 | require.Zero(t, d1.Slabs[0].Stats.PresentChunks) 20 | require.Zero(t, d1.Slabs[0].Stats.PresentBlocks) 21 | 22 | require.Equal(t, "1rswindywkyq2jmfpxd6n772jii3z5xz6ypfbb63c17k5il39hfm", tb.nixHash(mp1)) 23 | time.Sleep(200 * time.Millisecond) // batch delay 24 | d2 := tb.debug(daemon.DebugReq{IncludeSlabs: true}) 25 | require.NotZero(t, d2.Slabs[0].Stats.PresentChunks) 26 | require.NotZero(t, d2.Slabs[0].Stats.PresentBlocks) 27 | 28 | tb.dropCaches() 29 | 30 | require.Equal(t, "1rswindywkyq2jmfpxd6n772jii3z5xz6ypfbb63c17k5il39hfm", tb.nixHash(mp1)) 31 | d3 := tb.debug(daemon.DebugReq{IncludeSlabs: true}) 32 | require.Equal(t, d2.Stats.SlabReads, d3.Stats.SlabReads) 33 | require.Zero(t, d3.Stats.SlabReadErrs) 34 | 35 | // try explicit unmount 36 | tb.umount("qa22bifihaxyvn6q2a6w9m0nklqrk9wh-opusfile-0.12") 37 | } 38 | -------------------------------------------------------------------------------- /testvm.nix: -------------------------------------------------------------------------------- 1 | { 2 | hostPkgs ? import { 3 | config = { }; 4 | overlays = [ ]; 5 | }, 6 | testflags ? "", 7 | fstype ? "ext4", 8 | }: 9 | hostPkgs.testers.runNixOSTest ( 10 | { config, ... }: 11 | { 12 | name = "styxvmtest"; 13 | defaults._module.args = { inherit fstype; }; 14 | nodes.machine = ./vm-testsuite.nix; 15 | extraDriverArgs = 16 | let 17 | m = config.nodes.machine; 18 | origScript = "${m.system.build.vm}/bin/run-${m.networking.hostName}-vm"; 19 | mkfs = 20 | if fstype == "ext4" then 21 | "${hostPkgs.e2fsprogs}/bin/mkfs.ext4" 22 | else if fstype == "btrfs" then 23 | "${hostPkgs.btrfs-progs}/bin/mkfs.btrfs" 24 | else 25 | throw "unknown fs type"; 26 | newScript = hostPkgs.runCommand "testvm-start-script" { } '' 27 | sed -e ' 28 | s|/nix/store/[^ /]*/bin/mkfs[.]ext4|${mkfs}| 29 | s|,mount_tag=nix-store|&,multidevs=remap| 30 | ' < ${origScript} > $out 31 | chmod a+x $out 32 | ''; 33 | in 34 | [ "--start-scripts ${newScript}" ]; 35 | testScript = '' 36 | machine.wait_for_unit("default.target") 37 | machine.succeed("runstyxtest ${testflags}") 38 | ''; 39 | } 40 | ) 41 | -------------------------------------------------------------------------------- /pb/params.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package pb; 4 | option go_package = "github.com/dnr/styx/pb"; 5 | 6 | import "entry.proto"; 7 | 8 | // Parameters that have to be agreed on by manifester and daemon. 9 | message GlobalParams { 10 | // Default chunk shift (if not specified in entry) is always 16. 11 | reserved 1; 12 | // Digest algorithm, e.g. "sha256" 13 | string digest_algo = 2; 14 | // Bits of digest used, e.g. 192 15 | int32 digest_bits = 3; 16 | } 17 | 18 | // Parameters that can be used to configure a styx daemon. 19 | message DaemonParams { 20 | GlobalParams params = 1; 21 | // URL for manifester service, chunk reads, and chunk diffs. 22 | string manifester_url = 2; 23 | string manifest_cache_url = 5; 24 | string chunk_read_url = 3; 25 | string chunk_diff_url = 4; 26 | 27 | // Size to shard manifest. If missing, daemon uses a default. 28 | int64 shard_manifest_bytes = 6; 29 | } 30 | 31 | message SignedMessage { 32 | // Params for hashing/chunking contained data, and also used for signature. 33 | GlobalParams params = 1; 34 | // Single entry representing contained data. 35 | // Type must be REGULAR. 36 | // Path should represent message type and context. 37 | Entry msg = 2; 38 | 39 | // These should be the same length: 40 | repeated string key_id = 3; 41 | repeated bytes signature = 4; 42 | } 43 | -------------------------------------------------------------------------------- /common/chunkpool.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "sync" 4 | 5 | type ChunkPool struct { 6 | // if shift.MaxChunkShift > 20, add more here: 7 | p12, p14, p16, p18, p20 sync.Pool 8 | } 9 | 10 | func NewChunkPool() *ChunkPool { 11 | return &ChunkPool{ 12 | p12: sync.Pool{New: func() any { return make([]byte, 1<<12) }}, 13 | p14: sync.Pool{New: func() any { return make([]byte, 1<<14) }}, 14 | p16: sync.Pool{New: func() any { return make([]byte, 1<<16) }}, 15 | p18: sync.Pool{New: func() any { return make([]byte, 1<<18) }}, 16 | p20: sync.Pool{New: func() any { return make([]byte, 1<<20) }}, 17 | } 18 | } 19 | 20 | func (cp *ChunkPool) Get(size int) []byte { 21 | switch { 22 | case size <= 1<<12: 23 | return cp.p12.Get().([]byte) 24 | case size <= 1<<14: 25 | return cp.p14.Get().([]byte) 26 | case size <= 1<<16: 27 | return cp.p16.Get().([]byte) 28 | case size <= 1<<18: 29 | return cp.p18.Get().([]byte) 30 | case size <= 1<<20: 31 | return cp.p20.Get().([]byte) 32 | default: 33 | return make([]byte, size) 34 | } 35 | } 36 | 37 | func (cp *ChunkPool) Put(b []byte) { 38 | size := cap(b) 39 | switch { 40 | case size <= 1<<12: 41 | cp.p12.Put(b) 42 | case size <= 1<<14: 43 | cp.p14.Put(b) 44 | case size <= 1<<16: 45 | cp.p16.Put(b) 46 | case size <= 1<<18: 47 | cp.p18.Put(b) 48 | case size <= 1<<20: 49 | cp.p20.Put(b) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /pb/entry.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package pb; 4 | option go_package = "github.com/dnr/styx/pb"; 5 | 6 | import "manifest_meta.proto"; 7 | 8 | enum EntryType { 9 | UNKNOWN = 0; 10 | REGULAR = 1; 11 | DIRECTORY = 2; 12 | SYMLINK = 3; 13 | } 14 | 15 | // Mostly modeled on nar Header 16 | message Entry { 17 | string path = 1; // Path of the entry, relative inside the NAR 18 | EntryType type = 2; // File type (regular/directory/symlink) 19 | int64 size = 3; // Logical file size in bytes (only for REGULAR) 20 | bool executable = 4; // Set to true for files that are executable 21 | // For REGULAR and SYMLINK: 22 | // If size <= small_file_cutoff || type == SYMLINK, full data is here: 23 | bytes inline_data = 5; 24 | // Otherwise, this is a series of concatenated digests, one per chunk: 25 | bytes digests = 6; 26 | int32 digest_bytes = 7; // Number of bytes per digest (default: 24) 27 | int32 chunk_shift = 8; // Chunk size = 1 << chunk_shift (default: 16) 28 | // Debug data (only in debug output, not on network or db) 29 | int32 stats_inline_data = 100; 30 | int32 stats_present_chunks = 101; 31 | int32 stats_present_blocks = 102; 32 | repeated string debug_digests = 103; 33 | // Copy of manfest meta for chunked manifests so we can get it without indirection 34 | ManifestMeta manifest_meta = 200; 35 | } 36 | -------------------------------------------------------------------------------- /tests/remanifest_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestRemanifestOnNotFound(t *testing.T) { 12 | tb := newTestBase(t) 13 | tb.startAll() 14 | 15 | mp1 := tb.mount("qa22bifihaxyvn6q2a6w9m0nklqrk9wh-opusfile-0.12") 16 | 17 | // wipe chunk store 18 | ents, err := os.ReadDir(tb.chunkdir) 19 | require.NoError(t, err) 20 | for _, ent := range ents { 21 | os.Remove(filepath.Join(tb.chunkdir, ent.Name())) 22 | } 23 | 24 | // should succeed anyway 25 | require.Equal(t, "1rswindywkyq2jmfpxd6n772jii3z5xz6ypfbb63c17k5il39hfm", tb.nixHash(mp1)) 26 | } 27 | 28 | func TestRemanifestPrefetch(t *testing.T) { 29 | tb := newTestBase(t) 30 | tb.startAll() 31 | 32 | // mount to manifest and get chunks in cache, but not locally 33 | tb.mount("qa22bifihaxyvn6q2a6w9m0nklqrk9wh-opusfile-0.12") 34 | tb.umount("qa22bifihaxyvn6q2a6w9m0nklqrk9wh-opusfile-0.12") 35 | 36 | // wipe chunk store 37 | ents, err := os.ReadDir(tb.chunkdir) 38 | require.NoError(t, err) 39 | for _, ent := range ents { 40 | os.Remove(filepath.Join(tb.chunkdir, ent.Name())) 41 | } 42 | 43 | // should succeed anyway 44 | mp1 := tb.materialize("qa22bifihaxyvn6q2a6w9m0nklqrk9wh-opusfile-0.12") 45 | require.Equal(t, "1rswindywkyq2jmfpxd6n772jii3z5xz6ypfbb63c17k5il39hfm", tb.nixHash(mp1)) 46 | } 47 | -------------------------------------------------------------------------------- /tests/tarball_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "net/http" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/dnr/styx/common/client" 9 | "github.com/dnr/styx/daemon" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestTarball(t *testing.T) { 15 | tb := newTestBase(t) 16 | tb.startAll() 17 | 18 | tbRes := tb.tarball(tb.upstreamUrl + "tarballs/nix-1.0.tar.gz") 19 | assert.Equal(t, tbRes.Name, "nix-1.0.tar.gz") 20 | assert.Equal(t, tbRes.StorePathHash, "wfqgnyhizqjf38xdp5d29fdi0w6sws1g") 21 | assert.Equal(t, tbRes.NarHash, "b274771c9a0e4ed2f99de20ac3152654dba12183de2326729d02546dd0d50095") 22 | 23 | // materialize manually since we need a special "upstream" 24 | mp := filepath.Join(t.TempDir(), "mp") 25 | c := client.NewClient(filepath.Join(tb.cachedir, "styx.sock")) 26 | var res daemon.Status 27 | code, err := c.Call(daemon.MaterializePath, daemon.MaterializeReq{ 28 | Upstream: "http://localhost:7444", // daemon.fakeCacheBind 29 | StorePath: tbRes.StorePathHash + "-" + tbRes.Name, 30 | DestPath: mp, 31 | }, &res) 32 | require.NoError(t, err) 33 | require.Equal(t, code, http.StatusOK) 34 | require.True(t, res.Success, "error:", res.Error) 35 | // note same as nar hash above, in base32 36 | assert.Equal(t, "1580sp86sm02kmr2c8yyhchs3nsl4qaw62p2kpwx4khfk8f7fx5j", tb.nixHash(mp)) 37 | } 38 | -------------------------------------------------------------------------------- /pb/narinfo.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package pb; 4 | option go_package = "github.com/dnr/styx/pb"; 5 | 6 | // from github.com/nix-community/go-nix pkg/narinfo/types.go 7 | message NarInfo { 8 | string store_path = 1; // The full nix store path (/nix/store/…-pname-version) 9 | string url = 2; // The relative location to the .nar[.xz,…] file. Usually nar/$fileHash.nar[.xz] 10 | string compression = 3; // The compression method file at URL is compressed with (none,xz,…) 11 | string file_hash = 4; // The hash of the file at URL (nix string format) 12 | int64 file_size = 5; // The size of the file at URL, in bytes 13 | // The hash of the .nar file, after possible decompression (nix string format) 14 | // Identical to FileHash if no compression is used. 15 | string nar_hash = 6; 16 | // The size of the .nar file, after possible decompression, in bytes. 17 | // Identical to FileSize if no compression is used. 18 | int64 nar_size = 7; 19 | repeated string references = 8; // References to other store paths, contained in the .nar file 20 | string deriver = 9; // Path of the .drv for this store path 21 | string system = 10; // This doesn't seem to be used at all? 22 | repeated string signatures = 11; // Signatures, if any. 23 | string ca = 12; 24 | } 25 | -------------------------------------------------------------------------------- /tests/diff_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestDiffChunks(t *testing.T) { 10 | tb := newTestBase(t) 11 | tb.startAll() 12 | 13 | // these are very similar 144K packages so should diff well 14 | mp1 := tb.mount("qa22bifihaxyvn6q2a6w9m0nklqrk9wh-opusfile-0.12") 15 | require.Equal(t, "1rswindywkyq2jmfpxd6n772jii3z5xz6ypfbb63c17k5il39hfm", tb.nixHash(mp1)) 16 | d1 := tb.debug() 17 | require.Zero(t, d1.Stats.SingleReqs) 18 | require.NotZero(t, d1.Stats.BatchReqs) 19 | require.Zero(t, d1.Stats.DiffReqs) 20 | 21 | mp2 := tb.mount("kcyrz2y8si9ry5p8qkmj0gp41n01sa1y-opusfile-0.12") 22 | require.Equal(t, "0im7spp48afrbfv672bmrvrs0lg4md0qhyic8zkcgyc8xqwz1s5b", tb.nixHash(mp2)) 23 | d2 := tb.debug() 24 | require.Zero(t, d2.Stats.SingleReqs-d1.Stats.SingleReqs) 25 | require.Zero(t, d2.Stats.BatchReqs-d1.Stats.BatchReqs) 26 | require.NotZero(t, d2.Stats.DiffReqs-d1.Stats.DiffReqs) 27 | 28 | mp3 := tb.mount("53qwclnym7a6vzs937jjmsfqxlxlsf2y-opusfile-0.12") 29 | require.Equal(t, "0dm2277wfknq81wfwzxrasc9rif30fm03vxahndbqnn4gb9swqpq", tb.nixHash(mp3)) 30 | d3 := tb.debug() 31 | require.Zero(t, d3.Stats.SingleReqs-d2.Stats.SingleReqs) 32 | require.Zero(t, d3.Stats.BatchReqs-d2.Stats.BatchReqs) 33 | require.NotZero(t, d3.Stats.DiffReqs-d2.Stats.DiffReqs) 34 | 35 | require.Zero(t, d3.Stats.SingleErrs+d3.Stats.BatchErrs+d3.Stats.DiffErrs) 36 | } 37 | -------------------------------------------------------------------------------- /common/client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "net" 9 | "net/http" 10 | "net/url" 11 | "os" 12 | 13 | "github.com/dnr/styx/common" 14 | ) 15 | 16 | // simple client for json requests/responses over http over unix socket 17 | type StyxClient struct { 18 | cli *http.Client 19 | } 20 | 21 | func NewClient(addr string) *StyxClient { 22 | cli := &http.Client{ 23 | Transport: &http.Transport{ 24 | DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { 25 | var dialer net.Dialer 26 | return dialer.DialContext(ctx, "unix", addr) 27 | }, 28 | }, 29 | } 30 | return &StyxClient{cli: cli} 31 | } 32 | 33 | func (c *StyxClient) Call(path string, req, res any) (int, error) { 34 | u := &url.URL{ 35 | Scheme: "http", 36 | Host: "_", 37 | Path: path, 38 | } 39 | buf, err := json.Marshal(req) 40 | if err != nil { 41 | return 0, err 42 | } 43 | httpRes, err := c.cli.Post(u.String(), common.CTJson, bytes.NewReader(buf)) 44 | if err != nil { 45 | return 0, err 46 | } 47 | defer httpRes.Body.Close() 48 | return httpRes.StatusCode, json.NewDecoder(httpRes.Body).Decode(res) 49 | } 50 | 51 | func (c *StyxClient) CallAndPrint(path string, req any) error { 52 | var res any 53 | status, err := c.Call(path, req, &res) 54 | if err != nil { 55 | fmt.Println("call error:", err) 56 | return err 57 | } 58 | if status != http.StatusOK { 59 | fmt.Println("status:", status) 60 | } 61 | return json.NewEncoder(os.Stdout).Encode(res) 62 | } 63 | -------------------------------------------------------------------------------- /common/http.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "log" 8 | "net/http" 9 | "time" 10 | 11 | "github.com/avast/retry-go/v4" 12 | ) 13 | 14 | func RetryHttpRequest(ctx context.Context, method, url, cType string, body []byte) (*http.Response, error) { 15 | return retry.DoWithData( 16 | func() (*http.Response, error) { 17 | var bReader io.Reader 18 | if body != nil { 19 | bReader = bytes.NewReader(body) 20 | } 21 | req, err := http.NewRequestWithContext(ctx, method, url, bReader) 22 | if err != nil { 23 | return nil, retry.Unrecoverable(err) 24 | } 25 | if cType != "" { 26 | req.Header.Set("Content-Type", cType) 27 | } 28 | res, err := http.DefaultClient.Do(req) 29 | if err == nil && res.StatusCode != http.StatusOK { 30 | err = HttpErrorFromRes(res) 31 | res.Body.Close() 32 | } 33 | return ValOrErr(res, err) 34 | }, 35 | retry.Context(ctx), 36 | retry.UntilSucceeded(), 37 | retry.Delay(time.Second), 38 | retry.RetryIf(func(err error) bool { 39 | // retry on err or some 50x codes 40 | if status, ok := err.(HttpError); ok { 41 | switch status.Code() { 42 | case http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: 43 | return true 44 | default: 45 | return false 46 | } 47 | } else if IsContextError(err) { 48 | return false 49 | } 50 | return true 51 | }), 52 | retry.OnRetry(func(n uint, err error) { 53 | log.Printf("http error (%d): %v, retrying", n, err) 54 | })) 55 | } 56 | -------------------------------------------------------------------------------- /bin/deploy-styx: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euxo pipefail 3 | cd "$(dirname $(dirname "$0"))" 4 | 5 | tfget() { 6 | terraform -chdir=tf show -json | 7 | jq -r ".values.root_module.resources[] | select(.address == \"$1\") | .values.$2" 8 | } 9 | 10 | # manifester on lambda: 11 | repourl=$(tfget aws_ecr_repository.repo repository_url) 12 | skopeo --insecure-policy list-tags docker://$repourl >&/dev/null || 13 | aws ecr get-login-password | skopeo login --username AWS --password-stdin ${repourl%%/*} 14 | 15 | img=$(nix-build --no-out-link -A styx-lambda-image) 16 | tag=$(basename $img | cut -c-12) 17 | 18 | skopeo --insecure-policy copy \ 19 | docker-archive:$img \ 20 | docker://$repourl:$tag 21 | 22 | export TF_VAR_manifester_image_tag="$tag" 23 | 24 | # heavy worker: 25 | store="s3://styx-1/nixcache/?region=us-east-1&compression=zstd" 26 | charon=$(nix-build --no-out-link -A charon) 27 | 28 | nix store sign -v --key-file keys/styx-nixcache-test-1.secret $(nix-store -qR "$charon") 29 | nix copy -v --to "$store" "$charon" 30 | 31 | # write buildroot so this doesn't get collected 32 | { 33 | echo "meta { build_time: $(date +%s) charon_build: \"$(basename $charon)\" }" 34 | nix-store -qR "$charon" | sed 's,.*/\([^-]*\)-.*,store_path_hash: "\1",' 35 | } | 36 | protoc --encode=pb.BuildRoot -Ipb pb/buildroot.proto | 37 | zstd -c | 38 | aws s3 cp - "s3://styx-1/buildroot/charon@$(date -u +%FT%TZ)@c@c" --content-encoding zstd 39 | 40 | export TF_VAR_charon_storepath="$charon" 41 | 42 | terraform -chdir=tf apply 43 | 44 | # light worker: 45 | orq run 46 | -------------------------------------------------------------------------------- /tests/materialize_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestMaterialize(t *testing.T) { 11 | tb := newTestBase(t) 12 | tb.startAll() 13 | 14 | mp1 := tb.materialize("qa22bifihaxyvn6q2a6w9m0nklqrk9wh-opusfile-0.12") 15 | require.Equal(t, "1rswindywkyq2jmfpxd6n772jii3z5xz6ypfbb63c17k5il39hfm", tb.nixHash(mp1)) 16 | // TODO: check btrfs fi du to check that extents are shared 17 | 18 | mp2 := tb.materialize("kbi7qf642gsxiv51yqank8bnx39w3crd-calf-0.90.3") 19 | require.Equal(t, "1bhyfn2k8w41cx7ddarmjmwscas0946n6gw5mralx9lg0vbbcx6d", tb.nixHash(mp2)) 20 | 21 | mp3 := tb.materialize("v35ysx9k1ln4c6r7lj74204ss4bw7l5l-openssl-3.0.12-man") 22 | require.Equal(t, "0v60mg7qj7mfd27s1nnldb0041ln08xs1bw7zn1mmjiaq02myzlh", tb.nixHash(mp3)) 23 | 24 | mp4 := tb.materialize("3a7xq2qhxw2r7naqmc53akmx7yvz0mkf-less-is-more.patch") 25 | require.Equal(t, "13jlq14n974nn919530hnx4l46d0p2zyhx4lrd9b1k122dn7w9z5", tb.nixHash(mp4)) 26 | 27 | // for this one, mount then materialize, should use local manifest. 28 | // this isn't a great test since materialize will fall back to remote manifest if error 29 | // reading local, it'd be nice to disable that for this test. 30 | _ = tb.mount("cd1nbildgzzfryjg82njnn36i4ynyf8h-bash-interactive-5.1-p16-man") 31 | mp5 := tb.materialize("cd1nbildgzzfryjg82njnn36i4ynyf8h-bash-interactive-5.1-p16-man") 32 | man1 := filepath.Join(mp5, "share", "man", "man1", "bash.1.gz") 33 | require.Equal(t, "0s9d681f8smlsdvbp6lin9qrbsp3hz3dnf4pdhwi883v8l1486r7", tb.nixHash(man1)) 34 | } 35 | -------------------------------------------------------------------------------- /common/cdig/cdig.go: -------------------------------------------------------------------------------- 1 | package cdig 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/base64" 6 | "errors" 7 | "fmt" 8 | "unsafe" 9 | ) 10 | 11 | const ( 12 | Bytes = 24 13 | Bits = Bytes << 3 14 | Algo = "sha256" 15 | ) 16 | 17 | type ( 18 | CDig [Bytes]byte 19 | ) 20 | 21 | var Zero CDig 22 | 23 | var ErrInvalid = errors.New("invalid base64 digest") 24 | 25 | func Sum(b []byte) CDig { 26 | h := sha256.New() 27 | h.Write(b) 28 | var full [sha256.Size]byte 29 | return FromBytes(h.Sum(full[:0])) 30 | } 31 | 32 | func (dig CDig) String() string { 33 | return base64.RawURLEncoding.EncodeToString(dig[:]) 34 | } 35 | 36 | func (dig CDig) Check(b []byte) error { 37 | if got := Sum(b); got != dig { 38 | return fmt.Errorf("chunk digest mismatch %x != %x", got, dig) 39 | } 40 | return nil 41 | } 42 | 43 | // Note len(b) should be at least Bytes. 44 | func FromBytes(b []byte) (dig CDig) { 45 | copy(dig[:], b[:Bytes]) 46 | return 47 | } 48 | 49 | // Views a byte CDig slice as a CDig slice. This aliases memory! Be careful. 50 | func FromSliceAlias(b []byte) []CDig { 51 | p := unsafe.SliceData(b) 52 | dp := (*CDig)(unsafe.Pointer(p)) 53 | return unsafe.Slice(dp, len(b)/Bytes) 54 | } 55 | 56 | // Views a CDig slice as a byte slice. This aliases memory! Be careful. 57 | func ToSliceAlias(digests []CDig) []byte { 58 | p := unsafe.SliceData(digests) 59 | bp := (*byte)(unsafe.Pointer(p)) 60 | return unsafe.Slice(bp, len(digests)*Bytes) 61 | } 62 | 63 | func FromBase64(s string) (dig CDig, err error) { 64 | src := unsafe.Slice(unsafe.StringData(s), len(s)) 65 | var dst []byte 66 | dst, err = base64.RawURLEncoding.AppendDecode(dig[:0:Bytes], src) 67 | if err == nil && (&dst[0] != &dig[0] || len(dst) != Bytes) { 68 | err = ErrInvalid 69 | } 70 | return 71 | } 72 | -------------------------------------------------------------------------------- /cmd/charon/cobrautil.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "reflect" 7 | 8 | "github.com/spf13/cobra" 9 | "golang.org/x/exp/slices" 10 | ) 11 | 12 | type runE = func(*cobra.Command, []string) error 13 | 14 | func chainRunE(fs ...runE) runE { 15 | fs = slices.DeleteFunc(fs, func(e runE) bool { return e == nil }) 16 | if len(fs) == 1 { 17 | return fs[0] 18 | } 19 | return func(c *cobra.Command, args []string) error { 20 | for _, f := range fs { 21 | if err := f(c, args); err != nil { 22 | return err 23 | } 24 | } 25 | return nil 26 | } 27 | } 28 | 29 | func cmd(c *cobra.Command, stuff ...any) *cobra.Command { 30 | for _, thing := range stuff { 31 | switch t := thing.(type) { 32 | case func(*cobra.Command): 33 | t(c) 34 | case *cobra.Command: 35 | c.AddCommand(t) 36 | case runE: 37 | c.RunE = chainRunE(c.RunE, t) 38 | case func(*cobra.Command) runE: 39 | c.RunE = chainRunE(c.RunE, t(c)) 40 | default: 41 | log.Panicf("bad cmd structure: %T %v", t, t) 42 | } 43 | } 44 | return c 45 | } 46 | 47 | type ckey struct { 48 | t reflect.Type 49 | k any 50 | } 51 | 52 | func store[T any](c *cobra.Command, v T) { 53 | t := reflect.TypeOf((*T)(nil)).Elem() 54 | c.SetContext(context.WithValue(c.Context(), ckey{t: t}, v)) 55 | } 56 | 57 | func get[T any](c *cobra.Command) T { 58 | t := reflect.TypeOf((*T)(nil)).Elem() 59 | return c.Context().Value(ckey{t: t}).(T) 60 | } 61 | 62 | func storeKeyed[T any](c *cobra.Command, v T, key any) { 63 | t := reflect.TypeOf((*T)(nil)).Elem() 64 | c.SetContext(context.WithValue(c.Context(), ckey{t: t, k: key}, v)) 65 | } 66 | 67 | func getKeyed[T any](c *cobra.Command, key any) T { 68 | t := reflect.TypeOf((*T)(nil)).Elem() 69 | return c.Context().Value(ckey{t: t, k: key}).(T) 70 | } 71 | -------------------------------------------------------------------------------- /tf/charon-worker-ud.nix: -------------------------------------------------------------------------------- 1 | # This file is copied to /etc/nixos/configuration.nix and the config is built and activated. 2 | # Note this is templated by terraform, $ {} is TF, $${} is Nix. 3 | { 4 | config, 5 | pkgs, 6 | utils, 7 | ... 8 | }: 9 | { 10 | imports = [ ]; 11 | 12 | nix.settings.substituters = [ "${sub}" ]; 13 | nix.settings.trusted-public-keys = [ "${pubkey}" ]; 14 | 15 | systemd.services.charon = { 16 | description = "charon ci worker"; 17 | wantedBy = [ "multi-user.target" ]; 18 | serviceConfig = { 19 | # TODO: this works but it feels weird. should it use builtins.storePath? 20 | # but how do we set the cache and trusted key in that case? 21 | ExecStartPre = utils.escapeSystemdExecArgs [ 22 | "$${pkgs.nix}/bin/nix-store" 23 | "--realize" 24 | "${charon}" 25 | ]; 26 | ExecStart = utils.escapeSystemdExecArgs [ 27 | "${charon}/bin/charon" 28 | "worker" 29 | "--heavy" 30 | "--temporal_params" 31 | "${tmpssm}" 32 | "--cache_signkey_ssm" 33 | "${cachessm}" 34 | "--chunkbucket" 35 | "${bucket}" 36 | "--nix_pubkey" 37 | "${pubkey}" 38 | "--nix_pubkey" 39 | "${nixoskey}" 40 | "--styx_signkey_ssm" 41 | "${styxssm}" 42 | ]; 43 | Restart = "always"; 44 | }; 45 | }; 46 | 47 | services.vector = { 48 | enable = true; 49 | journaldAccess = true; 50 | settings = { 51 | sources.journald = { 52 | type = "journald"; 53 | exclude_matches.SYSLOG_IDENTIFIER = [ "kernel" ]; 54 | }; 55 | sinks.axiom = { 56 | type = "axiom"; 57 | inputs = [ "journald" ]; 58 | dataset = "${axiom_dataset}"; 59 | token = "${axiom_token}"; 60 | }; 61 | }; 62 | }; 63 | 64 | system.stateVersion = "25.11"; 65 | } 66 | -------------------------------------------------------------------------------- /cmd/styx/cobrautil.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "reflect" 7 | 8 | "github.com/spf13/cobra" 9 | "golang.org/x/exp/slices" 10 | ) 11 | 12 | type runE = func(*cobra.Command, []string) error 13 | 14 | func chainRunE(fs ...runE) runE { 15 | fs = slices.DeleteFunc(fs, func(e runE) bool { return e == nil }) 16 | if len(fs) == 1 { 17 | return fs[0] 18 | } 19 | return func(c *cobra.Command, args []string) error { 20 | for _, f := range fs { 21 | if err := f(c, args); err != nil { 22 | return err 23 | } 24 | } 25 | return nil 26 | } 27 | } 28 | 29 | func cmd(c *cobra.Command, stuff ...any) *cobra.Command { 30 | for _, thing := range stuff { 31 | switch t := thing.(type) { 32 | case func(*cobra.Command): 33 | t(c) 34 | case *cobra.Command: 35 | c.AddCommand(t) 36 | case runE: 37 | c.RunE = chainRunE(c.RunE, t) 38 | case func(*cobra.Command) runE: 39 | c.RunE = chainRunE(c.RunE, t(c)) 40 | default: 41 | log.Panicf("bad cmd structure: %T %v", t, t) 42 | } 43 | } 44 | return c 45 | } 46 | 47 | type ckey struct { 48 | t reflect.Type 49 | k any 50 | } 51 | 52 | func storer[T any](v T) runE { 53 | return func(c *cobra.Command, args []string) error { 54 | store(c, v) 55 | return nil 56 | } 57 | } 58 | 59 | func store[T any](c *cobra.Command, v T) { 60 | t := reflect.TypeOf((*T)(nil)).Elem() 61 | c.SetContext(context.WithValue(c.Context(), ckey{t: t}, v)) 62 | } 63 | 64 | func get[T any](c *cobra.Command) T { 65 | t := reflect.TypeOf((*T)(nil)).Elem() 66 | return c.Context().Value(ckey{t: t}).(T) 67 | } 68 | 69 | func storeKeyed[T any](c *cobra.Command, v T, key any) { 70 | t := reflect.TypeOf((*T)(nil)).Elem() 71 | c.SetContext(context.WithValue(c.Context(), ckey{t: t, k: key}, v)) 72 | } 73 | 74 | func getKeyed[T any](c *cobra.Command, key any) T { 75 | t := reflect.TypeOf((*T)(nil)).Elem() 76 | return c.Context().Value(ckey{t: t, k: key}).(T) 77 | } 78 | -------------------------------------------------------------------------------- /bin/runvm: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Use: 3 | # runvm # default (ext4 root fs) 4 | # runvm -t btrfs # to get a btrfs root fs 5 | 6 | fstype=ext4 7 | while getopts ":t:" opt; do 8 | case $opt in 9 | t) fstype=$OPTARG ;; 10 | :) echo "Option -$OPTARG requires an argument."; exit 1 ;; 11 | esac 12 | done 13 | 14 | set -eux 15 | 16 | case $fstype in 17 | ext4) mkfs=$(nix-build --no-out-link '' -A e2fsprogs)/bin/mkfs.ext4 ;; 18 | btrfs) mkfs=$(nix-build --no-out-link '' -A btrfs-progs)/bin/mkfs.btrfs ;; 19 | *) echo "unknown fstype, use ext4 or btrfs"; exit 1;; 20 | esac 21 | 22 | vm=$(VMFSTYPE=$fstype nix-build --no-out-link '' -A vm -I nixos-config=./vm-interactive.nix) 23 | if size="$(stty size | tr ' ' :)"; then 24 | # hack to transfer console size 25 | export QEMU_KERNEL_PARAMS="styx.consolesize=$size" 26 | fi 27 | export NIX_DISK_IMAGE=/tmp/testvm-$UID.qcow2 28 | keep='' 29 | for arg; do 30 | case $arg in 31 | --keep) keep=1 ;; 32 | esac 33 | done 34 | [[ $keep ]] || rm -f $NIX_DISK_IMAGE 35 | 36 | # some nix invocations are pretty ram-hungry 37 | export QEMU_OPTS="-m 4G ${QEMU_OPTS:-}" 38 | 39 | # This is kind of gross: if you're actually using styx, then your "/nix/store" 40 | # is not a single filesystem anymore, there's stuff mounded inside it. The nixos 41 | # qemu module shares /nix/store by default, but qemu gets really confused with 42 | # the multiple mount points and prints an error like 43 | # qemu-kvm: warning: 9p: Multiple devices detected in same VirtFS export, … 44 | # There's a flag to handle this, multidevs=remap, but we have to sneak it into 45 | # the vm startup script. 46 | # Also, replace the mkfs call (hardcoded to ext4 in qemu-vm.nix) with the 47 | # desired root fs type. 48 | tmp=$(mktemp) 49 | sed -e ' 50 | 2s|^|rm -f '"$tmp"';| 51 | s|,mount_tag=nix-store|&,multidevs=remap| 52 | s|/nix/store/[^ /]*/bin/mkfs[.]ext4|'"$mkfs"'| 53 | ' < $vm/bin/run-testvm-vm > $tmp 54 | exec bash -x $tmp 55 | -------------------------------------------------------------------------------- /ci/proto.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/dnr/styx/manifester" 7 | ) 8 | 9 | const ( 10 | workflowType = "ci" // matches function name 11 | 12 | taskQueue = "charon" 13 | heavyTaskQueue = "charon-heavy" 14 | ) 15 | 16 | type ( 17 | CiArgs struct { 18 | // constants 19 | 20 | // what to watch and build 21 | Channel string 22 | StyxRepo RepoConfig 23 | 24 | // where to copy it 25 | CopyDest string 26 | ManifestUpstream string 27 | PublicCacheUpstream string 28 | 29 | // state 30 | LastRelID string `json:",omitempty"` // "nixos-23.11.7609.5c2ec3a5c2ee" 31 | LastStyxCommit string `json:",omitempty"` 32 | PrevNames []string `json:",omitempty"` 33 | LastGC int64 `json:",omitempty"` // unix seconds 34 | } 35 | 36 | RepoConfig struct { 37 | Repo string 38 | Branch string 39 | } 40 | 41 | pollChannelReq struct { 42 | Channel string 43 | LastRelID string 44 | } 45 | pollChannelRes struct { 46 | RelID string 47 | } 48 | 49 | pollRepoReq struct { 50 | Config RepoConfig 51 | LastCommit string 52 | } 53 | pollRepoRes struct { 54 | Commit string 55 | } 56 | 57 | buildReq struct { 58 | // global args 59 | Args *CiArgs 60 | 61 | // build args 62 | RelID string 63 | StyxCommit string 64 | } 65 | buildRes struct { 66 | FakeError string 67 | Names []string 68 | ManifestStats manifester.Stats 69 | NewLastGC int64 `json:",omitempty"` 70 | GCSummary string `json:",omitempty"` 71 | } 72 | buildErrDetails struct { 73 | Logs string 74 | } 75 | 76 | notifyReq struct { 77 | Args *CiArgs 78 | RelID string 79 | StyxCommit string 80 | Error string 81 | ErrorDetails *buildErrDetails 82 | BuildElapsed time.Duration 83 | PrevNames, NewNames []string 84 | ManifestStats manifester.Stats 85 | GCSummary string `json:",omitempty"` 86 | } 87 | ) 88 | -------------------------------------------------------------------------------- /tests/chunked_manifest_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/dnr/styx/daemon" 9 | ) 10 | 11 | func TestChunkedManifest(t *testing.T) { 12 | tb := newTestBase(t) 13 | tb.startAll() 14 | 15 | // This is a relatively small package (5.7M) with a lot of files (5752), mostly symlinks. 16 | // Will be chunked manifest. 17 | mp1 := tb.mount("z2waz77lsh4pxs0jxgmpf16s7a3g7b7v-openssl-3.0.13-man") 18 | require.Equal(t, "1m9w6v5z6w73ii42xyfsgyckvl3zkk1bx5wzvsydd95jbfhz8aga", tb.nixHash(mp1)) 19 | 20 | d1 := tb.debug() 21 | tb.dropCaches() 22 | require.Equal(t, "1m9w6v5z6w73ii42xyfsgyckvl3zkk1bx5wzvsydd95jbfhz8aga", tb.nixHash(mp1)) 23 | d2 := tb.debug() 24 | require.Equal(t, d1.Stats.SlabReads, d2.Stats.SlabReads) 25 | 26 | // These should have very similar manifests, chunks should diff: 27 | mp3 := tb.mount("xd96wmj058ky40aywv72z63vdw9yzzzb-openssl-3.0.12-man") 28 | d4 := tb.debug(daemon.DebugReq{IncludeSlabs: true}) 29 | // just from mounting, we should have new diff reqs 30 | require.Greater(t, d4.Stats.DiffReqs, d2.Stats.DiffReqs) 31 | require.Zero(t, d4.Stats.DiffErrs) 32 | 33 | // Actually this one has identical contents, manifest chunks should be identical: 34 | mp4 := tb.mount("v35ysx9k1ln4c6r7lj74204ss4bw7l5l-openssl-3.0.12-man") 35 | d5 := tb.debug(daemon.DebugReq{IncludeSlabs: true}) 36 | require.Equal(t, d4.Slabs[0].Stats.TotalChunks, d5.Slabs[0].Stats.TotalChunks) 37 | require.Equal(t, d4.Slabs[0].Stats.PresentChunks, d5.Slabs[0].Stats.PresentChunks) 38 | 39 | require.Equal(t, "0v60mg7qj7mfd27s1nnldb0041ln08xs1bw7zn1mmjiaq02myzlh", tb.nixHash(mp3)) 40 | require.Equal(t, "0v60mg7qj7mfd27s1nnldb0041ln08xs1bw7zn1mmjiaq02myzlh", tb.nixHash(mp4)) 41 | } 42 | 43 | func TestNoOverflowBeforeSuper(t *testing.T) { 44 | tb := newTestBase(t) 45 | tb.startAll() 46 | 47 | mp1 := tb.mount("kbi7qf642gsxiv51yqank8bnx39w3crd-calf-0.90.3") 48 | require.Equal(t, "1bhyfn2k8w41cx7ddarmjmwscas0946n6gw5mralx9lg0vbbcx6d", tb.nixHash(mp1)) 49 | } 50 | -------------------------------------------------------------------------------- /common/map.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "maps" 5 | "sync" 6 | ) 7 | 8 | type SimpleSyncMap[K comparable, V any] struct { 9 | lock sync.Mutex 10 | m map[K]V 11 | } 12 | 13 | func NewSimpleSyncMap[K comparable, V any]() *SimpleSyncMap[K, V] { 14 | return &SimpleSyncMap[K, V]{m: make(map[K]V)} 15 | } 16 | 17 | func (ssm *SimpleSyncMap[K, V]) Has(k K) bool { 18 | ssm.lock.Lock() 19 | defer ssm.lock.Unlock() 20 | _, ok := ssm.m[k] 21 | return ok 22 | } 23 | 24 | func (ssm *SimpleSyncMap[K, V]) Get(k K) (V, bool) { 25 | ssm.lock.Lock() 26 | defer ssm.lock.Unlock() 27 | v, ok := ssm.m[k] 28 | return v, ok 29 | } 30 | 31 | func (ssm *SimpleSyncMap[K, V]) Put(k K, v V) { 32 | ssm.lock.Lock() 33 | defer ssm.lock.Unlock() 34 | ssm.m[k] = v 35 | } 36 | 37 | // If k is present, returns its value and true, otherwise sets k to v and returns v and false. 38 | func (ssm *SimpleSyncMap[K, V]) GetOrPut(k K, v V) (V, bool) { 39 | ssm.lock.Lock() 40 | defer ssm.lock.Unlock() 41 | existing, ok := ssm.m[k] 42 | if ok { 43 | return existing, true 44 | } 45 | ssm.m[k] = v 46 | return v, false 47 | } 48 | 49 | func (ssm *SimpleSyncMap[K, V]) Delete(k K) { 50 | ssm.lock.Lock() 51 | defer ssm.lock.Unlock() 52 | delete(ssm.m, k) 53 | } 54 | 55 | // Deletes entries where f(k, v) returns true. 56 | func (ssm *SimpleSyncMap[K, V]) DeleteFunc(f func(K, V) bool) { 57 | ssm.lock.Lock() 58 | defer ssm.lock.Unlock() 59 | maps.DeleteFunc(ssm.m, f) 60 | } 61 | 62 | // Calls f on the value of key k, while holding the lock. 63 | func (ssm *SimpleSyncMap[K, V]) WithValue(k K, f func(V)) { 64 | ssm.lock.Lock() 65 | defer ssm.lock.Unlock() 66 | if v, ok := ssm.m[k]; ok { 67 | f(v) 68 | } 69 | } 70 | 71 | // Atomically calls f with the result of Get(k) and then either sets k to a new value or 72 | // deletes k. 73 | func (ssm *SimpleSyncMap[K, V]) Modify(k K, f func(V, bool) (V, bool)) { 74 | ssm.lock.Lock() 75 | defer ssm.lock.Unlock() 76 | v, ok := ssm.m[k] 77 | if nv, nok := f(v, ok); nok { 78 | ssm.m[k] = nv 79 | } else { 80 | delete(ssm.m, k) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /pb/db.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package pb; 4 | option go_package = "github.com/dnr/styx/pb"; 5 | 6 | import "params.proto"; 7 | 8 | enum MountState { 9 | Unknown = 0; // initial value 10 | Requested = 1; // requested but not mounted yet 11 | Mounted = 2; // mounted 12 | MountError = 3; // got error mounting 13 | UnmountRequested = 4; // was mounted then requested unmount 14 | Unmounted = 5; // unmounted 15 | Deleted = 6; // deleted (not used now) 16 | Materialized = 7; // not mounted but copied out to fs 17 | } 18 | 19 | // key: "slab" / / 20 | // value: 21 | // bucket sequence holds next chunk 22 | 23 | // key: "slab" / / (high bit set) 24 | // value: empty 25 | // present if data has been fetched 26 | 27 | // key: "chunk" / 28 | // value: ( )* 29 | 30 | // key: "image" / 31 | // value: 32 | message DbImage { 33 | // full store path including name 34 | string store_path = 2; 35 | // which upstream this was from 36 | string upstream = 3; 37 | // system id from syschecker 38 | int64 syschecker_system = 4; 39 | 40 | // is it mounted and where? 41 | MountState mount_state = 5; 42 | string mount_point = 6; 43 | string last_mount_error = 7; 44 | 45 | // size of erofs image 46 | int64 image_size = 1; 47 | bool is_bare = 10; 48 | 49 | // nar size, if known 50 | int64 nar_size = 11; 51 | 52 | reserved 8 to 9; 53 | } 54 | 55 | // key: "manifest" / 56 | // value: SignedMessage (manifest envelope) 57 | 58 | // key: "meta" / "params" 59 | message DbParams { 60 | DaemonParams params = 1; 61 | repeated string pubkey = 2; 62 | } 63 | 64 | // key: "fakecache" / 65 | // value: 66 | message FakeCacheData { 67 | bytes narinfo = 1; 68 | string upstream = 2; 69 | int64 updated = 3; 70 | } 71 | -------------------------------------------------------------------------------- /daemon/prefetch.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "strings" 7 | 8 | "github.com/dnr/styx/common/cdig" 9 | "github.com/nix-community/go-nix/pkg/storepath" 10 | "go.etcd.io/bbolt" 11 | ) 12 | 13 | func (s *Server) handlePrefetchReq(ctx context.Context, r *PrefetchReq) (*Status, error) { 14 | if s.p() == nil { 15 | return nil, mwErr(http.StatusPreconditionFailed, "styx is not initialized, call 'styx init --params=...'") 16 | } 17 | 18 | haveReq := make(map[cdig.CDig]struct{}) 19 | var reqs []cdig.CDig 20 | 21 | err := s.db.View(func(tx *bbolt.Tx) error { 22 | var sphStr string 23 | p := r.Path 24 | if r.StorePath == "" { 25 | // TODO: to simplify things, this only supports images mounted at the standard 26 | // location. we can relax this if we keep a trie of active mounts. 27 | if !underDir(p, storepath.StoreDir) || p == storepath.StoreDir { 28 | return mwErr(http.StatusBadRequest, "path is not a valid store path") 29 | } 30 | p = p[len(storepath.StoreDir)+1:] // p starts with store path now 31 | storePath, _, _ := strings.Cut(p, "/") 32 | sphStr = storePath 33 | // strip off mountpoint. p should now have an initial / 34 | p = p[len(storePath):] 35 | if len(p) == 0 { 36 | p = "/" 37 | } 38 | } else { 39 | sphStr = r.StorePath 40 | } 41 | sph, sphStr, err := ParseSph(sphStr) 42 | if err != nil { 43 | return err 44 | } 45 | ents, err := s.getDigestsFromImage(tx, sph, false) 46 | if err != nil { 47 | return mwErr(http.StatusInternalServerError, "can't read manifest") 48 | } 49 | for _, e := range ents { 50 | if len(e.Digests) > 0 && underDir(e.Path, p) { 51 | digests := cdig.FromSliceAlias(e.Digests) 52 | for _, d := range digests { 53 | // just do a simple de-dup here, don't check presence 54 | if _, ok := haveReq[d]; !ok { 55 | haveReq[d] = struct{}{} 56 | reqs = append(reqs, d) 57 | } 58 | } 59 | } 60 | } 61 | if len(reqs) == 0 { 62 | return nil 63 | } 64 | return nil 65 | }) 66 | if err != nil || len(reqs) == 0 { 67 | return nil, err 68 | } 69 | return nil, s.requestPrefetch(ctx, reqs) 70 | } 71 | -------------------------------------------------------------------------------- /tests/recompress_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestRecompress(t *testing.T) { 11 | tb := newTestBase(t) 12 | tb.startAll() 13 | 14 | mp1 := tb.mount("cd1nbildgzzfryjg82njnn36i4ynyf8h-bash-interactive-5.1-p16-man") 15 | man1 := filepath.Join(mp1, "share", "man", "man1", "bash.1.gz") 16 | require.Equal(t, "0s9d681f8smlsdvbp6lin9qrbsp3hz3dnf4pdhwi883v8l1486r7", tb.nixHash(man1)) 17 | d1 := tb.debug() 18 | require.NotZero(t, d1.Stats.BatchReqs) 19 | require.Zero(t, d1.Stats.SingleReqs+d1.Stats.DiffReqs) 20 | 21 | mp2 := tb.mount("8vyj9c6g424mz0v3kvzkskhvzhwj6288-bash-interactive-5.2-p15-man") 22 | man2 := filepath.Join(mp2, "share", "man", "man1", "bash.1.gz") 23 | require.Equal(t, "0r2agiq8bzv09nsk11yidwvyjb5cfrp5wavq2jxqc9k6sh1256s9", tb.nixHash(man2)) 24 | d2 := tb.debug() 25 | require.NotZero(t, d2.Stats.DiffReqs) 26 | require.Greater(t, d2.Stats.DiffBytes, int64(0)) 27 | // the zstd diff between the uncompressed files is < 7kb. between compressed files is 95kb 28 | // (no savings). check that we did the recompress thing. 29 | require.Less(t, d2.Stats.DiffBytes, int64(10000)) 30 | 31 | require.Zero(t, d2.Stats.SingleErrs+d2.Stats.BatchErrs+d2.Stats.DiffErrs) 32 | } 33 | 34 | func TestMultiRecompress(t *testing.T) { 35 | tb := newTestBase(t) 36 | tb.startAll() 37 | 38 | _ = tb.materialize("z2waz77lsh4pxs0jxgmpf16s7a3g7b7v-openssl-3.0.13-man") 39 | d1 := tb.debug() 40 | require.NotZero(t, d1.Stats.BatchReqs) 41 | require.Zero(t, d1.Stats.SingleReqs+d1.Stats.DiffReqs) 42 | 43 | _ = tb.materialize("xd96wmj058ky40aywv72z63vdw9yzzzb-openssl-3.0.12-man") 44 | d2 := tb.debug() 45 | require.NotZero(t, d2.Stats.DiffReqs) 46 | require.Greater(t, d2.Stats.DiffBytes, int64(0)) 47 | 48 | // the zstd diff between the uncompressed files is ~128kb. between compressed files is ~1811kb. 49 | // check that we did the recompress thing. 50 | require.Less(t, d2.Stats.DiffBytes, int64(900*1024)) 51 | 52 | // check that we did it in a small number of total requests (this is the "multi" part) 53 | require.Less(t, d2.Stats.DiffReqs, int64(15)) 54 | 55 | require.Zero(t, d2.Stats.SingleErrs+d2.Stats.BatchErrs+d2.Stats.DiffErrs) 56 | } 57 | -------------------------------------------------------------------------------- /tests/vaporize_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "os/exec" 6 | "path/filepath" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestVaporize(t *testing.T) { 13 | tb := newTestBase(t) 14 | tb.startAll() 15 | 16 | tmp := tb.t.TempDir() 17 | 18 | testVaporize := func(name, filehash, datahash string, expected int64, doMount bool) { 19 | d1 := tb.debug() 20 | src := filepath.Join(tmp, name) 21 | cmd := fmt.Sprintf("xz -cd %s/nar/%s.nar.xz | nix-store --restore %s", TestdataDir, filehash, src) 22 | require.NoError(t, exec.Command("sh", "-c", cmd).Run()) 23 | // vaporize into slab 24 | tb.vaporize(src) 25 | // now try to materialize out of slab 26 | var dst string 27 | if doMount { 28 | dst = tb.mount(name) 29 | } else { 30 | dst = tb.materialize(name) 31 | } 32 | require.Equal(t, datahash, tb.nixHash(dst)) 33 | // should be requests for manifest chunks only, all data is in slab 34 | d2 := tb.debug() 35 | require.Equal(t, expected, d2.Stats.Sub(d1.Stats).TotalReqs()) 36 | // TODO: btrfs fi du to check that extents are shared 37 | } 38 | 39 | testVaporize( 40 | "qa22bifihaxyvn6q2a6w9m0nklqrk9wh-opusfile-0.12", 41 | "0h336qzb63kdqxwc5yjrxq61cjraz8jrav0m5rkrcvsb6w55rbll", 42 | "1rswindywkyq2jmfpxd6n772jii3z5xz6ypfbb63c17k5il39hfm", 43 | 0, 44 | false) 45 | 46 | testVaporize( 47 | "kbi7qf642gsxiv51yqank8bnx39w3crd-calf-0.90.3", 48 | "11r77sgdyqqfi8z36p104098g16cvvq6jvhypv0xw79jrqq33j7n", 49 | "1bhyfn2k8w41cx7ddarmjmwscas0946n6gw5mralx9lg0vbbcx6d", 50 | 0, // 1 manifest chunk. TODO: why doesn't this need a manifest chunk anymore? 51 | false) 52 | 53 | testVaporize( 54 | "v35ysx9k1ln4c6r7lj74204ss4bw7l5l-openssl-3.0.12-man", 55 | "1mv76iwv027rxgdb0i04www6nkx8hy5bxh8v8vjihr9pl5a37hpy", 56 | "0v60mg7qj7mfd27s1nnldb0041ln08xs1bw7zn1mmjiaq02myzlh", 57 | 1, // 1 manifest chunk 58 | true) 59 | 60 | testVaporize( 61 | "3a7xq2qhxw2r7naqmc53akmx7yvz0mkf-less-is-more.patch", 62 | "0a2jm36lynlaw4vxr4xnflxz93jadr4xw03ab2hgardshqij3y7c", 63 | "13jlq14n974nn919530hnx4l46d0p2zyhx4lrd9b1k122dn7w9z5", 64 | 0, 65 | false) 66 | 67 | testVaporize( 68 | "cd1nbildgzzfryjg82njnn36i4ynyf8h-bash-interactive-5.1-p16-man", 69 | "0r02jy5bmdl5gvflmm9yq3aqa12zbv4hkkv1lqhm6ps49xnxlq6c", 70 | "0anwmd9q85lyn4aid5glvzf5ikwr113zbrrpym1nf377r0ap298s", 71 | 0, 72 | true) 73 | } 74 | -------------------------------------------------------------------------------- /daemon/util.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | 8 | "golang.org/x/sys/unix" 9 | 10 | "github.com/dnr/styx/common/cdig" 11 | "github.com/dnr/styx/erofs" 12 | "github.com/dnr/styx/pb" 13 | ) 14 | 15 | func verifyParams(p *pb.GlobalParams) error { 16 | if p.DigestAlgo != cdig.Algo { 17 | return fmt.Errorf("built-in digest algo %s != %s; rebuild or use different params", 18 | cdig.Algo, p.DigestAlgo) 19 | } else if p.DigestBits != cdig.Bits { 20 | return fmt.Errorf("built-in digest bits %d != %d; rebuild or use different params", 21 | cdig.Bits, p.DigestBits) 22 | } 23 | return nil 24 | } 25 | 26 | func sphpsFromLoc(b []byte) []SphPrefix { 27 | b = b[6:] 28 | out := make([]SphPrefix, len(b)/sphPrefixBytes) 29 | for i := range out { 30 | out[i] = SphPrefixFromBytes(b[i*sphPrefixBytes : (i+1)*sphPrefixBytes]) 31 | } 32 | return out 33 | } 34 | 35 | func writeToTempFile(b []byte) (string, error) { 36 | f, err := os.CreateTemp("", "styx-diff") 37 | if err != nil { 38 | return "", err 39 | } 40 | defer f.Close() 41 | if _, err = f.Write(b); err != nil { 42 | return "", err 43 | } 44 | return f.Name(), nil 45 | } 46 | 47 | func makeManifestSph(sph Sph) Sph { 48 | // the "manifest sph" for a sph is the same with one bit flipped (will affect _end_ of base32 49 | // string form). note that this is its own inverse. 50 | sph[0] ^= 1 51 | return sph 52 | } 53 | 54 | type countReader struct { 55 | r io.Reader 56 | c int64 57 | } 58 | 59 | func (cr *countReader) Read(p []byte) (int, error) { 60 | n, err := cr.r.Read(p) 61 | cr.c += int64(n) 62 | return n, err 63 | } 64 | 65 | func underDir(p, dir string) bool { 66 | return len(p) >= len(dir) && p[:len(dir)] == dir && (len(p) == len(dir) || dir == "/" || p[len(dir)] == '/') 67 | } 68 | 69 | func isErofsMount(p string) (bool, error) { 70 | var st unix.Statfs_t 71 | err := unix.Statfs(p, &st) 72 | return st.Type == erofs.EROFS_MAGIC, err 73 | } 74 | 75 | func getMountNs(pid int) (string, error) { 76 | path := fmt.Sprintf("/proc/%d/ns/mnt", pid) 77 | return os.Readlink(path) 78 | } 79 | 80 | func havePrivateMountNs() (bool, error) { 81 | myMountNs, err := getMountNs(os.Getpid()) 82 | if err != nil { 83 | return false, err 84 | } 85 | parentMountNs, err := getMountNs(os.Getppid()) 86 | if err != nil { 87 | return false, err 88 | } 89 | return myMountNs != parentMountNs, nil 90 | } 91 | -------------------------------------------------------------------------------- /daemon/repair.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "regexp" 11 | "strconv" 12 | "strings" 13 | 14 | "go.etcd.io/bbolt" 15 | 16 | "github.com/dnr/styx/common" 17 | ) 18 | 19 | func (s *Server) handleRepairReq(ctx context.Context, r *RepairReq) (*Status, error) { 20 | // allow this even before "initialized" 21 | 22 | if r.Presence { 23 | for slab := uint16(0); slab < manifestSlabOffset; slab++ { 24 | tag, _ := s.SlabInfo(slab) 25 | backingPath := filepath.Join(s.cfg.CachePath, fscachePath(s.cfg.CacheDomain, tag)) 26 | if _, err := os.Stat(backingPath); os.IsNotExist(err) { 27 | break 28 | } 29 | s.repairPresence(slab, backingPath) 30 | } 31 | } 32 | 33 | return nil, nil 34 | } 35 | 36 | func (s *Server) repairPresence(slab uint16, path string) { 37 | blk := fmt.Sprintf("-b%d", s.blockShift.Size()) 38 | // TODO: use FIEMAP ioctl directly 39 | out, err := exec.Command(common.FilefragBin, "-evs", blk, path).Output() 40 | if err != nil { 41 | return 42 | } 43 | re := regexp.MustCompile(`\s*\d+:\s*(\d+)\.\.\s*(\d+):.*`) 44 | 45 | have := make(map[uint32]bool) 46 | for _, l := range strings.Split(string(out), "\n") { 47 | m := re.FindStringSubmatch(l) 48 | if len(m) < 3 { 49 | continue 50 | } 51 | start, err1 := strconv.Atoi(m[1]) 52 | end, err2 := strconv.Atoi(m[2]) 53 | if err1 != nil || err2 != nil { 54 | continue 55 | } 56 | for i := start; i <= end; i++ { 57 | have[uint32(i)] = true 58 | } 59 | } 60 | 61 | s.db.Update(func(tx *bbolt.Tx) error { 62 | sb := tx.Bucket(slabBucket).Bucket(slabKey(slab)) 63 | 64 | var all []uint32 65 | dbhave := make(map[uint32]bool) 66 | cur := sb.Cursor() 67 | var k []byte 68 | for k, _ = cur.First(); k != nil && addrFromKey(k) < presentMask; k, _ = cur.Next() { 69 | all = append(all, addrFromKey(k)) 70 | } 71 | for ; k != nil; k, _ = cur.Next() { 72 | dbhave[addrFromKey(k)&^presentMask] = true 73 | } 74 | 75 | for _, addr := range all { 76 | if !have[addr] && dbhave[addr] { 77 | log.Println("repair slab", slab, "addr", addr, "marked present but missing from block map") 78 | sb.Delete(addrKey(addr | presentMask)) 79 | } else if have[addr] && !dbhave[addr] { 80 | log.Println("repair slab", slab, "addr", addr, "in block map but not present") 81 | sb.Put(addrKey(addr|presentMask), []byte{}) 82 | } 83 | } 84 | 85 | return nil 86 | }) 87 | } 88 | -------------------------------------------------------------------------------- /common/systemd/systemd.go: -------------------------------------------------------------------------------- 1 | package systemd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | "math/rand" 8 | "net" 9 | "os" 10 | "strconv" 11 | "strings" 12 | 13 | "golang.org/x/sys/unix" 14 | ) 15 | 16 | type ( 17 | FdStore interface { 18 | Ready() 19 | GetFd(name string) (int, error) 20 | SaveFd(name string, fd int) 21 | RemoveFd(name string) 22 | } 23 | 24 | SystemdFdStore struct{} 25 | ) 26 | 27 | func (SystemdFdStore) GetFd(name string) (int, error) { 28 | pid, err := strconv.Atoi(os.Getenv("LISTEN_PID")) 29 | if err != nil || pid != os.Getpid() { 30 | return 0, errors.New("no fds passed") 31 | } 32 | nfds, err := strconv.Atoi(os.Getenv("LISTEN_FDS")) 33 | if err != nil || nfds == 0 { 34 | return 0, errors.New("no fds passed") 35 | } 36 | for fd := 3; fd < 3+nfds; fd++ { 37 | unix.CloseOnExec(fd) 38 | } 39 | for i, n := range strings.Split(os.Getenv("LISTEN_FDNAMES"), ":") { 40 | if n == name { 41 | return 3 + i, nil 42 | } 43 | } 44 | return 0, errors.New("name not found") 45 | } 46 | 47 | func (SystemdFdStore) SaveFd(name string, fd int) { 48 | addr := notifyAddr() 49 | if addr == nil { 50 | return 51 | } 52 | srcname := fmt.Sprintf("/tmp/styx-notify-src-%x", rand.Int63()) 53 | defer os.Remove(srcname) 54 | conn, err := net.ListenUnixgram("unixgram", &net.UnixAddr{Net: "unixgram", Name: srcname}) 55 | if err != nil { 56 | log.Println("error dialing unix socket", err) 57 | return 58 | } 59 | defer conn.Close() 60 | // set FDPOLL=0 since cachefiles uses POLLERR specially 61 | msg := fmt.Sprintf("FDSTORE=1\nFDNAME=%s\nFDPOLL=0\n", name) 62 | oob := unix.UnixRights(fd) 63 | if _, _, err := conn.WriteMsgUnix([]byte(msg), oob, addr); err != nil { 64 | log.Println("error writing to notify socket", err) 65 | } 66 | } 67 | 68 | func (SystemdFdStore) RemoveFd(name string) { 69 | send(fmt.Sprintf("FDSTOREREMOVE=1\nFDNAME=%s", name)) 70 | } 71 | 72 | func (SystemdFdStore) Ready() { 73 | send("READY=1") 74 | } 75 | 76 | func send(msg string) { 77 | addr := notifyAddr() 78 | if addr == nil { 79 | return 80 | } 81 | conn, err := net.DialUnix(addr.Net, nil, addr) 82 | if err != nil { 83 | return 84 | } 85 | defer conn.Close() 86 | if _, err := conn.Write([]byte(msg)); err != nil { 87 | log.Println("error writing to notify socket", err) 88 | } 89 | } 90 | 91 | func notifyAddr() *net.UnixAddr { 92 | if name := os.Getenv("NOTIFY_SOCKET"); name != "" { 93 | return &net.UnixAddr{Name: name, Net: "unixgram"} 94 | } 95 | return nil 96 | } 97 | -------------------------------------------------------------------------------- /tf/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/aws" { 5 | version = "5.44.0" 6 | hashes = [ 7 | "h1:QqMTKuyylmJ633mwNheXdFupfd5sozqCUTTSj89pnm8=", 8 | "zh:1224a42bb04574785549b89815d98bda11f6e9992352fc6c36c5622f3aea91c0", 9 | "zh:2a8d1095a2f1ab097f516d9e7e0d289337849eebb3fcc34f075070c65063f4fa", 10 | "zh:46cce11150eb4934196d9bff693b72d0494c85917ceb3c2914d5ff4a785af861", 11 | "zh:4a7c15d585ee747d17f4b3904851cd95cfbb920fa197aed3df78e8d7ef9609b6", 12 | "zh:508f1a85a0b0f93bf26341207d809bd55b60c8fdeede40097d91f30111fc6f5d", 13 | "zh:52f968ffc21240213110378d0ffb298cbd23e9157a6d01dfac5a4360492d69c2", 14 | "zh:5e9846b48ef03eb59541049e81b15cae8bc7696a3779ae4a5412fdce60bb24e0", 15 | "zh:850398aecaf7dc0231fc320fdd6dffe41836e07a54c8c7b40eb28e7525d3c0a9", 16 | "zh:8f87eeb05bdd1b873b6cfb3898dfad6402ac180dfa3c8f9754df8f85dcf92ca6", 17 | "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", 18 | "zh:c726b87cd6ed111536f875dccedecff21abc802a4087264515ffab113cac36dc", 19 | "zh:d57ea706d2f98b93c7b05b0c6bc3420de8e8cf2d0b6703085dc15ed239b2cc49", 20 | "zh:d5d1a21246e68c2a7a04c5619eb0ad5a81644f644c432cb690537b816a156de2", 21 | "zh:e869904cac41114b7e4ee66bcd2ce4585ed15ca842040a60cb47119f69472c91", 22 | "zh:f1a09f2f3ea72cbe795b865cf31ad9b1866a536a8050cf0bb93d3fa51069582e", 23 | ] 24 | } 25 | 26 | provider "registry.terraform.io/hashicorp/local" { 27 | version = "2.5.1" 28 | hashes = [ 29 | "h1:8oTPe2VUL6E2d3OcrvqyjI4Nn/Y/UEQN26WLk5O/B0g=", 30 | "zh:0af29ce2b7b5712319bf6424cb58d13b852bf9a777011a545fac99c7fdcdf561", 31 | "zh:126063ea0d79dad1f68fa4e4d556793c0108ce278034f101d1dbbb2463924561", 32 | "zh:196bfb49086f22fd4db46033e01655b0e5e036a5582d250412cc690fa7995de5", 33 | "zh:37c92ec084d059d37d6cffdb683ccf68e3a5f8d2eb69dd73c8e43ad003ef8d24", 34 | "zh:4269f01a98513651ad66763c16b268f4c2da76cc892ccfd54b401fff6cc11667", 35 | "zh:51904350b9c728f963eef0c28f1d43e73d010333133eb7f30999a8fb6a0cc3d8", 36 | "zh:73a66611359b83d0c3fcba2984610273f7954002febb8a57242bbb86d967b635", 37 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 38 | "zh:7ae387993a92bcc379063229b3cce8af7eaf082dd9306598fcd42352994d2de0", 39 | "zh:9e0f365f807b088646db6e4a8d4b188129d9ebdbcf2568c8ab33bddd1b82c867", 40 | "zh:b5263acbd8ae51c9cbffa79743fbcadcb7908057c87eb22fd9048268056efbc4", 41 | "zh:dfcd88ac5f13c0d04e24be00b686d069b4879cc4add1b7b1a8ae545783d97520", 42 | ] 43 | } 44 | -------------------------------------------------------------------------------- /vm-interactive.nix: -------------------------------------------------------------------------------- 1 | { 2 | config, 3 | lib, 4 | pkgs, 5 | ... 6 | }: 7 | { 8 | imports = [ 9 | ./vm-base.nix 10 | ./module 11 | 12 | ]; 13 | assertions = [ 14 | { 15 | assertion = config.virtualisation.diskImage != null; 16 | message = "must use disk image"; 17 | } 18 | ]; 19 | 20 | # enable all options 21 | services.styx.enable = true; 22 | 23 | # let styx handle everything 24 | nix.settings.styx-ondemand = [ ".*" ]; 25 | 26 | # use shared nixpkgs 27 | nix.nixPath = [ "nixpkgs=/tmp/nixpkgs" ]; 28 | 29 | # just console 30 | virtualisation.graphics = false; 31 | # provide nixpkgs and this dir for convenience 32 | virtualisation.sharedDirectories = { 33 | nixpkgs = { 34 | source = toString ; 35 | target = "/tmp/nixpkgs"; 36 | }; 37 | styxsrc = { 38 | source = toString ./.; 39 | target = "/tmp/styxsrc"; 40 | }; 41 | }; 42 | # set fstype of root fs 43 | virtualisation.fileSystems."/".fsType = lib.mkForce (builtins.getEnv "VMFSTYPE"); 44 | # ensure btrfs enabled 45 | system.requiredKernelConfig = with config.lib.kernelConfig; [ (isEnabled "BTRFS_FS") ]; 46 | 47 | # more convenience 48 | environment.shellAliases = { 49 | l = "less"; 50 | ll = "ls -l"; 51 | g = "grep"; 52 | }; 53 | environment.variables = { 54 | TMPDIR = "/tmp"; # tmpfs is too small to build stuff 55 | }; 56 | environment.systemPackages = with pkgs; [ 57 | file 58 | jq 59 | psmisc 60 | vim 61 | ]; 62 | 63 | # hack to transfer console size 64 | systemd.services."serial-getty@".serviceConfig.ExecStartPost = 65 | let 66 | fixconsole = pkgs.writeShellScript "fixconsole" '' 67 | #!${pkgs.runtimeShell} 68 | tty=/dev/$1 69 | for o in $( $tty 75 | ;; 76 | esac 77 | done 78 | ''; 79 | in 80 | "-${fixconsole} %i"; 81 | 82 | # auto-login as root 83 | services.getty.autologinUser = "root"; 84 | 85 | # auto-init with test1 params 86 | systemd.services."StyxInitTest1" = { 87 | description = "Init Styx Nix storage manager"; 88 | after = [ "network-online.target" ]; 89 | wants = [ "network-online.target" ]; 90 | wantedBy = [ "multi-user.target" ]; 91 | path = [ config.services.styx.package ]; 92 | serviceConfig = { 93 | ExecStart = "/run/current-system/sw/bin/StyxInitTest1"; 94 | Type = "oneshot"; 95 | }; 96 | }; 97 | } 98 | -------------------------------------------------------------------------------- /daemon/recompress.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "context" 7 | "fmt" 8 | "io" 9 | "os/exec" 10 | "path" 11 | "regexp" 12 | 13 | "github.com/dnr/styx/common" 14 | "github.com/dnr/styx/manifester" 15 | "github.com/dnr/styx/pb" 16 | ) 17 | 18 | var ( 19 | reManPage = regexp.MustCompile(`^/share/man/.*[.]gz$`) 20 | reLinuxKoXz = regexp.MustCompile(`^/lib/modules/[^/]+/kernel/.*[.]ko[.]xz$`) 21 | 22 | recompressManPage = []string{manifester.ExpandGz} 23 | // note: currently largest kernel module on my system (excluding kheaders) is 24 | // amdgpu.ko.xz at 3.4mb, 54 chunks (64kb), and expands to 24.4mb, which is 25 | // reasonable to pass through the chunk differ. 26 | // TODO: maybe get these args from looking at the base? or the chunk differ can look at 27 | // req and return them? or try several values and take the matching one? 28 | recompressLinuxKo = []string{manifester.ExpandXz, "--check=crc32", "--lzma2=dict=1MiB"} 29 | ) 30 | 31 | func isManPageGz(ent *pb.Entry) bool { 32 | return ent.Type == pb.EntryType_REGULAR && reManPage.MatchString(ent.Path) 33 | } 34 | func isLinuxKoXz(ent *pb.Entry) bool { 35 | // kheaders.ko.xz is mostly an embedded .tar.xz file (yes, again), so expanding it won't help. 36 | return ent.Type == pb.EntryType_REGULAR && reLinuxKoXz.MatchString(ent.Path) && 37 | path.Base(ent.Path) != "kheaders.ko.xz" 38 | } 39 | 40 | func getRecompressArgs(ent *pb.Entry) []string { 41 | switch { 42 | case isManPageGz(ent): 43 | return recompressManPage 44 | case isLinuxKoXz(ent): 45 | return recompressLinuxKo 46 | default: 47 | return nil 48 | } 49 | } 50 | 51 | func doDiffDecompress(ctx context.Context, data []byte, args []string) ([]byte, error) { 52 | if len(args) == 0 { 53 | return data, nil 54 | } 55 | switch args[0] { 56 | case manifester.ExpandGz: 57 | gz, err := gzip.NewReader(bytes.NewReader(data)) 58 | if err != nil { 59 | return nil, err 60 | } 61 | return io.ReadAll(gz) 62 | 63 | case manifester.ExpandXz: 64 | xz := exec.Command(common.XzBin, "-d") 65 | xz.Stdin = bytes.NewReader(data) 66 | return xz.Output() 67 | 68 | default: 69 | return nil, fmt.Errorf("unknown expander %q", args[0]) 70 | } 71 | } 72 | 73 | func doDiffRecompress(ctx context.Context, data []byte, args []string) ([]byte, error) { 74 | if len(args) == 0 { 75 | return data, nil 76 | } 77 | switch args[0] { 78 | case manifester.ExpandGz: 79 | gz := exec.Command(common.GzipBin, "-nc") 80 | gz.Stdin = bytes.NewReader(data) 81 | return gz.Output() 82 | 83 | case manifester.ExpandXz: 84 | xz := exec.Command(common.XzBin, append([]string{"-c"}, args[1:]...)...) 85 | xz.Stdin = bytes.NewReader(data) 86 | return xz.Output() 87 | 88 | default: 89 | return nil, fmt.Errorf("unknown expander %q", args[0]) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /tests/restart_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "golang.org/x/sys/unix" 8 | ) 9 | 10 | // Daemon restart with mounted filesystems. 11 | func TestRestart(t *testing.T) { 12 | tb := newTestBase(t) 13 | tb.startAll() 14 | 15 | // mount something with first daemon run 16 | mp1 := tb.mount("qa22bifihaxyvn6q2a6w9m0nklqrk9wh-opusfile-0.12") 17 | 18 | // stop it. cachefiles fd will be saved in test fdstore 19 | tb.daemon.Stop(false) 20 | tb.daemon = nil 21 | 22 | // start again 23 | tb.startDaemon() 24 | 25 | // check that we can read 26 | require.Equal(t, "1rswindywkyq2jmfpxd6n772jii3z5xz6ypfbb63c17k5il39hfm", tb.nixHash(mp1)) 27 | 28 | // this will fail if we didn't set up the slab read fd 29 | checkDiffAfterRestart(t, tb) 30 | } 31 | 32 | // Simulates reboot and starting with nothing mounted. 33 | func TestReboot(t *testing.T) { 34 | tb := newTestBase(t) 35 | tb.startAll() 36 | 37 | // mount something with first daemon run 38 | mp1 := tb.mount("qa22bifihaxyvn6q2a6w9m0nklqrk9wh-opusfile-0.12") 39 | mp2 := tb.mount("3a7xq2qhxw2r7naqmc53akmx7yvz0mkf-less-is-more.patch") 40 | 41 | // read 2 but not 1 42 | require.Equal(t, "13jlq14n974nn919530hnx4l46d0p2zyhx4lrd9b1k122dn7w9z5", tb.nixHash(mp2)) 43 | 44 | // stop and close devnode 45 | tb.daemon.Stop(true) 46 | tb.daemon = nil 47 | 48 | // unmount filesystems directly to simulate clean state after reboot 49 | // note that Stop unmounts the slab image 50 | require.NoError(t, unix.Unmount(mp1, 0)) 51 | require.NoError(t, unix.Unmount(mp2, 0)) 52 | 53 | // start again 54 | tb.startDaemon() 55 | 56 | // both should have been remounted automatically 57 | require.Equal(t, "13jlq14n974nn919530hnx4l46d0p2zyhx4lrd9b1k122dn7w9z5", tb.nixHash(mp2)) 58 | 59 | // already cached, no requests 60 | d1 := tb.debug() 61 | require.Zero(t, d1.Stats.SingleReqs+d1.Stats.BatchReqs+d1.Stats.DiffReqs) 62 | 63 | require.Equal(t, "1rswindywkyq2jmfpxd6n772jii3z5xz6ypfbb63c17k5il39hfm", tb.nixHash(mp1)) 64 | 65 | // this needs some requests 66 | d2 := tb.debug() 67 | require.Zero(t, d2.Stats.SingleReqs+d2.Stats.DiffReqs) 68 | require.NotZero(t, d2.Stats.BatchReqs) 69 | 70 | // this will fail if we didn't set up the slab read fd 71 | checkDiffAfterRestart(t, tb) 72 | } 73 | 74 | func checkDiffAfterRestart(t *testing.T, tb *testBase) { 75 | mp := tb.mount("kcyrz2y8si9ry5p8qkmj0gp41n01sa1y-opusfile-0.12") 76 | d1 := tb.debug() 77 | require.Equal(t, "0im7spp48afrbfv672bmrvrs0lg4md0qhyic8zkcgyc8xqwz1s5b", tb.nixHash(mp)) 78 | d2 := tb.debug() 79 | require.Zero(t, d2.Stats.SingleReqs-d1.Stats.SingleReqs) 80 | require.Zero(t, d2.Stats.BatchReqs-d1.Stats.BatchReqs) 81 | require.NotZero(t, d2.Stats.DiffReqs-d1.Stats.DiffReqs) 82 | require.Zero(t, d2.Stats.SingleErrs+d2.Stats.BatchErrs+d2.Stats.DiffErrs) 83 | } 84 | -------------------------------------------------------------------------------- /tests/prefetch_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestPrefetch(t *testing.T) { 11 | tb := newTestBase(t) 12 | tb.startAll() 13 | 14 | mp1 := tb.mount("qa22bifihaxyvn6q2a6w9m0nklqrk9wh-opusfile-0.12") 15 | sph := "qa22bifihaxyvn6q2a6w9m0nklqrk9wh" 16 | libDir := filepath.Join(mp1, "lib") 17 | shareDir := filepath.Join(mp1, "share") 18 | 19 | // nothing present yet 20 | // prefetch only share 21 | tb.prefetch(sph, "/share") 22 | d1 := tb.debug() 23 | require.Zero(t, d1.Stats.SingleReqs) 24 | require.EqualValues(t, 1, d1.Stats.BatchReqs) // fits in single req 25 | require.Zero(t, d1.Stats.DiffReqs) 26 | 27 | // read share dir, no reqs 28 | tb.nixHash(shareDir) 29 | d2 := tb.debug() 30 | require.Zero(t, d2.Stats.SingleReqs-d1.Stats.SingleReqs) 31 | require.Zero(t, d2.Stats.BatchReqs-d1.Stats.BatchReqs) 32 | require.Zero(t, d2.Stats.DiffReqs-d1.Stats.DiffReqs) 33 | 34 | // now prefetch lib 35 | tb.prefetch(sph, "/lib") 36 | d3 := tb.debug() 37 | require.Zero(t, d3.Stats.SingleReqs-d2.Stats.SingleReqs) 38 | require.EqualValues(t, 1, d3.Stats.BatchReqs-d2.Stats.BatchReqs) // fits in single req 39 | require.Zero(t, d3.Stats.DiffReqs-d2.Stats.DiffReqs) 40 | 41 | // read lib dir, no reqs 42 | tb.nixHash(libDir) 43 | d4 := tb.debug() 44 | require.Zero(t, d4.Stats.SingleReqs-d3.Stats.SingleReqs) 45 | require.Zero(t, d4.Stats.BatchReqs-d3.Stats.BatchReqs) 46 | require.Zero(t, d4.Stats.DiffReqs-d3.Stats.DiffReqs) 47 | 48 | // similar image 49 | mp2 := tb.mount("kcyrz2y8si9ry5p8qkmj0gp41n01sa1y-opusfile-0.12") 50 | sph2 := "kcyrz2y8si9ry5p8qkmj0gp41n01sa1y" 51 | 52 | // this one should do a diff (in one req) 53 | tb.prefetch(sph2, "/") 54 | d5 := tb.debug() 55 | require.Zero(t, d5.Stats.SingleReqs-d4.Stats.SingleReqs) 56 | require.Zero(t, d5.Stats.BatchReqs-d4.Stats.BatchReqs) 57 | require.EqualValues(t, 1, d5.Stats.DiffReqs-d4.Stats.DiffReqs) 58 | 59 | // read all, no reqs 60 | tb.nixHash(mp2) 61 | d6 := tb.debug() 62 | require.Zero(t, d6.Stats.SingleReqs-d5.Stats.SingleReqs) 63 | require.Zero(t, d6.Stats.BatchReqs-d5.Stats.BatchReqs) 64 | require.Zero(t, d6.Stats.DiffReqs-d5.Stats.DiffReqs) 65 | 66 | require.Zero(t, d6.Stats.SingleErrs+d6.Stats.BatchErrs+d6.Stats.DiffErrs) 67 | } 68 | 69 | func TestPrefetchLarge(t *testing.T) { 70 | tb := newTestBase(t) 71 | tb.startAll() 72 | 73 | mp1 := tb.mount("xpq4yhadyhazkcsggmqd7rsgvxb3kjy4-gnugrep-3.11") 74 | sph := "xpq4yhadyhazkcsggmqd7rsgvxb3kjy4" 75 | 76 | // 49 chunks total, but fits into one req because we increase the limit for prefetch 77 | tb.prefetch(sph, "/") 78 | d1 := tb.debug() 79 | require.Zero(t, d1.Stats.SingleReqs) 80 | require.EqualValues(t, 1, d1.Stats.BatchReqs) 81 | require.Zero(t, d1.Stats.DiffReqs) 82 | 83 | // read, no reqs 84 | tb.nixHash(mp1) 85 | d2 := tb.debug() 86 | require.Zero(t, d2.Stats.SingleReqs-d1.Stats.SingleReqs) 87 | require.Zero(t, d2.Stats.BatchReqs-d1.Stats.BatchReqs) 88 | require.Zero(t, d2.Stats.DiffReqs-d1.Stats.DiffReqs) 89 | } 90 | -------------------------------------------------------------------------------- /daemon/diff_test.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/dnr/styx/common/cdig" 7 | "github.com/dnr/styx/common/shift" 8 | "github.com/dnr/styx/pb" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | var d1 = "0123456789ytrewq6789poiu" 13 | var d2 = "hjkl30104mnop410019jjkka" 14 | var d3 = "zxcv95345asdfiijb632ooia" 15 | var testCShift shift.Shift = 16 // TODO: test with different sizes 16 | var testEntries = []*pb.Entry{ 17 | &pb.Entry{ 18 | Path: "/", 19 | }, 20 | &pb.Entry{ 21 | Path: "/inline", 22 | Size: 10, 23 | InlineData: []byte("hello file"), 24 | }, 25 | &pb.Entry{ 26 | Path: "/file", 27 | Size: testCShift.Size() + 100, 28 | Digests: []byte(d1 + d2), 29 | }, 30 | &pb.Entry{ 31 | Path: "/dir", 32 | }, 33 | &pb.Entry{ 34 | Path: "/dir/nextfile", 35 | Size: 500, 36 | Digests: []byte(d3), 37 | }, 38 | &pb.Entry{ 39 | Path: "/symlink", 40 | Size: 7, 41 | InlineData: []byte("target!"), 42 | }, 43 | &pb.Entry{ 44 | Path: "/lastfile", 45 | Size: testCShift.Size()*2 + 300, 46 | Digests: []byte(d3 + d2 + d1), 47 | }, 48 | } 49 | 50 | func fb(s string) cdig.CDig { 51 | return cdig.FromBytes([]byte(s)) 52 | } 53 | 54 | func TestDigestIterator(t *testing.T) { 55 | r := require.New(t) 56 | i := newDigestIterator(testEntries) 57 | 58 | r.Equal(testEntries[2], i.ent()) 59 | r.Equal(fb(d1), i.digest()) 60 | r.EqualValues(testCShift.Size(), i.size()) 61 | 62 | r.NotNil(i.next(1)) 63 | r.Equal(testEntries[2], i.ent()) 64 | r.Equal(fb(d2), i.digest()) 65 | r.EqualValues(100, i.size()) 66 | 67 | r.NotNil(i.next(1)) 68 | r.Equal(testEntries[4], i.ent()) 69 | r.Equal(fb(d3), i.digest()) 70 | r.EqualValues(500, i.size()) 71 | 72 | r.NotNil(i.next(1)) 73 | r.Equal(testEntries[6], i.ent()) 74 | r.Equal(fb(d3), i.digest()) 75 | r.EqualValues(testCShift.Size(), i.size()) 76 | 77 | r.NotNil(i.next(2)) 78 | r.Equal(testEntries[6], i.ent()) 79 | r.Equal(fb(d1), i.digest()) 80 | r.EqualValues(300, i.size()) 81 | 82 | r.Nil(i.next(1)) 83 | } 84 | 85 | func TestDigestIterator_FindFile(t *testing.T) { 86 | r := require.New(t) 87 | i := newDigestIterator(testEntries) 88 | 89 | r.True(i.findFile("/dir/nextfile")) 90 | r.Equal(testEntries[4], i.ent()) 91 | r.Equal(fb(d3), i.digest()) 92 | r.EqualValues(500, i.size()) 93 | 94 | r.False(i.findFile("/symlink"), "false even though file is present") 95 | r.True(i.findFile("/lastfile")) 96 | r.False(i.findFile("/dir/nextfile"), "doesn't go backwards") 97 | } 98 | 99 | func TestDigestIterator_ToFileStart(t *testing.T) { 100 | r := require.New(t) 101 | i := newDigestIterator(testEntries) 102 | 103 | r.NotNil(i.next(4)) 104 | r.Equal(testEntries[6], i.ent()) 105 | r.Equal(fb(d2), i.digest()) 106 | r.EqualValues(testCShift.Size(), i.size()) 107 | 108 | r.True(i.toFileStart()) 109 | r.Equal(testEntries[6], i.ent()) 110 | r.Equal(fb(d3), i.digest()) 111 | r.EqualValues(testCShift.Size(), i.size()) 112 | } 113 | -------------------------------------------------------------------------------- /cmd/styx/tarball.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | "os/exec" 8 | "strconv" 9 | "strings" 10 | "syscall" 11 | 12 | "github.com/dnr/styx/common/client" 13 | "github.com/dnr/styx/daemon" 14 | "github.com/spf13/cobra" 15 | ) 16 | 17 | type tarballArgs struct { 18 | outlink string 19 | printOnly bool 20 | shards int 21 | } 22 | 23 | var tarballCmd = cmd( 24 | &cobra.Command{ 25 | Use: "tarball ", 26 | Short: "'substitute' a generic tarball to the local store", 27 | Args: cobra.ExactArgs(1), 28 | }, 29 | withStyxClient, 30 | func(c *cobra.Command) runE { 31 | var args tarballArgs 32 | c.Flags().StringVarP(&args.outlink, "out-link", "o", "", "symlink this to output and register as nix root") 33 | c.Flags().BoolVarP(&args.printOnly, "print", "p", false, "print nix code for fixed-output derivation") 34 | c.Flags().IntVar(&args.shards, "shards", 0, "split up manifesting") 35 | return storer(&args) 36 | }, 37 | func(c *cobra.Command, args []string) error { 38 | cli := get[*client.StyxClient](c) 39 | targs := get[*tarballArgs](c) 40 | 41 | // ask daemon to ask manifester to ingest this tarball 42 | var resp daemon.TarballResp 43 | status, err := cli.Call(daemon.TarballPath, &daemon.TarballReq{ 44 | UpstreamUrl: args[0], 45 | Shards: targs.shards, 46 | }, &resp) 47 | if err != nil { 48 | fmt.Println("call error:", err) 49 | return err 50 | } else if status != http.StatusOK { 51 | fmt.Println("status:", status) 52 | } 53 | 54 | // if running in sudo, try to run commands as original user 55 | var spa syscall.SysProcAttr 56 | uid, erru := strconv.Atoi(os.Getenv("SUDO_UID")) 57 | gid, errg := strconv.Atoi(os.Getenv("SUDO_GID")) 58 | if erru == nil && errg == nil { 59 | spa.Credential = &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)} 60 | } 61 | 62 | if strings.ContainsFunc(resp.Name, func(r rune) bool { 63 | return r < 32 || r > 126 || r == '$' || r == '/' || r == '\\' || r == '"' 64 | }) { 65 | return fmt.Errorf("invalid name %q", resp.Name) 66 | } 67 | 68 | // we want to tell nix to just "substitute this", but it needs a derivation, so 69 | // construct a minimal derivation. this is not buildable, it just has the right 70 | // store path and is a FOD. the styx daemon acts as a "binary cache" for this store 71 | // path, so nix will ask styx to do the substitution. 72 | derivationStr := fmt.Sprintf(`derivation { 73 | name = "%s"; 74 | system = builtins.currentSystem; 75 | builder = "/bin/false"; 76 | outputHash = "%s:%s"; 77 | outputHashMode = "recursive"; 78 | }`, resp.Name, resp.NarHashAlgo, resp.NarHash) 79 | 80 | if targs.printOnly { 81 | fmt.Println(derivationStr) 82 | return nil 83 | } 84 | 85 | // realize the derivation 86 | cmd := exec.CommandContext(c.Context(), "nix-build", "-E", derivationStr, "--no-fallback") 87 | if targs.outlink != "" { 88 | cmd.Args = append(cmd.Args, "--out-link", targs.outlink) 89 | } else { 90 | cmd.Args = append(cmd.Args, "--no-out-link") 91 | } 92 | cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr 93 | cmd.SysProcAttr = &spa 94 | return cmd.Run() 95 | }, 96 | ) 97 | -------------------------------------------------------------------------------- /tests/gc_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | 7 | "github.com/dnr/styx/daemon" 8 | "github.com/dnr/styx/pb" 9 | "github.com/stretchr/testify/require" 10 | "golang.org/x/sys/unix" 11 | ) 12 | 13 | var gcUnmounted = map[pb.MountState]bool{ 14 | pb.MountState_Unmounted: true, 15 | } 16 | 17 | func TestGc(t *testing.T) { 18 | tb := newTestBase(t) 19 | tb.startAll() 20 | 21 | mp1 := tb.mount("xpq4yhadyhazkcsggmqd7rsgvxb3kjy4-gnugrep-3.11") 22 | tb.nixHash(mp1) 23 | mp2 := tb.mount("qa22bifihaxyvn6q2a6w9m0nklqrk9wh-opusfile-0.12") 24 | tb.nixHash(mp2) 25 | mp3 := tb.mount("8vyj9c6g424mz0v3kvzkskhvzhwj6288-bash-interactive-5.2-p15-man") 26 | tb.nixHash(mp3) 27 | mp4 := tb.mount("xd96wmj058ky40aywv72z63vdw9yzzzb-openssl-3.0.12-man") 28 | tb.nixHash(mp4) 29 | mp5 := tb.mount("3a7xq2qhxw2r7naqmc53akmx7yvz0mkf-less-is-more.patch") 30 | tb.nixHash(mp5) 31 | 32 | // unmount 2 and 4 33 | tb.umount("qa22bifihaxyvn6q2a6w9m0nklqrk9wh") 34 | tb.umount("xd96wmj058ky40aywv72z63vdw9yzzzb") 35 | 36 | gc1 := tb.gc(daemon.GcReq{DryRunFast: true, GcByState: gcUnmounted}) 37 | t.Log("gc1:", gc1) 38 | require.Equal(t, 2, gc1.DeleteImages) 39 | require.Equal(t, 3, gc1.RemainImages) 40 | require.Equal(t, 806, gc1.DeleteChunks) 41 | require.Equal(t, 53, gc1.RemainRefChunks) 42 | require.Equal(t, 53, gc1.RemainHaveChunks) 43 | require.Equal(t, 0, gc1.RewriteChunks) 44 | 45 | gc2 := tb.gc(daemon.GcReq{DryRunSlow: true, GcByState: gcUnmounted}) 46 | t.Log("gc2:", gc2) 47 | require.Equal(t, 3, gc2.PunchLocs) 48 | require.Equal(t, int64(4239360), gc2.PunchBytes) 49 | 50 | gc3 := tb.gc(daemon.GcReq{GcByState: gcUnmounted}) 51 | t.Log("gc3:", gc3) 52 | require.Equal(t, int64(4239360), gc3.PunchBytes) 53 | 54 | tb.dropCaches() 55 | 56 | // re-read remaining ones 57 | d1 := tb.debug() 58 | tb.nixHash(mp1) 59 | tb.nixHash(mp3) 60 | tb.nixHash(mp5) 61 | d2 := tb.debug() 62 | require.Zero(t, d2.Stats.Sub(d1.Stats).TotalReqs()) 63 | } 64 | 65 | // randomized test 66 | func TestGcShared(t *testing.T) { 67 | tb := newTestBase(t) 68 | tb.startAll() 69 | 70 | fetch := []string{ 71 | "cd1nbildgzzfryjg82njnn36i4ynyf8h-bash-interactive-5.1-p16-man", 72 | "8vyj9c6g424mz0v3kvzkskhvzhwj6288-bash-interactive-5.2-p15-man", 73 | "v35ysx9k1ln4c6r7lj74204ss4bw7l5l-openssl-3.0.12-man", 74 | "xd96wmj058ky40aywv72z63vdw9yzzzb-openssl-3.0.12-man", 75 | "z2waz77lsh4pxs0jxgmpf16s7a3g7b7v-openssl-3.0.13-man", 76 | "53qwclnym7a6vzs937jjmsfqxlxlsf2y-opusfile-0.12", 77 | "kcyrz2y8si9ry5p8qkmj0gp41n01sa1y-opusfile-0.12", 78 | "qa22bifihaxyvn6q2a6w9m0nklqrk9wh-opusfile-0.12", 79 | } 80 | // mount all 81 | mps := make([]string, len(fetch)) 82 | for _, j := range rand.Perm(len(mps)) { 83 | mps[j] = tb.mount(fetch[j]) 84 | } 85 | // read all (different order) 86 | for _, j := range rand.Perm(len(mps)) { 87 | tb.nixHash(mps[j]) 88 | } 89 | // unmount half 90 | var remaining []string 91 | for i, j := range rand.Perm(len(mps)) { 92 | if i < len(mps)/2 { 93 | tb.umount(fetch[j]) 94 | } else { 95 | remaining = append(remaining, mps[j]) 96 | } 97 | } 98 | mps = remaining 99 | 100 | gc := tb.gc(daemon.GcReq{GcByState: gcUnmounted}) 101 | t.Log("gc:", gc) 102 | 103 | unix.Sync() 104 | tb.dropCaches() 105 | unix.Sync() 106 | 107 | // re-read remaining ones 108 | d1 := tb.debug() 109 | for _, mp := range mps { 110 | tb.nixHash(mp) 111 | } 112 | d2 := tb.debug() 113 | // no errors, no requests 114 | require.Zero(t, d2.Stats.Sub(d1.Stats).TotalReqs()) 115 | } 116 | -------------------------------------------------------------------------------- /ci/tclient.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "errors" 7 | "log/slog" 8 | "maps" 9 | "slices" 10 | "strings" 11 | 12 | commonpb "go.temporal.io/api/common/v1" 13 | "go.temporal.io/sdk/client" 14 | "go.temporal.io/sdk/converter" 15 | "go.temporal.io/sdk/log" 16 | "go.temporal.io/sdk/temporal" 17 | "google.golang.org/grpc" 18 | "google.golang.org/grpc/metadata" 19 | 20 | "github.com/dnr/styx/common" 21 | ) 22 | 23 | func getDataConverter() converter.DataConverter { 24 | return converter.NewCodecDataConverter(converter.GetDefaultDataConverter(), zstdcodec{}) 25 | } 26 | 27 | func getTemporalClient(ctx context.Context, paramSrc string) (client.Client, string, error) { 28 | params, err := getParams(paramSrc) 29 | if err != nil { 30 | return nil, "", err 31 | } 32 | parts := strings.SplitN(params, "~", 3) 33 | if len(parts) < 3 { 34 | return nil, "", errors.New("bad params format") 35 | } 36 | hostPort, namespace, apiKey := parts[0], parts[1], parts[2] 37 | 38 | dc := getDataConverter() 39 | fc := temporal.NewDefaultFailureConverter(temporal.DefaultFailureConverterOptions{DataConverter: dc}) 40 | 41 | co := client.Options{ 42 | HostPort: hostPort, 43 | Namespace: namespace, 44 | DataConverter: dc, 45 | FailureConverter: fc, 46 | Logger: log.NewStructuredLogger(slog.Default()), 47 | } 48 | if apiKey != "" { 49 | co.Credentials = client.NewAPIKeyStaticCredentials(apiKey) 50 | // TODO: remove after go sdk does this automatically 51 | co.ConnectionOptions = client.ConnectionOptions{ 52 | TLS: &tls.Config{}, 53 | DialOptions: []grpc.DialOption{ 54 | grpc.WithUnaryInterceptor( 55 | func(ctx context.Context, method string, req any, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { 56 | return invoker( 57 | metadata.AppendToOutgoingContext(ctx, "temporal-namespace", namespace), 58 | method, 59 | req, 60 | reply, 61 | cc, 62 | opts..., 63 | ) 64 | }, 65 | ), 66 | }, 67 | } 68 | } 69 | c, err := client.DialContext(ctx, co) 70 | return c, namespace, err 71 | } 72 | 73 | type zstdcodec struct{} 74 | 75 | func (zstdcodec) Encode(payloads []*commonpb.Payload) ([]*commonpb.Payload, error) { 76 | z := common.GetZstdCtxPool().Get() 77 | defer common.GetZstdCtxPool().Put(z) 78 | out := slices.Clone(payloads) 79 | for i, p := range payloads { 80 | zd, err := z.Compress(nil, p.Data) 81 | if err != nil { 82 | return nil, err 83 | } 84 | if len(zd)+24 >= len(p.Data) { 85 | continue 86 | } 87 | np := &commonpb.Payload{ 88 | Metadata: maps.Clone(p.Metadata), 89 | Data: zd, 90 | } 91 | if np.Metadata == nil { 92 | np.Metadata = make(map[string][]byte) 93 | } 94 | np.Metadata["styx/cmp"] = []byte("zst") 95 | payloads[i] = np 96 | } 97 | return out, nil 98 | } 99 | 100 | func (zstdcodec) Decode(payloads []*commonpb.Payload) ([]*commonpb.Payload, error) { 101 | z := common.GetZstdCtxPool().Get() 102 | defer common.GetZstdCtxPool().Put(z) 103 | out := slices.Clone(payloads) 104 | for i, p := range payloads { 105 | cmp := string(p.Metadata["styx/cmp"]) 106 | if cmp != "zstd" && cmp != "zst" { 107 | continue 108 | } 109 | d, err := z.Decompress(nil, p.Data) 110 | if err != nil { 111 | return nil, err 112 | } 113 | np := &commonpb.Payload{ 114 | Metadata: maps.Clone(p.Metadata), 115 | Data: d, 116 | } 117 | delete(np.Metadata, "styx/cmp") 118 | payloads[i] = np 119 | } 120 | return out, nil 121 | } 122 | -------------------------------------------------------------------------------- /ci/config/default.nix: -------------------------------------------------------------------------------- 1 | # Stripped-down version of my (dnr)'s basic nix config. 2 | # If you're actually using this and want more stuff in here, let me know. 3 | 4 | { config, pkgs, ... }: 5 | { 6 | imports = [ 7 | 8 | ]; 9 | 10 | # builds custom kernel, patched nix, styx binary 11 | services.styx.enable = true; 12 | 13 | # build with latest kernel to get >= 6.8 14 | boot.kernelPackages = pkgs.linuxPackages_latest; 15 | 16 | # some kernel modules that I use that depend on the custom kernel: 17 | boot.extraModulePackages = with config.boot.kernelPackages; [ 18 | acpi_call 19 | v4l2loopback 20 | ]; 21 | 22 | # just enough to make nix-build not complain: 23 | fileSystems."/".device = "/dev/dummy"; 24 | boot.loader.grub.device = "nodev"; 25 | 26 | hardware.enableRedistributableFirmware = true; 27 | networking.networkmanager.enable = true; 28 | nixpkgs.config.allowUnfree = true; 29 | 30 | environment.systemPackages = with pkgs; [ 31 | 32 | ascii 33 | awscli2 34 | bc 35 | binutils-unwrapped 36 | borgbackup 37 | brotli 38 | btrfs-progs 39 | compsize 40 | cryptsetup 41 | curl 42 | darktable 43 | ddcutil 44 | diffstat 45 | difftastic 46 | diffutils 47 | direnv 48 | dmenu 49 | docker 50 | docker-compose 51 | dragon-drop 52 | dunst 53 | easyeffects 54 | evince 55 | ffmpeg 56 | file 57 | gcc 58 | gdb 59 | gh 60 | gimp 61 | git 62 | git-absorb 63 | gnome-icon-theme 64 | gnupg 65 | go 66 | gocryptfs 67 | (google-chrome.override { speechd-minimal = snappy; }) # hack to avoid bringing in speech deps. non-redistributable? 68 | guvcview 69 | hdparm 70 | hugin 71 | imagemagick 72 | jq 73 | jujutsu 74 | libnotify 75 | libsecret 76 | lm_sensors 77 | lsof 78 | ltrace 79 | magic-wormhole 80 | moreutils 81 | mplayer 82 | mpv 83 | nil 84 | nix-direnv 85 | nixfmt-rfc-style 86 | nixos-option 87 | nix-output-monitor 88 | nodejs 89 | notion 90 | nvme-cli 91 | obs-studio 92 | openssh 93 | openssl 94 | opusTools 95 | pavucontrol 96 | pciutils 97 | pipewire 98 | protobuf 99 | protoc-gen-go 100 | psmisc 101 | pulseaudio 102 | pv 103 | python3 104 | redshift 105 | ripgrep 106 | rsync 107 | screen 108 | scrot 109 | signal-desktop 110 | skopeo 111 | smem 112 | socat 113 | spacer 114 | spotify # non-redistributable? 115 | sqlite 116 | starship 117 | strace 118 | sxiv 119 | sysstat 120 | tcpdump 121 | terraform # non-redistributable? 122 | tig 123 | tree 124 | unzip 125 | usbutils 126 | v4l-utils 127 | vim 128 | wget 129 | wireguard-tools 130 | wireplumber 131 | xdotool 132 | xosd 133 | xsel 134 | xsettingsd 135 | xxd 136 | xz 137 | yt-dlp 138 | zip 139 | zoom-us # non-redistributable? 140 | zoxide 141 | zstd 142 | 143 | ]; 144 | 145 | services.fprintd.enable = true; 146 | services.fwupd.enable = true; 147 | services.tlp.enable = true; 148 | services.xserver.enable = true; 149 | services.zerotierone.enable = true; # non-redistributable? 150 | 151 | fonts.packages = [ 152 | pkgs.noto-fonts 153 | pkgs.noto-fonts-cjk-sans 154 | pkgs.noto-fonts-color-emoji 155 | pkgs.ubuntu-classic 156 | pkgs.nerd-fonts.ubuntu-mono 157 | ]; 158 | 159 | documentation.nixos.enable = false; 160 | 161 | system.stateVersion = "25.11"; 162 | } 163 | -------------------------------------------------------------------------------- /ci/notify.go: -------------------------------------------------------------------------------- 1 | package ci 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "regexp" 8 | "slices" 9 | "strings" 10 | "text/template" 11 | "time" 12 | 13 | "github.com/wneessen/go-mail" 14 | "golang.org/x/text/message" 15 | ) 16 | 17 | func (a *activities) Notify(ctx context.Context, req *notifyReq) error { 18 | params, err := getParams(a.cfg.SmtpParams) 19 | if err != nil { 20 | return err 21 | } 22 | parts := strings.SplitN(params, "~", 3) 23 | if len(parts) < 3 { 24 | return errors.New("bad params format") 25 | } 26 | 27 | hostPort, user, pass := parts[0], parts[1], parts[2] 28 | host := strings.Split(hostPort, ":")[0] 29 | 30 | now := time.Now() 31 | 32 | m := mail.NewMsg() 33 | if err := m.FromFormat("Charon CI", user); err != nil { 34 | return err 35 | } else if err = m.To(user); err != nil { 36 | return err 37 | } 38 | subj := "charon build " + now.Format(time.Stamp) + " " 39 | if req.Error != "" { 40 | subj += "error" 41 | } else { 42 | subj += "success" 43 | } 44 | m.Subject(subj) 45 | m.SetDateWithValue(now) 46 | 47 | args := map[string]any{ 48 | "req": req, 49 | "diff": makeDiff(req.PrevNames, req.NewNames), 50 | } 51 | err = m.SetBodyTextTemplate( 52 | template.Must(template.New("email").Funcs(map[string]any{ 53 | "fmt": message.NewPrinter(message.MatchLanguage("en")).Sprint, 54 | }).Parse(` 55 | {{if .req.Error}} 56 | Charon build error: 57 | 58 | {{.req.Error}} 59 | 60 | {{with .req.ErrorDetails}} 61 | Logs: 62 | {{.Logs}} 63 | {{end}} 64 | 65 | {{else}} 66 | Charon build complete! 67 | 68 | Nix channel: {{.req.RelID}} 69 | Styx commit: {{slice .req.StyxCommit 0 8}} 70 | 71 | Elapsed time: {{.req.BuildElapsed | fmt}} 72 | Manifests: {{.req.ManifestStats.Manifests | fmt}} 73 | Chunks: {{.req.ManifestStats.NewChunks | fmt}} new ⁄ {{.req.ManifestStats.TotalChunks | fmt}} total 74 | Bytes: {{.req.ManifestStats.NewUncmpBytes | fmt}} new ⁄ {{.req.ManifestStats.TotalUncmpBytes | fmt}} total 75 | Compressed bytes: {{.req.ManifestStats.NewCmpBytes | fmt}} new 76 | 77 | Packages: 78 | {{range .diff -}} 79 | {{.}} 80 | {{end}} 81 | {{end}} 82 | {{if .req.GCSummary}} 83 | 84 | GC summary: 85 | {{.req.GCSummary}} 86 | {{end}} 87 | `)), args) 88 | if err != nil { 89 | return err 90 | } 91 | 92 | c, err := mail.NewClient(host, mail.WithSSLPort(false), mail.WithSMTPAuth(mail.SMTPAuthPlain), 93 | mail.WithUsername(user), mail.WithPassword(pass)) 94 | if err != nil { 95 | return err 96 | } 97 | return c.DialAndSendWithContext(ctx, m) 98 | } 99 | 100 | func makeDiff(a, b []string) []string { 101 | as := splitNames(a) 102 | bs := splitNames(b) 103 | var out []string 104 | for bk, bv := range bs { 105 | if av, ok := as[bk]; ok { 106 | if av != bv { 107 | out = append(out, fmt.Sprintf("%s: %s ⮞ %s", bk, av, bv)) 108 | } 109 | } else { 110 | if bv != "" { 111 | out = append(out, fmt.Sprintf("%s: ∅ ⮞ %s", bk, bv)) 112 | } 113 | } 114 | } 115 | for ak, av := range as { 116 | if _, ok := bs[ak]; !ok { 117 | if av != "" { 118 | out = append(out, fmt.Sprintf("%s: %s ⮞ ∅", ak, av)) 119 | } 120 | } 121 | } 122 | slices.Sort(out) 123 | return out 124 | } 125 | 126 | func splitNames(ns []string) map[string]string { 127 | out := make(map[string]string) 128 | for _, n := range ns { 129 | m := _splitRe.FindStringSubmatch(n) 130 | if len(m) == 0 { 131 | continue 132 | } 133 | // ignore dups, just overwrite 134 | out[m[1]] = m[2] 135 | } 136 | return out 137 | } 138 | 139 | var _splitRe = regexp.MustCompile(`^(.+?)-([0-9].*?)(-man|-bin|-lib|-libgcc|-dev|-doc|-info|-getent|-xz|-zstd|-modules|-modules-shrunk|-drivers|--p11kit|-xxd)?$`) 140 | -------------------------------------------------------------------------------- /erofs/slab_image.go: -------------------------------------------------------------------------------- 1 | package erofs 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha256" 6 | "errors" 7 | "hash/crc32" 8 | 9 | "github.com/dnr/styx/common" 10 | "github.com/dnr/styx/common/shift" 11 | "golang.org/x/sys/unix" 12 | ) 13 | 14 | func SlabImageRead(devid string, slabBytes int64, blkShift shift.Shift, off uint64, buf []byte) error { 15 | if off != 0 { 16 | return errors.New("slab image read must be from start") 17 | } else if len(buf) < 4096 { 18 | return errors.New("slab image read must be at least 4k") 19 | } 20 | 21 | const incompat = EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 22 | 23 | const rootNid = 20 24 | const slabNid = 24 25 | const mappedBlkAddr = 1 26 | 27 | // setup super 28 | super := erofs_super_block{ 29 | Magic: EROFS_MAGIC, 30 | FeatureIncompat: incompat, 31 | BlkSzBits: common.TruncU8(blkShift), 32 | RootNid: common.TruncU16(rootNid), 33 | Inos: common.TruncU64(2), 34 | Blocks: common.TruncU32(1), 35 | MetaBlkAddr: common.TruncU32(0), 36 | ExtraDevices: common.TruncU16(1), 37 | DevtSlotOff: (EROFS_SUPER_OFFSET + EROFS_SUPER_SIZE) / EROFS_DEVT_SLOT_SIZE, 38 | } 39 | copy(super.VolumeName[:], "@"+devid) 40 | h := sha256.New() 41 | h.Write(super.VolumeName[:]) 42 | var hsum [sha256.Size]byte 43 | copy(super.Uuid[:], h.Sum(hsum[:])) 44 | 45 | c := crc32.NewIEEE() 46 | pack(c, &super) 47 | super.Checksum = c.Sum32() 48 | 49 | // setup dirents 50 | const numDirents = 3 51 | dirents := [numDirents]erofs_dirent{ 52 | {Nid: rootNid, NameOff: numDirents*12 + 0, FileType: EROFS_FT_DIR}, // "." 53 | {Nid: rootNid, NameOff: numDirents*12 + 1, FileType: EROFS_FT_DIR}, // ".." 54 | {Nid: slabNid, NameOff: numDirents*12 + 3, FileType: EROFS_FT_REG_FILE}, // "slab" 55 | } 56 | const direntNames = "...slab" 57 | const direntSize = len(dirents)*12 + len(direntNames) 58 | 59 | // writes to out will fill in buf 60 | out := bytes.NewBuffer(buf[:0:len(buf)]) 61 | // offset 0 62 | pad(out, rootNid<> blkShift), 106 | MappedBlkAddr: common.TruncU32(mappedBlkAddr), 107 | } 108 | copy(dev.Tag[:], devid) 109 | pack(out, dev) 110 | // offset 1280 111 | 112 | // fill in rest with zero 113 | pad(out, int64(out.Available())) 114 | return nil 115 | } 116 | -------------------------------------------------------------------------------- /module/default.nix: -------------------------------------------------------------------------------- 1 | { 2 | config, 3 | lib, 4 | pkgs, 5 | ... 6 | }: 7 | let 8 | styx = import ../. { inherit pkgs; }; 9 | cfg = config.services.styx; 10 | in 11 | with lib; 12 | { 13 | options = { 14 | services.styx = { 15 | enable = mkEnableOption "Styx storage manager for Nix"; 16 | enablePatchedNix = mkEnableOption "Patched Nix for Styx"; 17 | enableNixSettings = mkEnableOption "nix.conf settings for Styx"; 18 | enableStyxNixCache = mkEnableOption "Add binary cache for Styx and related packages"; 19 | enableKernelOptions = mkEnableOption "Enable required kernel config for Styx (erofs+cachefiles)"; 20 | package = mkOption { 21 | description = "Styx package"; 22 | type = types.package; 23 | default = styx.styx-local; 24 | }; 25 | }; 26 | }; 27 | 28 | config = mkMerge [ 29 | (mkIf (cfg.enable || cfg.enablePatchedNix) { 30 | nix.package = styx.patchedNix; 31 | }) 32 | 33 | (mkIf (cfg.enable || cfg.enableNixSettings) { 34 | nix.settings = { 35 | substituters = mkForce [ 36 | # Add "?styx=1" to default substituter. 37 | # TODO: can we do this in overlay style to filter the previous value? 38 | "https://cache.nixos.org/?styx=1" 39 | # styx serves narinfo on localhost for fods only 40 | "http://localhost:7444/?styx=1" 41 | ]; 42 | styx-ondemand = [ ]; 43 | styx-materialize = [ ]; 44 | }; 45 | }) 46 | 47 | (mkIf (cfg.enable || cfg.enableStyxNixCache) { 48 | nix.settings = { 49 | # Use binary cache to avoid rebuilds: 50 | extra-substituters = [ "https://styx-1.s3.amazonaws.com/nixcache/?styx=1" ]; 51 | extra-trusted-public-keys = [ "styx-nixcache-test-1:IbJB9NG5antB2WpE+aE5QzmXapT2yLQb8As/FRkbm3Q=" ]; 52 | }; 53 | }) 54 | 55 | (mkIf (cfg.enable || cfg.enableKernelOptions) { 56 | # Need to turn on these kernel config options: 57 | assertions = [ 58 | { 59 | assertion = lib.versionAtLeast config.boot.kernelPackages.kernel.version "6.8"; 60 | message = "Styx requires at least a 6.8 kernel"; 61 | } 62 | ]; 63 | boot.kernelPatches = [ 64 | { 65 | name = "styx"; 66 | patch = null; 67 | structuredExtraConfig = { 68 | CACHEFILES_ONDEMAND = lib.kernel.yes; 69 | EROFS_FS_ONDEMAND = lib.kernel.yes; 70 | }; 71 | } 72 | ]; 73 | }) 74 | 75 | (mkIf cfg.enable { 76 | # Tag configuration so we can easily find a non-Styx config if we broke everything. 77 | system.nixos.tags = [ "styx" ]; 78 | 79 | # expose cli 80 | environment.systemPackages = [ 81 | cfg.package 82 | styx.StyxInitTest1 83 | ]; 84 | 85 | # main service 86 | systemd.services.styx = { 87 | description = "Nix storage manager"; 88 | wantedBy = [ "sysinit.target" ]; 89 | before = [ 90 | "sysinit.target" 91 | "shutdown.target" 92 | ]; 93 | conflicts = [ "shutdown.target" ]; 94 | requires = [ 95 | "modprobe@cachefiles.service" 96 | ]; 97 | after = [ 98 | "local-fs.target" 99 | "modprobe@cachefiles.service" 100 | ]; 101 | # TODO: restartTriggers? 102 | unitConfig = { 103 | DefaultDependencies = false; 104 | RequiresMountsFor = [ 105 | "/var/cache/styx" 106 | "/nix/store" 107 | ]; 108 | }; 109 | serviceConfig = { 110 | # Use unshare directly instead of PrivateMounts so that our new mounts 111 | # are propagated normally, but we can remount /nix/store rw. 112 | ExecStart = "${pkgs.util-linux}/bin/unshare -m --propagation unchanged ${cfg.package}/bin/styx daemon"; 113 | SyslogIdentifier = "styx"; 114 | Type = "notify"; 115 | NotifyAccess = "all"; 116 | FileDescriptorStoreMax = "1"; 117 | FileDescriptorStorePreserve = "yes"; 118 | LimitNOFILE = "500000"; 119 | Restart = "on-failure"; 120 | }; 121 | }; 122 | }) 123 | ]; 124 | } 125 | -------------------------------------------------------------------------------- /daemon/proto.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "go.etcd.io/bbolt" 5 | 6 | "github.com/dnr/styx/pb" 7 | ) 8 | 9 | var ( 10 | // protocol is json over http over unix socket 11 | // socket is path.Join(CachePath, Socket) 12 | // accessible to root only! 13 | Socket = "styx.sock" 14 | InitPath = "/init" 15 | MountPath = "/mount" 16 | UmountPath = "/umount" 17 | MaterializePath = "/materialize" 18 | VaporizePath = "/vaporize" 19 | PrefetchPath = "/prefetch" 20 | TarballPath = "/tarball" 21 | GcPath = "/gc" 22 | DebugPath = "/debug" 23 | RepairPath = "/repair" 24 | ) 25 | 26 | type ( 27 | InitReq struct { 28 | PubKeys []string 29 | Params pb.DaemonParams 30 | } 31 | // returns Status 32 | 33 | MountReq struct { 34 | Upstream string 35 | StorePath string 36 | MountPoint string 37 | NarSize int64 `json:",omitempty"` // optional 38 | } 39 | // returns Status 40 | 41 | UmountReq struct { 42 | StorePath string 43 | } 44 | // returns Status 45 | 46 | MaterializeReq struct { 47 | Upstream string 48 | StorePath string 49 | DestPath string 50 | NarSize int64 `json:",omitempty"` // optional 51 | } 52 | // returns Status 53 | 54 | VaporizeReq struct { 55 | Path string // absolute path to data to vaporize into store (required) 56 | Name string // store path name, defaults to basename of Path 57 | } 58 | // returns Status 59 | 60 | PrefetchReq struct { 61 | // absolute path of file or directory to prefetch (unless using StorePath) 62 | Path string 63 | // optional, if set use this StorePath and consider Path under it 64 | StorePath string 65 | } 66 | // returns Status 67 | 68 | TarballReq struct { 69 | UpstreamUrl string 70 | Shards int 71 | } 72 | TarballResp struct { 73 | ResolvedUrl string 74 | StorePathHash string 75 | Name string 76 | NarHash string 77 | NarHashAlgo string 78 | } 79 | 80 | GcReq struct { 81 | DryRunFast bool 82 | DryRunSlow bool 83 | GcByState map[pb.MountState]bool 84 | } 85 | GcResp struct { 86 | // always filled in 87 | DeleteImagesByState map[pb.MountState]int 88 | RemainImagesByState map[pb.MountState]int 89 | DeleteImages int 90 | RemainImages int 91 | DeleteManifests int // should match DeleteImages 92 | DeleteChunks int 93 | RemainRefChunks int 94 | RemainHaveChunks int // should match RemainRefChunks 95 | RewriteChunks int 96 | 97 | // only filled in on dry-run-slow or real run 98 | PunchLocs int 99 | PunchBytes int64 100 | } 101 | 102 | RepairReq struct { 103 | Presence bool `json:",omitempty"` 104 | Remanifest *MountReq `json:",omitempty"` 105 | } 106 | // returns Status 107 | 108 | DebugReq struct { 109 | IncludeAllImages bool `json:",omitempty"` 110 | IncludeImages []string `json:",omitempty"` // list of base32 sph 111 | 112 | IncludeSlabs bool `json:",omitempty"` 113 | IncludeAllChunks bool `json:",omitempty"` 114 | IncludeChunks []string `json:",omitempty"` // list of base64 digests 115 | 116 | IncludeChunkSharing bool `json:",omitempty"` 117 | } 118 | DebugResp struct { 119 | Params *pb.DbParams 120 | Stats Stats 121 | DbStats bbolt.Stats 122 | Images map[string]DebugImage `json:",omitempty"` 123 | Slabs []*DebugSlabInfo `json:",omitempty"` 124 | Chunks map[string]*DebugChunkInfo `json:",omitempty"` 125 | 126 | ChunkSharingDist map[int]int `json:",omitempty"` 127 | } 128 | DebugSizeStats struct { 129 | TotalChunks int 130 | TotalBlocks int 131 | PresentChunks int 132 | PresentBlocks int 133 | } 134 | DebugSlabInfo struct { 135 | Index uint16 136 | Stats DebugSizeStats 137 | ChunkSizeDist map[uint32]int 138 | } 139 | DebugChunkInfo struct { 140 | Slab uint16 141 | Addr uint32 142 | StorePaths []string 143 | Present bool 144 | } 145 | DebugImage struct { 146 | Image *pb.DbImage 147 | Manifest *pb.Manifest 148 | ManifestChunks []string 149 | Stats DebugSizeStats 150 | } 151 | 152 | Status struct { 153 | Success bool `json:",omitempty"` 154 | Error string `json:",omitempty"` 155 | } 156 | ) 157 | -------------------------------------------------------------------------------- /daemon/stats.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import "sync/atomic" 4 | 5 | type ( 6 | daemonStats struct { 7 | manifestCacheReqs atomic.Int64 // total manifest cache requests 8 | manifestCacheHits atomic.Int64 // requests that got a hit 9 | manifestReqs atomic.Int64 // requests for new manifest 10 | manifestErrs atomic.Int64 // requests for new manifest that got an error 11 | slabReads atomic.Int64 // read requests to slab 12 | slabReadErrs atomic.Int64 // failed read requests to slab 13 | singleReqs atomic.Int64 // chunk request count 14 | singleBytes atomic.Int64 // chunk bytes received (uncompressed) 15 | singleErrs atomic.Int64 // chunk request error count 16 | batchReqs atomic.Int64 // no-base diff request count 17 | batchBytes atomic.Int64 // no-base diff bytes received (compressed) 18 | batchErrs atomic.Int64 // no-base diff request error count 19 | diffReqs atomic.Int64 // with-base diff request count 20 | diffBytes atomic.Int64 // with-base diff bytes received (compressed) 21 | diffErrs atomic.Int64 // with-base diff request error count 22 | recompressReqs atomic.Int64 // reqs with recompression 23 | extraReqs atomic.Int64 // extra read-ahead reqs (beyond 1 per read) 24 | } 25 | 26 | Stats struct { 27 | ManifestCacheReqs int64 // total manifest cache requests 28 | ManifestCacheHits int64 // requests that got a hit 29 | ManifestReqs int64 // requests for new manifest 30 | ManifestErrs int64 // requests for new manifest that got an error 31 | SlabReads int64 // read requests to slab 32 | SlabReadErrs int64 // failed read requests to slab 33 | SingleReqs int64 // chunk request count 34 | SingleBytes int64 // chunk bytes received (uncompressed) 35 | SingleErrs int64 // chunk request error count 36 | BatchReqs int64 // no-base diff request count 37 | BatchBytes int64 // no-base diff bytes received (compressed) 38 | BatchErrs int64 // no-base diff request error count 39 | DiffReqs int64 // with-base diff request count 40 | DiffBytes int64 // with-base diff bytes received (compressed) 41 | DiffErrs int64 // with-base diff request error count 42 | RecompressReqs int64 // reqs with recompression 43 | ExtraReqs int64 // extra read-ahead reqs (beyond 1 per read) 44 | } 45 | ) 46 | 47 | func (s *daemonStats) export() Stats { 48 | return Stats{ 49 | ManifestCacheReqs: s.manifestCacheReqs.Load(), 50 | ManifestCacheHits: s.manifestCacheHits.Load(), 51 | ManifestReqs: s.manifestReqs.Load(), 52 | ManifestErrs: s.manifestErrs.Load(), 53 | SlabReads: s.slabReads.Load(), 54 | SlabReadErrs: s.slabReadErrs.Load(), 55 | SingleReqs: s.singleReqs.Load(), 56 | SingleBytes: s.singleBytes.Load(), 57 | SingleErrs: s.singleErrs.Load(), 58 | BatchReqs: s.batchReqs.Load(), 59 | BatchBytes: s.batchBytes.Load(), 60 | BatchErrs: s.batchErrs.Load(), 61 | DiffReqs: s.diffReqs.Load(), 62 | DiffBytes: s.diffBytes.Load(), 63 | DiffErrs: s.diffErrs.Load(), 64 | RecompressReqs: s.recompressReqs.Load(), 65 | ExtraReqs: s.extraReqs.Load(), 66 | } 67 | } 68 | 69 | func (a Stats) Sub(b Stats) Stats { 70 | return Stats{ 71 | ManifestCacheReqs: a.ManifestCacheReqs - b.ManifestCacheReqs, 72 | ManifestCacheHits: a.ManifestCacheHits - b.ManifestCacheHits, 73 | ManifestReqs: a.ManifestReqs - b.ManifestReqs, 74 | ManifestErrs: a.ManifestErrs - b.ManifestErrs, 75 | SlabReads: a.SlabReads - b.SlabReads, 76 | SlabReadErrs: a.SlabReadErrs - b.SlabReadErrs, 77 | SingleReqs: a.SingleReqs - b.SingleReqs, 78 | SingleBytes: a.SingleBytes - b.SingleBytes, 79 | SingleErrs: a.SingleErrs - b.SingleErrs, 80 | BatchReqs: a.BatchReqs - b.BatchReqs, 81 | BatchBytes: a.BatchBytes - b.BatchBytes, 82 | BatchErrs: a.BatchErrs - b.BatchErrs, 83 | DiffReqs: a.DiffReqs - b.DiffReqs, 84 | DiffBytes: a.DiffBytes - b.DiffBytes, 85 | DiffErrs: a.DiffErrs - b.DiffErrs, 86 | RecompressReqs: a.RecompressReqs - b.RecompressReqs, 87 | ExtraReqs: a.ExtraReqs - b.ExtraReqs, 88 | } 89 | } 90 | 91 | func (a Stats) TotalReqs() int64 { 92 | return a.SingleReqs + a.BatchReqs + a.DiffReqs 93 | } 94 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/dnr/styx 2 | 3 | go 1.24.0 4 | 5 | toolchain go1.24.6 6 | 7 | replace github.com/DataDog/zstd => github.com/dnr/datadog-zstd-go v0.0.0-20250916100506-79345d875ce7 8 | 9 | require ( 10 | github.com/DataDog/zstd v1.5.7 11 | github.com/avast/retry-go/v4 v4.6.0 12 | github.com/aws/aws-lambda-go v1.46.0 13 | github.com/aws/aws-sdk-go-v2 v1.27.2 14 | github.com/aws/aws-sdk-go-v2/config v1.27.3 15 | github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.11 16 | github.com/aws/aws-sdk-go-v2/service/s3 v1.51.0 17 | github.com/aws/aws-sdk-go-v2/service/ssm v1.49.5 18 | github.com/axiomhq/axiom-go v0.21.1 19 | github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40 20 | github.com/nix-community/go-nix v0.0.0-20231219074122-93cb24a86856 21 | github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 22 | github.com/spf13/cobra v1.8.1 23 | github.com/stretchr/testify v1.10.0 24 | github.com/wneessen/go-mail v0.4.2 25 | go.etcd.io/bbolt v1.4.3 26 | go.temporal.io/api v1.43.2 27 | go.temporal.io/sdk v1.32.1 28 | golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e 29 | golang.org/x/sync v0.10.0 30 | golang.org/x/sys v0.36.0 31 | golang.org/x/text v0.18.0 32 | google.golang.org/grpc v1.66.1 33 | google.golang.org/protobuf v1.34.2 34 | ) 35 | 36 | require ( 37 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect 38 | github.com/aws/aws-sdk-go-v2/credentials v1.17.3 // indirect 39 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.1 // indirect 40 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.9 // indirect 41 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.9 // indirect 42 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect 43 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.1 // indirect 44 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect 45 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.1 // indirect 46 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.1 // indirect 47 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.1 // indirect 48 | github.com/aws/aws-sdk-go-v2/service/sso v1.20.0 // indirect 49 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.0 // indirect 50 | github.com/aws/aws-sdk-go-v2/service/sts v1.28.0 // indirect 51 | github.com/aws/smithy-go v1.20.2 // indirect 52 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 53 | github.com/davecgh/go-spew v1.1.1 // indirect 54 | github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect 55 | github.com/felixge/httpsnoop v1.0.4 // indirect 56 | github.com/go-logr/logr v1.4.2 // indirect 57 | github.com/go-logr/stdr v1.2.2 // indirect 58 | github.com/gogo/protobuf v1.3.2 // indirect 59 | github.com/golang/mock v1.6.0 // indirect 60 | github.com/google/go-querystring v1.1.0 // indirect 61 | github.com/google/uuid v1.6.0 // indirect 62 | github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect 63 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect 64 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 65 | github.com/jmespath/go-jmespath v0.4.0 // indirect 66 | github.com/klauspost/compress v1.17.9 // indirect 67 | github.com/klauspost/cpuid/v2 v2.0.9 // indirect 68 | github.com/minio/sha256-simd v1.0.0 // indirect 69 | github.com/mr-tron/base58 v1.2.0 // indirect 70 | github.com/multiformats/go-multihash v0.2.1 // indirect 71 | github.com/multiformats/go-varint v0.0.6 // indirect 72 | github.com/nexus-rpc/sdk-go v0.1.1 // indirect 73 | github.com/pborman/uuid v1.2.1 // indirect 74 | github.com/pmezard/go-difflib v1.0.0 // indirect 75 | github.com/robfig/cron v1.2.0 // indirect 76 | github.com/spaolacci/murmur3 v1.1.0 // indirect 77 | github.com/spf13/pflag v1.0.6 // indirect 78 | github.com/stretchr/objx v0.5.2 // indirect 79 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect 80 | go.opentelemetry.io/otel v1.30.0 // indirect 81 | go.opentelemetry.io/otel/metric v1.30.0 // indirect 82 | go.opentelemetry.io/otel/trace v1.30.0 // indirect 83 | golang.org/x/crypto v0.27.0 // indirect 84 | golang.org/x/net v0.29.0 // indirect 85 | golang.org/x/time v0.3.0 // indirect 86 | google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect 87 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 88 | gopkg.in/yaml.v3 v3.0.1 // indirect 89 | lukechampine.com/blake3 v1.1.6 // indirect 90 | ) 91 | -------------------------------------------------------------------------------- /common/signature.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "crypto/rand" 5 | "fmt" 6 | "os" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/nix-community/go-nix/pkg/narinfo/signature" 11 | "google.golang.org/protobuf/proto" 12 | 13 | "github.com/dnr/styx/pb" 14 | ) 15 | 16 | func LoadPubKeys(keys []string) ([]signature.PublicKey, error) { 17 | var out []signature.PublicKey 18 | for _, pk := range keys { 19 | if k, err := signature.ParsePublicKey(pk); err != nil { 20 | return nil, err 21 | } else { 22 | out = append(out, k) 23 | } 24 | } 25 | return out, nil 26 | } 27 | 28 | func LoadSecretKeys(keyfiles []string) ([]signature.SecretKey, error) { 29 | var out []signature.SecretKey 30 | for _, path := range keyfiles { 31 | if skdata, err := os.ReadFile(path); err != nil { 32 | return nil, err 33 | } else if k, err := signature.LoadSecretKey(string(skdata)); err != nil { 34 | return nil, err 35 | } else { 36 | out = append(out, k) 37 | } 38 | } 39 | return out, nil 40 | } 41 | 42 | // Embedded message must be inline in entry. 43 | func VerifyInlineMessage( 44 | keys []signature.PublicKey, 45 | expectedContext string, 46 | b []byte, 47 | msg proto.Message, 48 | ) error { 49 | entry, _, err := VerifyMessageAsEntry(keys, expectedContext, b) 50 | if err != nil { 51 | return err 52 | } 53 | if entry.Size != int64(len(entry.InlineData)) { 54 | return fmt.Errorf("SignedMessage missing inline data") 55 | } 56 | return proto.Unmarshal(entry.InlineData, msg) 57 | } 58 | 59 | func VerifyMessageAsEntry(keys []signature.PublicKey, expectedContext string, b []byte) (*pb.Entry, *pb.GlobalParams, error) { 60 | if len(keys) == 0 { 61 | return nil, nil, fmt.Errorf("no public keys provided") 62 | } 63 | 64 | var sm pb.SignedMessage 65 | err := proto.Unmarshal(b, &sm) 66 | if err != nil { 67 | return nil, nil, fmt.Errorf("error unmarshaling SignedMessage: %w", err) 68 | } else if sm.Msg == nil { 69 | return nil, nil, fmt.Errorf("SignedMessage missing entry") 70 | } else if sm.Msg.Path != expectedContext && !strings.HasPrefix(sm.Msg.Path, expectedContext+"/") { 71 | return nil, nil, fmt.Errorf("SignedMessage context mismatch: %q != %q", sm.Msg.Path, expectedContext) 72 | } else if len(sm.Msg.Digests) > 0 && sm.Params == nil { 73 | return nil, nil, fmt.Errorf("SignedMessage with chunks must have params") 74 | } 75 | 76 | sigs := make([]signature.Signature, min(len(sm.KeyId), len(sm.Signature))) 77 | if len(sigs) == 0 { 78 | return nil, nil, fmt.Errorf("no signatures in SignedMessage") 79 | } 80 | for i := range sigs { 81 | sigs[i].Name = sm.KeyId[i] 82 | sigs[i].Data = sm.Signature[i] 83 | } 84 | 85 | fingerprint := entryFingerprint(sm.Msg) 86 | if !signature.VerifyFirst(fingerprint, sigs, keys) { 87 | return nil, nil, fmt.Errorf("signature verification failed") 88 | } 89 | 90 | return sm.Msg, sm.Params, nil 91 | } 92 | 93 | func SignInlineMessage(keys []signature.SecretKey, context string, msg proto.Message) ([]byte, error) { 94 | b, err := proto.Marshal(msg) 95 | if err != nil { 96 | return nil, fmt.Errorf("error marshaling msg: %w", err) 97 | } 98 | return SignMessageAsEntry(keys, nil, &pb.Entry{ 99 | Path: context, 100 | Type: pb.EntryType_REGULAR, 101 | Size: int64(len(b)), 102 | InlineData: b, 103 | }) 104 | } 105 | 106 | func SignMessageAsEntry(keys []signature.SecretKey, params *pb.GlobalParams, e *pb.Entry) ([]byte, error) { 107 | sm := &pb.SignedMessage{ 108 | Msg: e, 109 | Params: params, 110 | KeyId: make([]string, len(keys)), 111 | Signature: make([][]byte, len(keys)), 112 | } 113 | 114 | fingerprint := entryFingerprint(sm.Msg) 115 | for i, k := range keys { 116 | sig, err := k.Sign(rand.Reader, fingerprint) 117 | if err != nil { 118 | return nil, err 119 | } 120 | sm.KeyId[i] = sig.Name 121 | sm.Signature[i] = sig.Data 122 | } 123 | 124 | return proto.Marshal(sm) 125 | } 126 | 127 | func entryFingerprint(e *pb.Entry) string { 128 | // TODO: do we need to include params here? 129 | var sb strings.Builder 130 | sb.Grow(40 + len(e.Path) + len(e.InlineData) + len(e.Digests)) 131 | sb.WriteString("styx-signed-message-1") 132 | sb.WriteByte(0) 133 | if strings.IndexByte(e.Path, 0) != -1 { 134 | panic("nil in entry path") 135 | } 136 | sb.WriteString(e.Path) 137 | sb.WriteByte(0) 138 | sb.WriteString(strconv.Itoa(int(e.Size))) 139 | if len(e.InlineData) > 0 { 140 | sb.WriteByte(1) 141 | sb.Write(e.InlineData) 142 | } else { 143 | sb.WriteByte(2) 144 | sb.Write(e.Digests) 145 | } 146 | return sb.String() 147 | } 148 | -------------------------------------------------------------------------------- /manifester/proto.go: -------------------------------------------------------------------------------- 1 | package manifester 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/base64" 6 | "fmt" 7 | ) 8 | 9 | const ( 10 | ChunkDiffMaxDigests = 256 11 | ChunkDiffMaxBytes = 16 << 20 12 | ) 13 | 14 | var ( 15 | // protocol is (mostly) json over http 16 | ManifestPath = "/manifest" 17 | ChunkDiffPath = "/chunkdiff" 18 | 19 | // chunk read protocol 20 | ChunkReadPath = "/chunk/" // digest as final path component 21 | ManifestCachePath = "/manifest/" // cache key as final path component 22 | 23 | BuildRootPath = "/buildroot/" // written by manifester, read only by gc 24 | 25 | ExpandGz = "gz" 26 | ExpandXz = "xz" 27 | 28 | // header value is base64(proto-encoded pb.Lengths) 29 | LengthsHeader = "x-styx-lengths" 30 | ) 31 | 32 | type ( 33 | ManifestReq struct { 34 | Upstream string // url of nix binary cache or generic file 35 | StorePathHash string 36 | 37 | // TODO: this should really be a new request type 38 | BuildMode string `json:",omitempty"` 39 | 40 | // TODO: move this to pb and embed a GlobalParams? 41 | DigestAlgo string 42 | DigestBits int 43 | 44 | // used for tarball cache lookups only 45 | ETag string 46 | 47 | // sharded manifesting (not in cache key, only shard 0 writes to cache) 48 | ShardTotal int `json:",omitempty"` 49 | ShardIndex int `json:",omitempty"` 50 | } 51 | // response is SignedManifest 52 | 53 | // deprecated: use pb.ManifesterChunkDiffReq 54 | DeprecatedChunkDiffReq struct { 55 | Bases []byte 56 | Reqs []byte 57 | 58 | // If set: Bases and Reqs each comprise one single file in the given compression 59 | // format. Pass each one through this decompressor before diffing. 60 | ExpandBeforeDiff string `json:",omitempty"` 61 | } 62 | 63 | // Response is compressed concatenation of reqs, using bases as compression base, 64 | // with ChunkDiffStats (json) appended after that (also compressed). 65 | // Bases and Reqs do not need to be the same length. 66 | // (Caller must know the lengths of reqs ahead of time to be able to split the result.) 67 | // Max number of digests in each is 256, and max size of base data or req data is 16 MiB. 68 | // (256 * 64 KiB chunks = 16 MiB, larger chunks have lower limit on digests.) 69 | // Note if running on lambda: after the first 6 MB of streamed data, bandwidth is limited. 70 | // So aim for responses to be < 6 MB. 71 | 72 | // Server will add header named LengthsHeader with lengths of req data for each individual 73 | // request. Note that if not expanding, the caller will already know that information, 74 | // since it knows the size of each requested chunk. If expanding, though, it doesn't know 75 | // the expanded size. If there's only one request, similarly, the caller can determine the 76 | // expanded size since it's the full reconstructed body (minus stats). But for more than 77 | // one request, there's no additional framing, so the length header will be needed to 78 | // separate them. 79 | 80 | // ChunkDiffStats must contain only integers! (For now, since we sometimes scan backwards 81 | // to find the start of the stats. We can relax this requirement if we write a reverse json 82 | // parser.) 83 | ChunkDiffStats struct { 84 | Reqs int `json:"reqs"` 85 | Expands int `json:"exps"` 86 | BaseChunks int `json:"baseC"` 87 | BaseBytes int `json:"baseB"` 88 | ReqChunks int `json:"reqC"` 89 | ReqBytes int `json:"reqB"` 90 | DiffBytes int `json:"diffB"` 91 | DlTotalMs int64 `json:"dlMs"` 92 | ZstdMs int64 `json:"zstdMs"` 93 | } 94 | ) 95 | 96 | func (r *ManifestReq) CacheKey() string { 97 | if (r.BuildMode == ModeGenericTarball) != (r.ETag != "") { 98 | panic("ETag must only be used with ModeGenericTarball") 99 | } else if (r.BuildMode == ModeNar) != (r.StorePathHash != "") { 100 | panic("StorePathHash must only be used with ModeNar") 101 | } 102 | 103 | h := sha256.New() 104 | h.Write([]byte("styx-manifest-cache-v1\n")) 105 | h.Write([]byte(fmt.Sprintf("u=%s\n", r.Upstream))) 106 | switch r.BuildMode { 107 | case ModeNar: 108 | h.Write([]byte(fmt.Sprintf("h=%s\n", r.StorePathHash))) 109 | case ModeGenericTarball: 110 | h.Write([]byte(fmt.Sprintf("m=%s\n", ModeGenericTarball))) 111 | h.Write([]byte(fmt.Sprintf("e=%s\n", r.ETag))) 112 | default: 113 | panic("unknown BuildMode") 114 | } 115 | // use fixed 16 for compatibility, chunk size is now variable 116 | h.Write([]byte(fmt.Sprintf("p=16:%s:%d\n", r.DigestAlgo, r.DigestBits))) 117 | // note: SmallFileCutoff is not part of key, client may get different one from requested 118 | return "v1-" + base64.RawURLEncoding.EncodeToString(h.Sum(nil))[:36] 119 | } 120 | -------------------------------------------------------------------------------- /cmd/charon/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "log/slog" 7 | "time" 8 | 9 | axiom_slog_adapter "github.com/axiomhq/axiom-go/adapters/slog" 10 | "github.com/spf13/cobra" 11 | 12 | "github.com/dnr/styx/ci" 13 | ) 14 | 15 | func withAxiomLogs(c *cobra.Command) runE { 16 | useAxiom := c.Flags().Bool("log_axiom", false, "") 17 | 18 | return func(c *cobra.Command, args []string) error { 19 | if *useAxiom { 20 | h, err := axiom_slog_adapter.New() 21 | if err != nil { 22 | return err 23 | } 24 | c.PostRunE = chainRunE(c.PostRunE, func(*cobra.Command, []string) error { h.Close(); return nil }) 25 | slog.SetDefault(slog.New(h)) 26 | } 27 | return nil 28 | } 29 | } 30 | 31 | func withWorkerConfig(c *cobra.Command) runE { 32 | var cfg ci.WorkerConfig 33 | 34 | c.Flags().StringVar(&cfg.TemporalParams, "temporal_params", "", "source for temporal params") 35 | c.Flags().StringVar(&cfg.SmtpParams, "smtp_params", "", "source for smtp params") 36 | 37 | c.Flags().BoolVar(&cfg.RunWorker, "worker", false, "run temporal workflow+activity worker") 38 | c.Flags().BoolVar(&cfg.RunScaler, "scaler", true, "run scaler on worker") 39 | c.Flags().BoolVar(&cfg.RunHeavyWorker, "heavy", false, "run temporal heavy worker (on ec2)") 40 | 41 | c.Flags().DurationVar(&cfg.ScaleInterval, "scale_interval", time.Minute, "scaler interval") 42 | c.Flags().StringVar(&cfg.AsgGroupName, "scale_group_name", "", "scaler asg group name") 43 | 44 | c.Flags().StringVar(&cfg.CacheSignKeySSM, "cache_signkey_ssm", "", "sign nix cache with key from SSM") 45 | 46 | // manifest builder cfg 47 | c.Flags().StringArrayVar(&cfg.ManifestPubKeys, "nix_pubkey", nil, "verify narinfo with this public key") 48 | c.Flags().StringArrayVar(&cfg.ManifestSignKeySSM, "styx_signkey_ssm", nil, "sign manifest with key from SSM") 49 | 50 | // chunk store write config 51 | c.Flags().StringVar(&cfg.CSWCfg.ChunkBucket, "chunkbucket", "", "s3 bucket to put chunks") 52 | c.Flags().IntVar(&cfg.CSWCfg.ZstdEncoderLevel, "zstd_level", 9, "encoder level for zstd chunks") 53 | 54 | return func(c *cobra.Command, args []string) error { 55 | store(c, cfg) 56 | return nil 57 | } 58 | } 59 | 60 | func withStartConfig(c *cobra.Command) runE { 61 | var cfg ci.StartConfig 62 | 63 | c.Flags().StringVar(&cfg.TemporalParams, "temporal_params", "keys/temporal-creds-charon.secret", "source for temporal params") 64 | 65 | // might use these: 66 | c.Flags().StringVar(&cfg.Args.Channel, "nix_channel", "nixos-25.11", "nix channel to watch/build") 67 | c.Flags().StringVar(&cfg.Args.StyxRepo.Branch, "styx_branch", "release", "branch of styx repo to watch/build") 68 | 69 | // probably don't use these: 70 | const bucket = "styx-1" 71 | const subdir = "nixcache" 72 | const region = "us-east-1" 73 | const level = 9 74 | defCopyDest := fmt.Sprintf("s3://%s/%s/?region=%s&compression=zstd&compression-level=%d", bucket, subdir, region, level) 75 | // note missing region since it's us-east-1. also note trailing slash must be present to match cache key. 76 | defUpstream := fmt.Sprintf("https://%s.s3.amazonaws.com/%s/", bucket, subdir) 77 | c.Flags().StringVar(&cfg.Args.StyxRepo.Repo, "styx_repo", "https://github.com/dnr/styx/", "url of styx repo") 78 | c.Flags().StringVar(&cfg.Args.CopyDest, "copy_dest", defCopyDest, "store path for copying built packages") 79 | c.Flags().StringVar(&cfg.Args.ManifestUpstream, "manifest_upstream", defUpstream, "read-only url for dest store") 80 | c.Flags().StringVar(&cfg.Args.PublicCacheUpstream, "public_upstream", "https://cache.nixos.org/", "read-only url for public cache") 81 | 82 | return func(c *cobra.Command, args []string) error { 83 | store(c, cfg) 84 | return nil 85 | } 86 | } 87 | 88 | func withGCConfig(c *cobra.Command) runE { 89 | var cfg ci.GCConfig 90 | c.Flags().StringVar(&cfg.Bucket, "bucket", "styx-1", "s3 bucket") 91 | c.Flags().DurationVar(&cfg.MaxAge, "max_age", 210*24*time.Hour, "gc age") 92 | return func(c *cobra.Command, args []string) error { 93 | store(c, cfg) 94 | return nil 95 | } 96 | } 97 | 98 | func main() { 99 | root := cmd( 100 | &cobra.Command{ 101 | Use: "charon", 102 | Short: "charon - CI for styx", 103 | }, 104 | cmd( 105 | &cobra.Command{Use: "worker", Short: "act as temporal worker"}, 106 | withAxiomLogs, 107 | withWorkerConfig, 108 | func(c *cobra.Command, args []string) error { 109 | return ci.RunWorker(c.Context(), get[ci.WorkerConfig](c)) 110 | }, 111 | ), 112 | cmd( 113 | &cobra.Command{Use: "start", Short: "start ci workflow"}, 114 | withStartConfig, 115 | func(c *cobra.Command, args []string) error { 116 | return ci.Start(c.Context(), get[ci.StartConfig](c)) 117 | }, 118 | ), 119 | cmd( 120 | &cobra.Command{Use: "gclocal", Short: "run gc from this process (mostly for testing)"}, 121 | withGCConfig, 122 | func(c *cobra.Command, args []string) error { 123 | return ci.GCLocal(c.Context(), get[ci.GCConfig](c)) 124 | }, 125 | ), 126 | ) 127 | if err := root.Execute(); err != nil { 128 | log.Fatal(err) 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /tf/lambda.tf: -------------------------------------------------------------------------------- 1 | 2 | // iam for lambda: 3 | 4 | data "aws_iam_policy_document" "assume_role_lambda" { 5 | statement { 6 | principals { 7 | type = "Service" 8 | identifiers = ["lambda.amazonaws.com"] 9 | } 10 | actions = ["sts:AssumeRole"] 11 | } 12 | } 13 | 14 | data "aws_iam_policy_document" "manifester_get_parameter" { 15 | statement { 16 | actions = ["ssm:GetParameter"] 17 | resources = ["${aws_ssm_parameter.manifester_signkey.arn}"] 18 | } 19 | } 20 | 21 | resource "aws_iam_role" "iam_for_lambda" { 22 | name = "iam_for_lambda_styx" 23 | assume_role_policy = data.aws_iam_policy_document.assume_role_lambda.json 24 | managed_policy_arns = ["arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"] 25 | inline_policy { 26 | name = "get-parameter" 27 | policy = data.aws_iam_policy_document.manifester_get_parameter.json 28 | } 29 | } 30 | 31 | // ecr: 32 | 33 | resource "aws_ecr_repository" "repo" { 34 | name = "styx" 35 | } 36 | 37 | // s3: 38 | 39 | resource "aws_s3_bucket" "styx" { 40 | bucket = "styx-1" 41 | } 42 | 43 | data "aws_iam_policy_document" "styx_bucket_policy" { 44 | statement { 45 | principals { 46 | type = "*" 47 | identifiers = ["*"] 48 | } 49 | actions = ["s3:GetObject"] 50 | resources = [aws_s3_bucket.styx.arn, "${aws_s3_bucket.styx.arn}/*"] 51 | } 52 | statement { 53 | principals { 54 | type = "AWS" 55 | identifiers = [ 56 | aws_iam_role.iam_for_lambda.arn, 57 | aws_iam_role.iam_for_charon.arn, 58 | ] 59 | } 60 | actions = ["s3:*"] 61 | resources = [aws_s3_bucket.styx.arn, "${aws_s3_bucket.styx.arn}/*"] 62 | } 63 | } 64 | 65 | resource "aws_s3_bucket_policy" "styx" { 66 | bucket = aws_s3_bucket.styx.id 67 | policy = data.aws_iam_policy_document.styx_bucket_policy.json 68 | } 69 | 70 | resource "aws_s3_bucket_lifecycle_configuration" "styx" { 71 | bucket = aws_s3_bucket.styx.id 72 | rule { 73 | id = "nixcache-ttl" 74 | status = "Enabled" 75 | filter { 76 | prefix = "nixcache/" 77 | } 78 | abort_incomplete_multipart_upload { days_after_initiation = 1 } 79 | expiration { days = 365 } 80 | } 81 | } 82 | 83 | resource "aws_s3_bucket_public_access_block" "styx" { 84 | bucket = aws_s3_bucket.styx.id 85 | block_public_acls = false 86 | block_public_policy = false 87 | ignore_public_acls = false 88 | restrict_public_buckets = false 89 | } 90 | 91 | resource "aws_s3_object" "styx-cache-info" { 92 | bucket = aws_s3_bucket.styx.id 93 | key = "nixcache/nix-cache-info" 94 | content = "StoreDir: /nix/store\nPriority: 90\n" 95 | } 96 | 97 | resource "aws_s3_object" "styx-params-test-1" { 98 | bucket = aws_s3_bucket.styx.id 99 | key = "params/test-1" 100 | source = "../params/test-1.signed" 101 | } 102 | 103 | // parameter store 104 | 105 | resource "aws_ssm_parameter" "manifester_signkey" { 106 | name = "styx-manifester-signkey-test-1" 107 | type = "SecureString" 108 | value = file("../keys/styx-test-1.secret") 109 | } 110 | 111 | // lambda: 112 | 113 | variable "manifester_image_tag" {} 114 | 115 | variable "lambda_memory_sizes" { 116 | type = list(number) 117 | default = [500, 1500] # MB 118 | } 119 | 120 | resource "aws_lambda_function" "manifester" { 121 | count = length(var.lambda_memory_sizes) 122 | 123 | package_type = "Image" 124 | image_uri = "${aws_ecr_repository.repo.repository_url}:${var.manifester_image_tag}" 125 | 126 | function_name = count.index == 0 ? "styx-manifester" : "styx-manifester-${count.index}" 127 | role = aws_iam_role.iam_for_lambda.arn 128 | 129 | architectures = ["x86_64"] # TODO: can we make it run on arm? 130 | 131 | memory_size = var.lambda_memory_sizes[count.index] 132 | ephemeral_storage { 133 | size = 512 # MB 134 | } 135 | timeout = 300 # seconds 136 | image_config { 137 | command = [ 138 | "manifester", 139 | # must be in the same region: 140 | "--chunkbucket=${aws_s3_bucket.styx.id}", 141 | "--styx_ssm_signkey=${aws_ssm_parameter.manifester_signkey.name}", 142 | # Uncomment these to allow manifester to build from styx nix cache on-demand. 143 | # This shouldn't be necessary since CI pre-builds manifests and they should 144 | # be cached. 145 | # "--allowed_upstream=cache.nixos.org", 146 | # "--allowed_upstream=${aws_s3_bucket.styx.id}.s3.amazonaws.com", 147 | # "--nix_pubkey=cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", 148 | # "--nix_pubkey=${trimspace(file("../keys/styx-nixcache-test-1.public"))}", 149 | ] 150 | } 151 | 152 | # logging and metrics with axiom 153 | environment { 154 | variables = { 155 | AXIOM_TOKEN = trimspace(file("../keys/axiom-styx-lambda.secret")) 156 | AXIOM_DATASET = "styx" 157 | } 158 | } 159 | } 160 | 161 | resource "aws_lambda_function_url" "manifester" { 162 | count = length(var.lambda_memory_sizes) 163 | 164 | function_name = aws_lambda_function.manifester[count.index].function_name 165 | authorization_type = "NONE" 166 | invoke_mode = "RESPONSE_STREAM" 167 | } 168 | -------------------------------------------------------------------------------- /daemon/fscache.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "math/bits" 5 | "strconv" 6 | 7 | "github.com/dnr/styx/common" 8 | ) 9 | 10 | // References: 11 | // https://www.kernel.org/doc/html/latest/filesystems/caching/cachefiles.html 12 | 13 | type ( 14 | cachefiles_msg struct { 15 | // struct cachefiles_msg { 16 | // __u32 msg_id; 17 | MsgId uint32 `struc:"little"` 18 | // __u32 opcode; 19 | OpCode uint32 `struc:"little"` 20 | // __u32 len; 21 | Len uint32 `struc:"little"` 22 | // __u32 object_id; 23 | ObjectId uint32 `struc:"little"` 24 | // __u8 data[]; 25 | // }; 26 | } 27 | 28 | cachefiles_open struct { 29 | // struct cachefiles_open { 30 | // __u32 volume_key_size; 31 | VolumeKeySize uint32 `struc:"little,sizeof=VolumeKey"` 32 | // __u32 cookie_key_size; 33 | CookieKeySize uint32 `struc:"little,sizeof=CookieKey"` 34 | // __u32 fd; 35 | Fd uint32 `struc:"little"` 36 | // __u32 flags; 37 | Flags uint32 `struc:"little"` 38 | // __u8 data[]; 39 | VolumeKey []byte 40 | CookieKey []byte 41 | // }; 42 | } 43 | // data contains the volume_key followed directly by the cookie_key. The volume key is a 44 | // NUL-terminated string; the cookie key is binary data. 45 | // volume_key_size indicates the size of the volume key in bytes. 46 | // cookie_key_size indicates the size of the cookie key in bytes. 47 | // fd indicates an anonymous fd referring to the cache file, through which the user daemon can 48 | // perform write/llseek file operations on the cache file. 49 | 50 | // The user daemon should reply the OPEN request by issuing a "copen" (complete open) 51 | // command on the devnode: 52 | // copen , 53 | // msg_id must match the msg_id field of the OPEN request. 54 | // When >= 0, cache_size indicates the size of the cache file; when < 0, cache_size 55 | // indicates any error code encountered by the user daemon. 56 | 57 | // When a cookie withdrawn, a CLOSE request (opcode CACHEFILES_OP_CLOSE) will be sent to 58 | // the user daemon. This tells the user daemon to close all anonymous fds associated with 59 | // the given object_id. The CLOSE request has no extra payload, and shouldn't be replied. 60 | 61 | cachefiles_read struct { 62 | // struct cachefiles_read { 63 | // __u64 off; 64 | Off uint64 `struc:"little"` 65 | // __u64 len; 66 | Len uint64 `struc:"little"` 67 | // }; 68 | } 69 | // off indicates the starting offset of the requested file range. 70 | // len indicates the length of the requested file range. 71 | 72 | // When it receives a READ request, the user daemon should fetch the requested data and 73 | // write it to the cache file identified by object_id. 74 | // When it has finished processing the READ request, the user daemon should reply by using 75 | // the CACHEFILES_IOC_READ_COMPLETE ioctl on one of the anonymous fds associated with the 76 | // object_id given in the READ request. The ioctl is of the form: 77 | // ioctl(fd, CACHEFILES_IOC_READ_COMPLETE, msg_id); 78 | // where: 79 | // fd is one of the anonymous fds associated with the object_id given. 80 | // msg_id must match the msg_id field of the READ request. 81 | ) 82 | 83 | const ( 84 | /* 85 | * Fscache ensures that the maximum length of cookie key is 255. The volume key 86 | * is controlled by netfs, and generally no bigger than 255. 87 | */ 88 | CACHEFILES_MSG_MAX_SIZE = 1024 89 | 90 | CACHEFILES_OP_OPEN = 0 91 | CACHEFILES_OP_CLOSE = 1 92 | CACHEFILES_OP_READ = 2 93 | 94 | CACHEFILES_IOC_READ_COMPLETE = _IOC_WRITE<<_IOC_DIRSHIFT | 0x98<<_IOC_TYPESHIFT | 1<<_IOC_NRSHIFT | 4<<_IOC_SIZESHIFT 95 | 96 | _IOC_WRITE = 1 97 | _IOC_NRBITS = 8 98 | _IOC_TYPEBITS = 8 99 | _IOC_SIZEBITS = 14 100 | _IOC_DIRBITS = 2 101 | _IOC_NRSHIFT = 0 102 | _IOC_TYPESHIFT = _IOC_NRSHIFT + _IOC_NRBITS 103 | _IOC_SIZESHIFT = _IOC_TYPESHIFT + _IOC_TYPEBITS 104 | _IOC_DIRSHIFT = _IOC_SIZESHIFT + _IOC_SIZEBITS 105 | ) 106 | 107 | // derived from linux kernel: 108 | // len(data) must be multiple of 4, padded with zeros 109 | func _fscache_hash(salt uint32, data []byte) uint32 { 110 | y := salt 111 | var a, x uint32 112 | for len(data) > 0 { 113 | a = uint32(data[0]) | uint32(data[1])<<8 | uint32(data[2])<<16 | uint32(data[3])<<24 114 | data = data[4:] 115 | x ^= a 116 | y ^= x 117 | x = bits.RotateLeft32(x, 7) 118 | x += y 119 | y = bits.RotateLeft32(y, 20) 120 | y *= 9 121 | } 122 | return __hash_32(y ^ __hash_32(x)) 123 | } 124 | 125 | func __hash_32(val uint32) uint32 { return val * 0x61C88647 } 126 | 127 | func fscachePath(domainid, fsid string) string { 128 | // This doesn't implement the more complicated splitting and encoding logic, it only works 129 | // on short ascii names, but that's all we use. 130 | volume := "erofs," + domainid 131 | volkey := make([]byte, (len(volume)+2+3)&^3) 132 | volkey[0] = common.TruncU8(len(volume)) 133 | copy(volkey[1:], volume) 134 | seed := _fscache_hash(0, volkey) 135 | 136 | var hash uint32 137 | if len(fsid)&3 == 0 { 138 | hash = _fscache_hash(seed, []byte(fsid)) 139 | } else { 140 | padded := make([]byte, (len(fsid)+3)&^3) 141 | copy(padded, []byte(fsid)) 142 | hash = _fscache_hash(seed, padded) 143 | } 144 | 145 | return "cache/I" + volume + "/@" + strconv.FormatUint(uint64(hash&0xff), 16) + "/D" + fsid 146 | } 147 | -------------------------------------------------------------------------------- /bin/make-styx-include: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # The current Styx settings support either an include list or an exclude list. 4 | # To use Styx for "everything except core system packages", you might prefer an 5 | # exclude list. However, it only looks at package names and not closures, so 6 | # it'd be easy to miss a key package. To be safer, we can remove whole closures 7 | # from the system closure and take the result as an include list. That also 8 | # defaults new and unknown packages to excluded. 9 | 10 | import sys, os, subprocess, re 11 | 12 | # Packages that we shouldn't use Styx for until it's more stable. This should be 13 | # enough to repair a broken system. These are regex matches (anchored at start) 14 | # against package names. 15 | if True: 16 | EXCLUDE = [ 17 | 'bash-', 18 | 'coreutils-', 19 | 'initrd-linux-', 20 | 'linux-[0-9]', 21 | 'nix-', 22 | 'notion-', 23 | 'procps-', 24 | 'psmisc-', 25 | 'sudo-', 26 | 'systemd-', 27 | 'util-linux-', 28 | 'xorg-', 29 | 'vim-', 30 | ] 31 | 32 | if True: 33 | # Exclude some more filesystem-oriented packages. 34 | EXCLUDE += [ 35 | 'brotli-', 36 | 'btrfs-progs-', 37 | 'hdparm-', 38 | 'lvm2-', 39 | 'nfs-utils-', 40 | 'unzip-', 41 | 'xz-', 42 | 'zip-', 43 | 'zstd-', 44 | ] 45 | 46 | if True: 47 | # Exclude anything included in most systemd units, but ignore some units 48 | # that include combined system paths. This should ensure a system can mostly boot. 49 | EXCLUDE += ['unit-(?!accounts-daemon|dbus|polkit|systemd-udevd|udevd|suid-sgid-wrappers).*'] 50 | 51 | if True: 52 | # Exclude anything included any systemd units. This makes things pretty safe 53 | # but excludes many more packages. 54 | EXCLUDE += ['unit-.*'] 55 | 56 | 57 | CURRENT_SYSTEM = '/run/current-system' 58 | 59 | def closure(base): 60 | out = subprocess.check_output(['nix-store', '-qR', base]) 61 | return set(out.decode().splitlines()) 62 | 63 | def disk_size(pkgs): 64 | inp = '\x00'.join(pkgs) + '\x00' 65 | out = subprocess.check_output(['du', '-scm', '--files0-from=-'], input=inp.encode()) 66 | return int(out.decode().splitlines()[-1].split()[0]) 67 | 68 | def make_includes(pkgs, excluded): 69 | # This automatically filters out stuff without a version number. 70 | # Packages without a version number are mostly locally-built anyway. 71 | ver = re.compile(r'(.+?)-([\d.-]+)$') 72 | verout = re.compile(r'(.+?)-([\d.-]+)(-bin|-data|-doc|-info|-man|-dev|-lib)?$') 73 | # Match firmware patterns: 74 | fw = re.compile(r'(.+?-[Ff]irmware)-([\d.-]+)-(xz|zstd)$') 75 | fwnover = re.compile(r'(.+?-firmware)-(xz|zstd)$') 76 | # Other hacks: 77 | spotify = re.compile(r'(spotify)-.*$') # has letters in "version" 78 | wk = re.compile(r'(webkitgtk)-([\d.+abi=-]+)') # +abi=... 79 | 80 | incs = set() 81 | for p in pkgs: 82 | p = p[44:] 83 | if m := ver.match(p): 84 | incs.add(f'{m.group(1)}-.*') 85 | elif m := verout.match(p): 86 | incs.add(f'{m.group(1)}-.*{m.group(3)}') 87 | elif m := fw.match(p): 88 | incs.add(f'{m.group(1)}-.*-{m.group(3)}') 89 | elif m := fwnover.match(p): 90 | incs.add(f'{m.group(1)}-{m.group(2)}') 91 | elif m := spotify.match(p) or wk.match(p): 92 | incs.add(f'{m.group(1)}-.*') 93 | else: 94 | print(f"# can't make pattern for {p}") 95 | 96 | # if we have with named outputs and without, we can drop the named 97 | for inc in list(incs): 98 | r = re.compile(inc) 99 | for sub in [j for j in incs if len(j) > len(inc) and r.match(j)]: 100 | print(f"# {sub} subsumed by {inc}") 101 | incs.discard(sub) 102 | 103 | # if we accidentally included too much by stripping version numbers, 104 | # just drop them. this isn't too many things. 105 | for inc in list(incs): 106 | r = re.compile(inc) 107 | if ex := [e[44:] for e in excluded if r.match(e[44:])]: 108 | print(f"# oops, {inc} would have matched excluded {', '.join(ex)}") 109 | incs.discard(inc) 110 | 111 | return incs 112 | 113 | def apply_includes(incs, pkgs): 114 | cre = [re.compile(i) for i in incs] 115 | return set(p for p in pkgs if any(r.match(p[44:]) for r in cre)) 116 | 117 | def main(): 118 | system = closure(CURRENT_SYSTEM) 119 | pkgs = set(system) 120 | for e in EXCLUDE: 121 | for r in set([p for p in pkgs if re.match(e, p[44:])]): 122 | pkgs -= closure(r) 123 | #for nr in prevpkgs - pkgs: print(f"# removed {nr} because of {r} <- {e}") 124 | 125 | incs = make_includes(pkgs, system - pkgs) 126 | 127 | for inc in sorted(incs): print(inc) 128 | 129 | if True: 130 | print("------") 131 | print("full size:", disk_size(system)) 132 | included = apply_includes(incs, system) 133 | print("included paths:", len(included)) 134 | print("included size:", disk_size(included)) 135 | # note sizes are not quite right if hard-linking 136 | 137 | 138 | if __name__ == '__main__': 139 | main() 140 | -------------------------------------------------------------------------------- /common/errgroup/errgroup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package errgroup provides synchronization, error propagation, and Context 6 | // cancelation for groups of goroutines working on subtasks of a common task. 7 | // 8 | // [errgroup.Group] is related to [sync.WaitGroup] but adds handling of tasks 9 | // returning errors. 10 | // 11 | // Modifications for styx: 12 | // - Embedded Context so can be used as a Context instead of passing both around. 13 | // - Added Limit() method to get the limit, so that doesn't have to be passed separately either 14 | // when both are needed. 15 | // - Added Cancel() to immediately cancel as if a called function returned an error. 16 | // - Added SetWorkLimit() 17 | // 18 | // Note: use context.Cause(group) to get the pending error without calling Wait(). 19 | package errgroup 20 | 21 | import ( 22 | "context" 23 | "fmt" 24 | "sync" 25 | ) 26 | 27 | type token struct{} 28 | 29 | // A Group is a collection of goroutines working on subtasks that are part of 30 | // the same overall task. 31 | // 32 | // A zero Group is valid, has no limit on the number of active goroutines, 33 | // and does not cancel on error. 34 | type Group struct { 35 | context.Context 36 | 37 | cancel func(error) 38 | 39 | wg sync.WaitGroup 40 | 41 | sem chan token 42 | workSem chan token 43 | 44 | errOnce sync.Once 45 | err error 46 | } 47 | 48 | func (g *Group) done() { 49 | if g.workSem != nil { 50 | <-g.workSem 51 | } 52 | if g.sem != nil { 53 | <-g.sem 54 | } 55 | g.wg.Done() 56 | } 57 | 58 | // WithContext returns a new Group and an associated Context derived from ctx. 59 | // 60 | // The derived Context is canceled the first time a function passed to Go 61 | // returns a non-nil error or the first time Wait returns, whichever occurs 62 | // first. 63 | func WithContext(ctx context.Context) *Group { 64 | ctx, cancel := context.WithCancelCause(ctx) 65 | return &Group{Context: ctx, cancel: cancel} 66 | } 67 | 68 | // Wait blocks until all function calls from the Go method have returned, then 69 | // returns the first non-nil error (if any) from them. 70 | func (g *Group) Wait() error { 71 | g.wg.Wait() 72 | if g.cancel != nil { 73 | g.cancel(g.err) 74 | } 75 | return g.err 76 | } 77 | 78 | // Go calls the given function in a new goroutine. 79 | // It blocks until the new goroutine can be added without the number of 80 | // active goroutines in the group exceeding the configured limit. 81 | // 82 | // The first call to return a non-nil error cancels the group's context, if the 83 | // group was created by calling WithContext. The error will be returned by Wait. 84 | func (g *Group) Go(f func() error) { 85 | if g.sem != nil { 86 | g.sem <- token{} 87 | } 88 | 89 | g._go(f) 90 | } 91 | 92 | // TryGo calls the given function in a new goroutine only if the number of 93 | // active goroutines in the group is currently below the configured limit. 94 | // 95 | // The return value reports whether the goroutine was started. 96 | func (g *Group) TryGo(f func() error) bool { 97 | if g.sem != nil { 98 | select { 99 | case g.sem <- token{}: 100 | // Note: this allows barging iff channels in general allow barging. 101 | default: 102 | return false 103 | } 104 | } 105 | 106 | g._go(f) 107 | return true 108 | } 109 | 110 | func (g *Group) _go(f func() error) { 111 | g.wg.Add(1) 112 | go func() { 113 | defer g.done() 114 | 115 | if g.workSem != nil { 116 | g.workSem <- token{} 117 | } 118 | 119 | if err := f(); err != nil { 120 | g.Cancel(err) 121 | } 122 | }() 123 | } 124 | 125 | // SetLimit limits the number of active goroutines in this group to at most n. 126 | // A negative value indicates no limit. 127 | // 128 | // Any subsequent call to the Go method will block until it can add an active 129 | // goroutine without exceeding the configured limit. 130 | // 131 | // The limit must not be modified while any goroutines in the group are active. 132 | func (g *Group) SetLimit(n int) { 133 | if n < 0 { 134 | g.sem = nil 135 | return 136 | } 137 | if len(g.sem) != 0 { 138 | panic(fmt.Errorf("errgroup: modify limit while %v goroutines in the group are still active", len(g.sem))) 139 | } 140 | g.sem = make(chan token, n) 141 | } 142 | 143 | // SetWorkLimit limits the number of goroutines doing work in this group to at most n. 144 | // Goroutines will be created (Go() will return) without regard to the limit, but blocked on a 145 | // semaphore. A negative value indicates no limit. 146 | func (g *Group) SetWorkLimit(n int) { 147 | if n < 0 { 148 | g.workSem = nil 149 | return 150 | } 151 | if len(g.workSem) != 0 { 152 | panic(fmt.Errorf("errgroup: modify work limit while %v goroutines in the group are still active", len(g.workSem))) 153 | } 154 | g.workSem = make(chan token, n) 155 | } 156 | 157 | // Limit returns the limit size 158 | func (g *Group) Limit() int { 159 | return cap(g.sem) 160 | } 161 | 162 | // WorkLimit returns the work limit size 163 | func (g *Group) WorkLimit() int { 164 | return cap(g.workSem) 165 | } 166 | 167 | // Cancel cancels the group context with an error, as if a function started by Do returned that error. 168 | func (g *Group) Cancel(err error) { 169 | g.errOnce.Do(func() { 170 | g.err = err 171 | if g.cancel != nil { 172 | g.cancel(g.err) 173 | } 174 | }) 175 | } 176 | -------------------------------------------------------------------------------- /cmd/styx/internal.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io" 7 | "log" 8 | "net/http" 9 | "os" 10 | "strings" 11 | 12 | "github.com/nix-community/go-nix/pkg/narinfo/signature" 13 | "github.com/spf13/cobra" 14 | "google.golang.org/protobuf/encoding/protojson" 15 | 16 | "github.com/dnr/styx/common" 17 | "github.com/dnr/styx/common/cdig" 18 | "github.com/dnr/styx/daemon" 19 | "github.com/dnr/styx/manifester" 20 | "github.com/dnr/styx/pb" 21 | ) 22 | 23 | type ( 24 | remanifestReq struct { 25 | cacheurl string 26 | requrl string 27 | reqIfFound bool 28 | reqIfNot bool 29 | upstream string 30 | storePath string 31 | } 32 | ) 33 | 34 | func withInFile(c *cobra.Command, args []string) error { 35 | in, err := os.Open(args[0]) 36 | if err != nil { 37 | return err 38 | } 39 | storeKeyed(c, in, "in") 40 | c.PostRunE = chainRunE(c.PostRunE, func(c *cobra.Command, args []string) error { 41 | return in.Close() 42 | }) 43 | return nil 44 | } 45 | 46 | func withOutFile(c *cobra.Command, args []string) error { 47 | out, err := os.OpenFile(args[1], os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) 48 | if err != nil { 49 | return err 50 | } 51 | storeKeyed(c, out, "out") 52 | c.PostRunE = chainRunE(c.PostRunE, func(c *cobra.Command, args []string) error { 53 | return out.Close() 54 | }) 55 | return nil 56 | } 57 | 58 | func withRemanifestReq(c *cobra.Command) runE { 59 | var req remanifestReq 60 | c.Flags().StringVar(&req.cacheurl, "cacheurl", "", "manifest cache url") 61 | c.MarkFlagRequired("cacheurl") 62 | c.Flags().StringVar(&req.requrl, "requrl", "", "manifester request url") 63 | c.MarkFlagRequired("requrl") 64 | c.Flags().BoolVar(&req.reqIfFound, "request_if_found", false, "rerequest if found") 65 | c.Flags().BoolVar(&req.reqIfNot, "request_if_not", false, "rerequest if not found") 66 | c.Flags().StringVar(&req.upstream, "upstream", "https://cache.nixos.org/", "remanifest upstream") 67 | c.Flags().StringVar(&req.storePath, "storepath", "", "remanifest store path") 68 | return storer(&req) 69 | } 70 | 71 | func internalCmd() *cobra.Command { 72 | return cmd( 73 | &cobra.Command{Use: "internal", Short: "internal commands"}, 74 | cmd( 75 | &cobra.Command{ 76 | Use: "signdaemonparams ", 77 | Args: cobra.ExactArgs(2), 78 | }, 79 | withSignKeys, 80 | withInFile, 81 | withOutFile, 82 | func(c *cobra.Command, args []string) error { 83 | in := getKeyed[*os.File](c, "in") 84 | out := getKeyed[*os.File](c, "out") 85 | keys := get[[]signature.SecretKey](c) 86 | var params pb.DaemonParams 87 | var sb []byte 88 | if b, err := io.ReadAll(in); err != nil { 89 | return err 90 | } else if err = protojson.Unmarshal(b, ¶ms); err != nil { 91 | return err 92 | } else if sb, err = common.SignInlineMessage(keys, common.DaemonParamsContext, ¶ms); err != nil { 93 | return err 94 | } else if _, err = out.Write(sb); err != nil { 95 | return err 96 | } 97 | return nil 98 | }, 99 | ), 100 | cmd( 101 | &cobra.Command{ 102 | Use: "remanifest", 103 | Short: "re-request manifests", 104 | }, 105 | withRemanifestReq, 106 | func(c *cobra.Command, args []string) error { 107 | req := get[*remanifestReq](c) 108 | _, sphStr, _ := daemon.ParseSph(req.storePath) 109 | mReq := manifester.ManifestReq{ 110 | Upstream: req.upstream, 111 | StorePathHash: sphStr, 112 | DigestAlgo: cdig.Algo, 113 | DigestBits: int(cdig.Bits), 114 | } 115 | 116 | mcread := manifester.NewChunkStoreReadUrl(req.cacheurl, manifester.ManifestCachePath) 117 | 118 | doRequest := false 119 | if _, err := mcread.Get(c.Context(), mReq.CacheKey(), nil); err == nil { 120 | log.Println("manifest cache hit for", sphStr) 121 | doRequest = req.reqIfFound 122 | } else { 123 | log.Println("manifest not found for", sphStr, err) 124 | doRequest = req.reqIfNot 125 | } 126 | 127 | if !doRequest { 128 | return nil 129 | } 130 | 131 | reqBytes, err := json.Marshal(mReq) 132 | if err != nil { 133 | return err 134 | } 135 | 136 | u := strings.TrimSuffix(req.requrl, "/") + manifester.ManifestPath 137 | hReq, err := http.NewRequestWithContext(c.Context(), http.MethodPost, u, bytes.NewReader(reqBytes)) 138 | if err != nil { 139 | return err 140 | } 141 | hReq.Header.Set("Content-Type", common.CTJson) 142 | res, err := http.DefaultClient.Do(hReq) 143 | if err == nil { 144 | if res.StatusCode != http.StatusOK { 145 | err = common.HttpErrorFromRes(res) 146 | } 147 | res.Body.Close() 148 | } 149 | if err == nil { 150 | log.Println("manifest request for", sphStr, "success") 151 | } else { 152 | log.Println("manifest request for", sphStr, "failed", err) 153 | } 154 | return nil 155 | }, 156 | ), 157 | cmd( 158 | &cobra.Command{ 159 | Use: "mcachekey", 160 | Short: "print manifest cache key", 161 | Args: cobra.ExactArgs(2), 162 | }, 163 | func(c *cobra.Command, args []string) error { 164 | mReq := manifester.ManifestReq{ 165 | Upstream: args[0], 166 | StorePathHash: args[1], 167 | DigestAlgo: cdig.Algo, 168 | DigestBits: int(cdig.Bits), 169 | } 170 | log.Printf("req: %#v\nkey: %s\n", mReq, mReq.CacheKey()) 171 | return nil 172 | }, 173 | ), 174 | ) 175 | } 176 | -------------------------------------------------------------------------------- /pb/manifest.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.36.10 4 | // protoc v6.32.1 5 | // source: manifest.proto 6 | 7 | package pb 8 | 9 | import ( 10 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 11 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 12 | reflect "reflect" 13 | sync "sync" 14 | unsafe "unsafe" 15 | ) 16 | 17 | const ( 18 | // Verify that this generated code is sufficiently up-to-date. 19 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 20 | // Verify that runtime/protoimpl is sufficiently up-to-date. 21 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 22 | ) 23 | 24 | type Manifest struct { 25 | state protoimpl.MessageState `protogen:"open.v1"` 26 | Params *GlobalParams `protobuf:"bytes,1,opt,name=params,proto3" json:"params,omitempty"` 27 | Entries []*Entry `protobuf:"bytes,3,rep,name=entries,proto3" json:"entries,omitempty"` 28 | // build parameters 29 | SmallFileCutoff int32 `protobuf:"varint,2,opt,name=small_file_cutoff,json=smallFileCutoff,proto3" json:"small_file_cutoff,omitempty"` 30 | // Metadata on how this was generated 31 | Meta *ManifestMeta `protobuf:"bytes,10,opt,name=meta,proto3" json:"meta,omitempty"` 32 | unknownFields protoimpl.UnknownFields 33 | sizeCache protoimpl.SizeCache 34 | } 35 | 36 | func (x *Manifest) Reset() { 37 | *x = Manifest{} 38 | mi := &file_manifest_proto_msgTypes[0] 39 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 40 | ms.StoreMessageInfo(mi) 41 | } 42 | 43 | func (x *Manifest) String() string { 44 | return protoimpl.X.MessageStringOf(x) 45 | } 46 | 47 | func (*Manifest) ProtoMessage() {} 48 | 49 | func (x *Manifest) ProtoReflect() protoreflect.Message { 50 | mi := &file_manifest_proto_msgTypes[0] 51 | if x != nil { 52 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 53 | if ms.LoadMessageInfo() == nil { 54 | ms.StoreMessageInfo(mi) 55 | } 56 | return ms 57 | } 58 | return mi.MessageOf(x) 59 | } 60 | 61 | // Deprecated: Use Manifest.ProtoReflect.Descriptor instead. 62 | func (*Manifest) Descriptor() ([]byte, []int) { 63 | return file_manifest_proto_rawDescGZIP(), []int{0} 64 | } 65 | 66 | func (x *Manifest) GetParams() *GlobalParams { 67 | if x != nil { 68 | return x.Params 69 | } 70 | return nil 71 | } 72 | 73 | func (x *Manifest) GetEntries() []*Entry { 74 | if x != nil { 75 | return x.Entries 76 | } 77 | return nil 78 | } 79 | 80 | func (x *Manifest) GetSmallFileCutoff() int32 { 81 | if x != nil { 82 | return x.SmallFileCutoff 83 | } 84 | return 0 85 | } 86 | 87 | func (x *Manifest) GetMeta() *ManifestMeta { 88 | if x != nil { 89 | return x.Meta 90 | } 91 | return nil 92 | } 93 | 94 | var File_manifest_proto protoreflect.FileDescriptor 95 | 96 | const file_manifest_proto_rawDesc = "" + 97 | "\n" + 98 | "\x0emanifest.proto\x12\x02pb\x1a\ventry.proto\x1a\x13manifest_meta.proto\x1a\fparams.proto\"\xab\x01\n" + 99 | "\bManifest\x12(\n" + 100 | "\x06params\x18\x01 \x01(\v2\x10.pb.GlobalParamsR\x06params\x12#\n" + 101 | "\aentries\x18\x03 \x03(\v2\t.pb.EntryR\aentries\x12*\n" + 102 | "\x11small_file_cutoff\x18\x02 \x01(\x05R\x0fsmallFileCutoff\x12$\n" + 103 | "\x04meta\x18\n" + 104 | " \x01(\v2\x10.pb.ManifestMetaR\x04metaB\x18Z\x16github.com/dnr/styx/pbb\x06proto3" 105 | 106 | var ( 107 | file_manifest_proto_rawDescOnce sync.Once 108 | file_manifest_proto_rawDescData []byte 109 | ) 110 | 111 | func file_manifest_proto_rawDescGZIP() []byte { 112 | file_manifest_proto_rawDescOnce.Do(func() { 113 | file_manifest_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_manifest_proto_rawDesc), len(file_manifest_proto_rawDesc))) 114 | }) 115 | return file_manifest_proto_rawDescData 116 | } 117 | 118 | var file_manifest_proto_msgTypes = make([]protoimpl.MessageInfo, 1) 119 | var file_manifest_proto_goTypes = []any{ 120 | (*Manifest)(nil), // 0: pb.Manifest 121 | (*GlobalParams)(nil), // 1: pb.GlobalParams 122 | (*Entry)(nil), // 2: pb.Entry 123 | (*ManifestMeta)(nil), // 3: pb.ManifestMeta 124 | } 125 | var file_manifest_proto_depIdxs = []int32{ 126 | 1, // 0: pb.Manifest.params:type_name -> pb.GlobalParams 127 | 2, // 1: pb.Manifest.entries:type_name -> pb.Entry 128 | 3, // 2: pb.Manifest.meta:type_name -> pb.ManifestMeta 129 | 3, // [3:3] is the sub-list for method output_type 130 | 3, // [3:3] is the sub-list for method input_type 131 | 3, // [3:3] is the sub-list for extension type_name 132 | 3, // [3:3] is the sub-list for extension extendee 133 | 0, // [0:3] is the sub-list for field type_name 134 | } 135 | 136 | func init() { file_manifest_proto_init() } 137 | func file_manifest_proto_init() { 138 | if File_manifest_proto != nil { 139 | return 140 | } 141 | file_entry_proto_init() 142 | file_manifest_meta_proto_init() 143 | file_params_proto_init() 144 | type x struct{} 145 | out := protoimpl.TypeBuilder{ 146 | File: protoimpl.DescBuilder{ 147 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 148 | RawDescriptor: unsafe.Slice(unsafe.StringData(file_manifest_proto_rawDesc), len(file_manifest_proto_rawDesc)), 149 | NumEnums: 0, 150 | NumMessages: 1, 151 | NumExtensions: 0, 152 | NumServices: 0, 153 | }, 154 | GoTypes: file_manifest_proto_goTypes, 155 | DependencyIndexes: file_manifest_proto_depIdxs, 156 | MessageInfos: file_manifest_proto_msgTypes, 157 | }.Build() 158 | File_manifest_proto = out.File 159 | file_manifest_proto_goTypes = nil 160 | file_manifest_proto_depIdxs = nil 161 | } 162 | -------------------------------------------------------------------------------- /tf/ci.tf: -------------------------------------------------------------------------------- 1 | 2 | // iam for heavy worker on ec2: 3 | 4 | data "aws_iam_policy_document" "assume_role_ec2" { 5 | statement { 6 | principals { 7 | type = "Service" 8 | identifiers = ["ec2.amazonaws.com"] 9 | } 10 | actions = ["sts:AssumeRole"] 11 | } 12 | } 13 | 14 | data "aws_iam_policy_document" "charon_get_parameters" { 15 | statement { 16 | actions = ["ssm:GetParameter"] 17 | resources = [ 18 | aws_ssm_parameter.charon_temporal_params.arn, 19 | aws_ssm_parameter.charon_signkey.arn, 20 | aws_ssm_parameter.manifester_signkey.arn, 21 | ] 22 | } 23 | } 24 | 25 | resource "aws_iam_role" "iam_for_charon" { 26 | name = "iam_for_ec2_charon" 27 | assume_role_policy = data.aws_iam_policy_document.assume_role_ec2.json 28 | inline_policy { 29 | name = "get-parameter" 30 | policy = data.aws_iam_policy_document.charon_get_parameters.json 31 | } 32 | } 33 | 34 | // iam for scaler only 35 | 36 | data "aws_iam_policy_document" "charon_asg_scaler" { 37 | statement { 38 | actions = [ 39 | "autoscaling:DescribeAutoScalingGroups", 40 | "autoscaling:SetDesiredCapacity", 41 | ] 42 | resources = [ 43 | aws_autoscaling_group.charon_asg.arn, 44 | ] 45 | } 46 | } 47 | 48 | resource "aws_iam_user" "charon_asg_scaler" { 49 | name = "charon_asg_scaler" 50 | } 51 | 52 | resource "aws_iam_user_policy" "charon_asg_scaler" { 53 | user = aws_iam_user.charon_asg_scaler.name 54 | policy = data.aws_iam_policy_document.charon_asg_scaler.json 55 | } 56 | 57 | resource "aws_iam_access_key" "charon_asg_scaler" { 58 | user = aws_iam_user.charon_asg_scaler.name 59 | } 60 | 61 | resource "local_sensitive_file" "charon_asg_scaler" { 62 | content = <<-EOF 63 | [default] 64 | aws_access_key_id = ${aws_iam_access_key.charon_asg_scaler.id} 65 | aws_secret_access_key = ${aws_iam_access_key.charon_asg_scaler.secret} 66 | EOF 67 | filename = "../keys/charon-asg-scaler-creds.secret" 68 | file_permission = 0600 69 | } 70 | 71 | // parameter store 72 | 73 | resource "aws_ssm_parameter" "charon_signkey" { 74 | name = "styx-charon-signkey-test-1" 75 | type = "SecureString" 76 | value = trimspace(file("../keys/styx-nixcache-test-1.secret")) 77 | } 78 | 79 | resource "aws_ssm_parameter" "charon_temporal_params" { 80 | name = "styx-charon-temporal-params" 81 | type = "SecureString" 82 | value = trimspace(file("../keys/temporal-creds-charon.secret")) 83 | } 84 | 85 | // security group 86 | 87 | resource "aws_security_group" "worker_sg" { 88 | name = "charon-worker-sg" 89 | description = "Security group for workers" 90 | 91 | ingress { 92 | from_port = 22 93 | to_port = 22 94 | protocol = "tcp" 95 | cidr_blocks = ["0.0.0.0/0"] 96 | } 97 | 98 | egress { 99 | from_port = 0 100 | to_port = 0 101 | protocol = "-1" 102 | cidr_blocks = ["0.0.0.0/0"] 103 | } 104 | } 105 | 106 | // instance profile 107 | 108 | resource "aws_iam_instance_profile" "charon_worker" { 109 | name = "charon_worker_profile" 110 | role = aws_iam_role.iam_for_charon.name 111 | } 112 | 113 | // ssh key 114 | 115 | resource "aws_key_pair" "my_ssh_key" { 116 | public_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHLw2kct3mhDYpJyrchof00gDxCVqgepql0OoRSNbdkY" 117 | } 118 | 119 | // asg 120 | 121 | data "aws_ami" "nixos_x86_64" { 122 | owners = ["427812963091"] 123 | most_recent = true 124 | filter { 125 | name = "name" 126 | values = ["nixos/25.11*"] 127 | } 128 | filter { 129 | name = "architecture" 130 | values = ["x86_64"] 131 | } 132 | } 133 | 134 | variable "charon_storepath" {} 135 | 136 | resource "aws_launch_template" "charon_worker" { 137 | name_prefix = "charon-worker" 138 | image_id = data.aws_ami.nixos_x86_64.id 139 | security_group_names = [aws_security_group.worker_sg.name] 140 | iam_instance_profile { 141 | arn = aws_iam_instance_profile.charon_worker.arn 142 | } 143 | key_name = aws_key_pair.my_ssh_key.id 144 | user_data = base64encode(templatefile("charon-worker-ud.nix", { 145 | sub = "https://${aws_s3_bucket.styx.id}.s3.amazonaws.com/nixcache/" 146 | pubkey = trimspace(file("../keys/styx-nixcache-test-1.public")) 147 | charon = var.charon_storepath 148 | tmpssm = aws_ssm_parameter.charon_temporal_params.name 149 | cachessm = aws_ssm_parameter.charon_signkey.id 150 | bucket = aws_s3_bucket.styx.id 151 | styxssm = aws_ssm_parameter.manifester_signkey.name 152 | nixoskey = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" 153 | axiom_dataset = "styx" 154 | axiom_token = trimspace(file("../keys/axiom-styx-charon-heavy.secret")) 155 | })) 156 | block_device_mappings { 157 | device_name = "/dev/xvda" 158 | ebs { 159 | volume_size = 60 160 | volume_type = "gp3" 161 | } 162 | } 163 | instance_type = "c7a.4xlarge" 164 | instance_market_options { 165 | market_type = "spot" 166 | spot_options { 167 | max_price = "0.40" # c7a.4xlarge on-demand: 0.8211 168 | } 169 | } 170 | } 171 | 172 | resource "aws_autoscaling_group" "charon_asg" { 173 | name = "charon-asg" 174 | 175 | min_size = 0 176 | max_size = 1 177 | #desired_capacity = 0 178 | 179 | # Note: 1e apparently has no c7a.4xlarge? that alias is account-specific, another account 180 | # will have to use a different set here. 181 | availability_zones = ["us-east-1a", "us-east-1b", "us-east-1c", "us-east-1d", "us-east-1f"] 182 | 183 | launch_template { 184 | id = aws_launch_template.charon_worker.id 185 | version = "$Latest" 186 | } 187 | 188 | health_check_type = "EC2" 189 | } 190 | -------------------------------------------------------------------------------- /daemon/catalog.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "log" 7 | "net/http" 8 | "strings" 9 | 10 | "github.com/nix-community/go-nix/pkg/nixbase32" 11 | "github.com/nix-community/go-nix/pkg/storepath" 12 | "go.etcd.io/bbolt" 13 | ) 14 | 15 | const ( 16 | // Use only 80 bits for reverse references to save space in db. 17 | // Collisions may be possible but would only lead to suboptimal diff base choices. 18 | sphPrefixBytes = 10 19 | ) 20 | 21 | type ( 22 | Sph [storepath.PathHashSize]byte 23 | SphPrefix [sphPrefixBytes]byte 24 | 25 | catalogResult struct { 26 | reqName string 27 | baseName string 28 | baseHash Sph 29 | reqHash Sph 30 | } 31 | ) 32 | 33 | func ParseSph(s string) (sph Sph, sphStr string, err error) { 34 | sph, sphStr, _, err = parseSphAndName(s) 35 | return 36 | } 37 | 38 | func ParseSphAndName(s string) (sph Sph, sphStr, spName string, err error) { 39 | sph, sphStr, spName, err = parseSphAndName(s) 40 | if err == nil && spName == "" { 41 | err = mwErr(http.StatusBadRequest, "store path missing name") 42 | } 43 | return 44 | } 45 | 46 | func parseSphAndName(s string) (sph Sph, sphStr, spName string, err error) { 47 | sphStr, spName, _ = strings.Cut(s, "-") 48 | if len(sphStr) != 32 { 49 | err = mwErr(http.StatusBadRequest, "path is not a valid store path") 50 | return 51 | } 52 | var n int 53 | n, err = nixbase32.Decode(sph[:], []byte(sphStr)) 54 | if err != nil || n != len(sph) { 55 | err = mwErr(http.StatusBadRequest, "path is not a valid store path") 56 | } 57 | return 58 | } 59 | 60 | func (res *catalogResult) usingBase() bool { 61 | return res.baseName != "" 62 | } 63 | 64 | func (s Sph) String() string { 65 | return nixbase32.EncodeToString(s[:]) 66 | } 67 | 68 | func SphFromBytes(b []byte) (sph Sph) { 69 | copy(sph[:], b) 70 | return 71 | } 72 | 73 | func SphPrefixFromBytes(b []byte) (sphp SphPrefix) { 74 | copy(sphp[:], b) 75 | return 76 | } 77 | 78 | // sph prefix -> rest of name 79 | func (s *Server) catalogFindName(tx *bbolt.Tx, reqHashPrefix SphPrefix) (Sph, string) { 80 | cur := tx.Bucket(catalogRBucket).Cursor() 81 | // Note that Seek on this prefix will find the first key that matches it. 82 | // It may be the "wrong" one due to a collision since we use only half the bytes. 83 | // That means less than ideal diffing but it won't break anything. 84 | k, v := cur.Seek(reqHashPrefix[:]) 85 | if len(k) < sphPrefixBytes { 86 | return Sph{}, "" 87 | } else if !bytes.Equal(k[:sphPrefixBytes], reqHashPrefix[:]) { 88 | var zerop SphPrefix 89 | log.Printf("mismatched sphp, wanted %s, got %s", 90 | nixbase32.EncodeToString(bytes.Join([][]byte{reqHashPrefix[:], zerop[:]}, nil)), 91 | nixbase32.EncodeToString(k)) 92 | return Sph{}, "" 93 | } 94 | return SphFromBytes(k), string(v) 95 | } 96 | 97 | // given a hash, find another hash that we think is the most similar candidate 98 | func (s *Server) catalogFindBase(tx *bbolt.Tx, reqHashPrefix SphPrefix) (catalogResult, error) { 99 | reqHash, reqName := s.catalogFindName(tx, reqHashPrefix) 100 | return s.catalogFindBaseFromHashAndName(tx, reqHash, reqName) 101 | } 102 | 103 | func (s *Server) catalogFindBaseFromHashAndName(tx *bbolt.Tx, reqHash Sph, reqName string) (catalogResult, error) { 104 | if len(reqName) == 0 { 105 | return catalogResult{}, errors.New("store path hash not found") 106 | } else if len(reqName) < 3 { 107 | return catalogResult{}, errors.New("name too short") 108 | } else if reqName == "source" { 109 | // TODO: need contents similarity for this one 110 | return catalogResult{}, errors.New("can't handle 'source'") 111 | } 112 | 113 | // The "name" part of store paths sometimes has a nice pname-version split like 114 | // "rsync-3.2.6". But also can be something like "rtl8723bs-firmware-2017-04-06-xz" or 115 | // "sane-desc-generate-entries-unsupported-scanners.patch" or 116 | // "python3.10-websocket-client-1.4.1" or "lz4-1.9.4-dev" or of course just "source". 117 | // 118 | // So given another store path name, how do we find suitable candidates? We're looking for 119 | // something where just the version has changed, or maybe an exact match of the name. Let's 120 | // look at segments separated by dashes. We can definitely reject anything that doesn't 121 | // share at least one segment. We should also reject anything that doesn't have the same 122 | // number of segments, since those are probably other outputs or otherwise separate things. 123 | // Then we can pick one that has the most segments in common. 124 | 125 | firstDash := strings.IndexByte(reqName, '-') 126 | numDashes := strings.Count(reqName, "-") 127 | var start string 128 | if firstDash < 0 { 129 | start = reqName 130 | } else { 131 | start = reqName[:firstDash+1] 132 | } 133 | startb := []byte(start) 134 | 135 | var bestmatch int 136 | var besthash Sph 137 | var bestname string 138 | 139 | // look at everything that matches up to the first dash 140 | cur := tx.Bucket(catalogFBucket).Cursor() 141 | for k, _ := cur.Seek(startb); k != nil && bytes.HasPrefix(k, startb); k, _ = cur.Next() { 142 | name, hash, found := bytes.Cut(k, []byte{0}) 143 | if !found { 144 | continue // this is a bug 145 | } 146 | sph := SphFromBytes(hash) 147 | if sph != reqHash && bytes.Count(name, []byte{'-'}) == numDashes { 148 | // take last best instead of first since it's probably more recent 149 | if match := matchLen(reqName, name); match >= bestmatch { 150 | bestmatch = match 151 | bestname = string(name) 152 | besthash = sph 153 | } 154 | } 155 | } 156 | 157 | if bestname == "" { 158 | return catalogResult{}, errors.New("no diff base for " + reqName) 159 | } 160 | 161 | return catalogResult{ 162 | reqName: reqName, 163 | baseName: bestname, 164 | baseHash: besthash, 165 | reqHash: reqHash, 166 | }, nil 167 | } 168 | 169 | func matchLen(a string, b []byte) int { 170 | i := 0 171 | for ; i < len(a) && i < len(b) && a[i] == b[i]; i++ { 172 | } 173 | return i 174 | } 175 | -------------------------------------------------------------------------------- /daemon/debug.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "context" 5 | "encoding/binary" 6 | "log" 7 | 8 | "go.etcd.io/bbolt" 9 | "google.golang.org/protobuf/proto" 10 | 11 | "github.com/dnr/styx/common" 12 | "github.com/dnr/styx/common/cdig" 13 | "github.com/dnr/styx/pb" 14 | ) 15 | 16 | func (s *Server) handleDebugReq(ctx context.Context, r *DebugReq) (*DebugResp, error) { 17 | // allow this even before "initialized" 18 | 19 | res := &DebugResp{ 20 | DbStats: s.db.Stats(), 21 | } 22 | return res, s.db.View(func(tx *bbolt.Tx) error { 23 | // meta 24 | var dp pb.DbParams 25 | _ = proto.Unmarshal(tx.Bucket(metaBucket).Get(metaParams), &dp) 26 | res.Params = &dp 27 | 28 | // stats 29 | res.Stats = s.stats.export() 30 | 31 | // images 32 | if r.IncludeAllImages || len(r.IncludeImages) > 0 { 33 | res.Images = make(map[string]DebugImage) 34 | 35 | doImage := func(k, v []byte) { 36 | if v == nil { 37 | return 38 | } 39 | var img pb.DbImage 40 | if err := proto.Unmarshal(v, &img); err != nil { 41 | log.Print("unmarshal error iterating images", err) 42 | return 43 | } 44 | 45 | m, mdigs, err := s.getManifestLocal(tx, string(k)) 46 | if err != nil { 47 | log.Print("unmarshal getting manifest iterating images", err) 48 | return 49 | } 50 | var mchunks []string 51 | for _, mdig := range mdigs { 52 | mchunks = append(mchunks, mdig.String()) 53 | } 54 | var tchunks, tblocks, pchunks, pblocks int 55 | for _, ent := range m.Entries { 56 | ent.StatsInlineData = int32(len(ent.InlineData)) 57 | digests := cdig.FromSliceAlias(ent.Digests) 58 | tchunks += len(digests) 59 | cshift := ent.ChunkShiftDef() 60 | for i, dig := range digests { 61 | chunkSize := cshift.FileChunkSize(ent.Size, i == len(digests)-1) 62 | blocks := s.blockShift.Blocks(chunkSize) 63 | tblocks += int(blocks) 64 | if _, present := s.digestPresent(tx, dig); present { 65 | ent.StatsPresentChunks++ 66 | ent.StatsPresentBlocks += int32(blocks) 67 | pchunks += 1 68 | pblocks += int(blocks) 69 | } 70 | ent.DebugDigests = append(ent.DebugDigests, dig.String()) 71 | } 72 | ent.InlineData = nil 73 | ent.Digests = nil 74 | } 75 | 76 | res.Images[img.StorePath] = DebugImage{ 77 | Image: &img, 78 | Manifest: m, 79 | ManifestChunks: mchunks, 80 | Stats: DebugSizeStats{ 81 | TotalChunks: tchunks, 82 | TotalBlocks: tblocks, 83 | PresentChunks: pchunks, 84 | PresentBlocks: pblocks, 85 | }, 86 | } 87 | img.StorePath = "" 88 | } 89 | 90 | if r.IncludeAllImages { 91 | cur := tx.Bucket(imageBucket).Cursor() 92 | for k, v := cur.First(); k != nil; k, v = cur.Next() { 93 | doImage(k, v) 94 | } 95 | } else { 96 | for _, img := range r.IncludeImages { 97 | doImage([]byte(img), tx.Bucket(imageBucket).Get([]byte(img))) 98 | } 99 | } 100 | } 101 | 102 | // slabs 103 | if r.IncludeSlabs { 104 | slabroot := tx.Bucket(slabBucket) 105 | cur := slabroot.Cursor() 106 | for k, _ := cur.First(); k != nil; k, _ = cur.Next() { 107 | blockSizes := make(map[uint32]uint32) 108 | sb := slabroot.Bucket(k) 109 | si := DebugSlabInfo{ 110 | Index: binary.BigEndian.Uint16(k), 111 | ChunkSizeDist: make(map[uint32]int), 112 | } 113 | scur := sb.Cursor() 114 | for sk, _ := scur.First(); sk != nil; { 115 | nextSk, _ := scur.Next() 116 | addr := addrFromKey(sk) 117 | if addr&presentMask == 0 { 118 | var nextAddr uint32 119 | if nextSk != nil && nextSk[0]&0x80 == 0 { 120 | nextAddr = addrFromKey(nextSk) 121 | } else { 122 | nextAddr = common.TruncU32(sb.Sequence()) 123 | } 124 | blockSize := uint32(nextAddr - addr) 125 | blockSizes[addr] = blockSize 126 | si.Stats.TotalChunks++ 127 | si.Stats.TotalBlocks += int(blockSize) 128 | si.ChunkSizeDist[blockSize]++ 129 | } else { 130 | si.Stats.PresentChunks++ 131 | si.Stats.PresentBlocks += int(blockSizes[addr&^presentMask]) 132 | } 133 | sk = nextSk 134 | } 135 | res.Slabs = append(res.Slabs, &si) 136 | } 137 | } 138 | 139 | // chunks 140 | if r.IncludeAllChunks || len(r.IncludeChunks) > 0 { 141 | slabroot := tx.Bucket(slabBucket) 142 | cb := tx.Bucket(chunkBucket) 143 | res.Chunks = make(map[string]*DebugChunkInfo) 144 | 145 | doChunk := func(k, v []byte) { 146 | if len(k) < cdig.Bytes { 147 | log.Println("too short chunk key in bucket", k) 148 | return 149 | } 150 | var ci DebugChunkInfo 151 | loc := loadLoc(v) 152 | ci.Slab, ci.Addr = loc.SlabId, loc.Addr 153 | for _, sphp := range sphpsFromLoc(v) { 154 | sph, name := s.catalogFindName(tx, sphp) 155 | ci.StorePaths = append(ci.StorePaths, sph.String()+"-"+name) 156 | } 157 | ci.Present = slabroot.Bucket(slabKey(ci.Slab)).Get(addrKey(ci.Addr|presentMask)) != nil 158 | res.Chunks[cdig.FromBytes(k).String()] = &ci 159 | } 160 | 161 | if r.IncludeAllChunks { 162 | cur := cb.Cursor() 163 | for k, v := cur.First(); k != nil; k, v = cur.Next() { 164 | doChunk(k, v) 165 | } 166 | } else { 167 | for _, cstr := range r.IncludeChunks { 168 | dig, err := cdig.FromBase64(cstr) 169 | if err != nil { 170 | log.Println("debug chunk parse error", cstr, err) 171 | } else if v := cb.Get(dig[:]); v == nil { 172 | log.Println("debug chunk missing", cstr) 173 | } else { 174 | doChunk(dig[:], v) 175 | } 176 | } 177 | } 178 | } 179 | 180 | // chunk sharing 181 | if r.IncludeChunkSharing { 182 | m := make(map[int]int) 183 | cur := tx.Bucket(chunkBucket).Cursor() 184 | for k, v := cur.First(); k != nil; k, v = cur.Next() { 185 | m[(len(v)-6)/sphPrefixBytes]++ 186 | } 187 | res.ChunkSharingDist = m 188 | } 189 | 190 | return nil 191 | }) 192 | } 193 | -------------------------------------------------------------------------------- /pb/manifest_meta.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.36.10 4 | // protoc v6.32.1 5 | // source: manifest_meta.proto 6 | 7 | package pb 8 | 9 | import ( 10 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 11 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 12 | reflect "reflect" 13 | sync "sync" 14 | unsafe "unsafe" 15 | ) 16 | 17 | const ( 18 | // Verify that this generated code is sufficiently up-to-date. 19 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 20 | // Verify that runtime/protoimpl is sufficiently up-to-date. 21 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 22 | ) 23 | 24 | type ManifestMeta struct { 25 | state protoimpl.MessageState `protogen:"open.v1"` 26 | // meta info for what this manifest was generated from 27 | NarinfoUrl string `protobuf:"bytes,1,opt,name=narinfo_url,json=narinfoUrl,proto3" json:"narinfo_url,omitempty"` // url that narinfo was fetched from 28 | Narinfo *NarInfo `protobuf:"bytes,2,opt,name=narinfo,proto3" json:"narinfo,omitempty"` // parsed or generated narinfo (includes references, signatures, etc.) 29 | GenericTarballOriginal string `protobuf:"bytes,3,opt,name=generic_tarball_original,json=genericTarballOriginal,proto3" json:"generic_tarball_original,omitempty"` // generic tarball origin and final url 30 | GenericTarballResolved string `protobuf:"bytes,4,opt,name=generic_tarball_resolved,json=genericTarballResolved,proto3" json:"generic_tarball_resolved,omitempty"` 31 | Generator string `protobuf:"bytes,10,opt,name=generator,proto3" json:"generator,omitempty"` // software version of generator 32 | GeneratedTime int64 `protobuf:"varint,11,opt,name=generated_time,json=generatedTime,proto3" json:"generated_time,omitempty"` // timestamp when this was generated (unix seconds) 33 | unknownFields protoimpl.UnknownFields 34 | sizeCache protoimpl.SizeCache 35 | } 36 | 37 | func (x *ManifestMeta) Reset() { 38 | *x = ManifestMeta{} 39 | mi := &file_manifest_meta_proto_msgTypes[0] 40 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 41 | ms.StoreMessageInfo(mi) 42 | } 43 | 44 | func (x *ManifestMeta) String() string { 45 | return protoimpl.X.MessageStringOf(x) 46 | } 47 | 48 | func (*ManifestMeta) ProtoMessage() {} 49 | 50 | func (x *ManifestMeta) ProtoReflect() protoreflect.Message { 51 | mi := &file_manifest_meta_proto_msgTypes[0] 52 | if x != nil { 53 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 54 | if ms.LoadMessageInfo() == nil { 55 | ms.StoreMessageInfo(mi) 56 | } 57 | return ms 58 | } 59 | return mi.MessageOf(x) 60 | } 61 | 62 | // Deprecated: Use ManifestMeta.ProtoReflect.Descriptor instead. 63 | func (*ManifestMeta) Descriptor() ([]byte, []int) { 64 | return file_manifest_meta_proto_rawDescGZIP(), []int{0} 65 | } 66 | 67 | func (x *ManifestMeta) GetNarinfoUrl() string { 68 | if x != nil { 69 | return x.NarinfoUrl 70 | } 71 | return "" 72 | } 73 | 74 | func (x *ManifestMeta) GetNarinfo() *NarInfo { 75 | if x != nil { 76 | return x.Narinfo 77 | } 78 | return nil 79 | } 80 | 81 | func (x *ManifestMeta) GetGenericTarballOriginal() string { 82 | if x != nil { 83 | return x.GenericTarballOriginal 84 | } 85 | return "" 86 | } 87 | 88 | func (x *ManifestMeta) GetGenericTarballResolved() string { 89 | if x != nil { 90 | return x.GenericTarballResolved 91 | } 92 | return "" 93 | } 94 | 95 | func (x *ManifestMeta) GetGenerator() string { 96 | if x != nil { 97 | return x.Generator 98 | } 99 | return "" 100 | } 101 | 102 | func (x *ManifestMeta) GetGeneratedTime() int64 { 103 | if x != nil { 104 | return x.GeneratedTime 105 | } 106 | return 0 107 | } 108 | 109 | var File_manifest_meta_proto protoreflect.FileDescriptor 110 | 111 | const file_manifest_meta_proto_rawDesc = "" + 112 | "\n" + 113 | "\x13manifest_meta.proto\x12\x02pb\x1a\rnarinfo.proto\"\x8f\x02\n" + 114 | "\fManifestMeta\x12\x1f\n" + 115 | "\vnarinfo_url\x18\x01 \x01(\tR\n" + 116 | "narinfoUrl\x12%\n" + 117 | "\anarinfo\x18\x02 \x01(\v2\v.pb.NarInfoR\anarinfo\x128\n" + 118 | "\x18generic_tarball_original\x18\x03 \x01(\tR\x16genericTarballOriginal\x128\n" + 119 | "\x18generic_tarball_resolved\x18\x04 \x01(\tR\x16genericTarballResolved\x12\x1c\n" + 120 | "\tgenerator\x18\n" + 121 | " \x01(\tR\tgenerator\x12%\n" + 122 | "\x0egenerated_time\x18\v \x01(\x03R\rgeneratedTimeB\x18Z\x16github.com/dnr/styx/pbb\x06proto3" 123 | 124 | var ( 125 | file_manifest_meta_proto_rawDescOnce sync.Once 126 | file_manifest_meta_proto_rawDescData []byte 127 | ) 128 | 129 | func file_manifest_meta_proto_rawDescGZIP() []byte { 130 | file_manifest_meta_proto_rawDescOnce.Do(func() { 131 | file_manifest_meta_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_manifest_meta_proto_rawDesc), len(file_manifest_meta_proto_rawDesc))) 132 | }) 133 | return file_manifest_meta_proto_rawDescData 134 | } 135 | 136 | var file_manifest_meta_proto_msgTypes = make([]protoimpl.MessageInfo, 1) 137 | var file_manifest_meta_proto_goTypes = []any{ 138 | (*ManifestMeta)(nil), // 0: pb.ManifestMeta 139 | (*NarInfo)(nil), // 1: pb.NarInfo 140 | } 141 | var file_manifest_meta_proto_depIdxs = []int32{ 142 | 1, // 0: pb.ManifestMeta.narinfo:type_name -> pb.NarInfo 143 | 1, // [1:1] is the sub-list for method output_type 144 | 1, // [1:1] is the sub-list for method input_type 145 | 1, // [1:1] is the sub-list for extension type_name 146 | 1, // [1:1] is the sub-list for extension extendee 147 | 0, // [0:1] is the sub-list for field type_name 148 | } 149 | 150 | func init() { file_manifest_meta_proto_init() } 151 | func file_manifest_meta_proto_init() { 152 | if File_manifest_meta_proto != nil { 153 | return 154 | } 155 | file_narinfo_proto_init() 156 | type x struct{} 157 | out := protoimpl.TypeBuilder{ 158 | File: protoimpl.DescBuilder{ 159 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 160 | RawDescriptor: unsafe.Slice(unsafe.StringData(file_manifest_meta_proto_rawDesc), len(file_manifest_meta_proto_rawDesc)), 161 | NumEnums: 0, 162 | NumMessages: 1, 163 | NumExtensions: 0, 164 | NumServices: 0, 165 | }, 166 | GoTypes: file_manifest_meta_proto_goTypes, 167 | DependencyIndexes: file_manifest_meta_proto_depIdxs, 168 | MessageInfos: file_manifest_meta_proto_msgTypes, 169 | }.Build() 170 | File_manifest_meta_proto = out.File 171 | file_manifest_meta_proto_goTypes = nil 172 | file_manifest_meta_proto_depIdxs = nil 173 | } 174 | -------------------------------------------------------------------------------- /daemon/tarball.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "context" 5 | "encoding/hex" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "log" 10 | "net" 11 | "net/http" 12 | "regexp" 13 | "time" 14 | 15 | "github.com/dnr/styx/common" 16 | "github.com/dnr/styx/common/cdig" 17 | "github.com/dnr/styx/manifester" 18 | "github.com/dnr/styx/pb" 19 | "github.com/nix-community/go-nix/pkg/hash" 20 | "github.com/nix-community/go-nix/pkg/narinfo" 21 | "github.com/nix-community/go-nix/pkg/nixbase32" 22 | "go.etcd.io/bbolt" 23 | "google.golang.org/protobuf/proto" 24 | ) 25 | 26 | // local binary cache to serve narinfo 27 | 28 | var errNotFound = errors.New("not found") 29 | 30 | func (s *Server) getFakeCacheData(sphStr string) (*pb.FakeCacheData, error) { 31 | var data pb.FakeCacheData 32 | err := s.db.View(func(tx *bbolt.Tx) error { 33 | v := tx.Bucket(fakeCacheBucket).Get([]byte(sphStr)) 34 | if v == nil { 35 | return errNotFound 36 | } 37 | return proto.Unmarshal(v, &data) 38 | }) 39 | return common.ValOrErr(&data, err) 40 | } 41 | 42 | func (s *Server) startFakeCacheServer() (err error) { 43 | l, err := net.Listen("tcp", fakeCacheBind) 44 | if err != nil { 45 | return fmt.Errorf("failed to listen on local tcp socket %s: %w", fakeCacheBind, err) 46 | } 47 | mux := http.NewServeMux() 48 | mux.HandleFunc("/nix-cache-info", s.getFakeCacheInfo) 49 | mux.HandleFunc("/", s.getFakeNarinfo) 50 | s.shutdownWait.Add(1) 51 | go func() { 52 | defer s.shutdownWait.Done() 53 | srv := &http.Server{Handler: mux} 54 | go srv.Serve(l) 55 | <-s.shutdownChan 56 | log.Printf("stopping fake cache server") 57 | srv.Close() 58 | }() 59 | return nil 60 | } 61 | 62 | func (s *Server) getFakeCacheInfo(w http.ResponseWriter, r *http.Request) { 63 | if r.Method != http.MethodGet { 64 | http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) 65 | return 66 | } 67 | w.Header().Set("Content-Type", "text/x-nix-cache-info") 68 | w.Write([]byte("StoreDir: /nix/store\nWantMassQuery: 0\nPriority: 90\n")) 69 | } 70 | 71 | var reNarinfoPath = regexp.MustCompile(`^/([` + nixbase32.Alphabet + `]+)\.narinfo$`) 72 | 73 | func (s *Server) getFakeNarinfo(w http.ResponseWriter, r *http.Request) { 74 | if r.Method != http.MethodGet && r.Method != http.MethodHead { 75 | http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) 76 | } else if m := reNarinfoPath.FindStringSubmatch(r.URL.Path); m == nil { 77 | http.NotFound(w, r) 78 | } else if data, err := s.getFakeCacheData(m[1]); err != nil { 79 | http.NotFound(w, r) 80 | } else { 81 | w.Header().Set("Content-Type", "text/x-nix-narinfo") 82 | w.Write(data.Narinfo) 83 | } 84 | } 85 | 86 | // request handler 87 | 88 | func (s *Server) handleTarballReq(ctx context.Context, r *TarballReq) (*TarballResp, error) { 89 | if s.p() == nil { 90 | return nil, mwErr(http.StatusPreconditionFailed, "styx is not initialized, call 'styx init --params=...'") 91 | } 92 | 93 | // we only have a url at this point, not a sph. the url may redirect to a more permanent 94 | // url. do a head request to resolve and get at least an etag if possible. 95 | // TODO: consider "lockable tarball protocol" 96 | res, err := common.RetryHttpRequest(ctx, http.MethodHead, r.UpstreamUrl, "", nil) 97 | if err != nil { 98 | return nil, err 99 | } 100 | io.Copy(io.Discard, res.Body) 101 | res.Body.Close() 102 | 103 | resolved := res.Request.URL.String() 104 | 105 | // use generic tarball build mode 106 | mReq := manifester.ManifestReq{ 107 | Upstream: resolved, 108 | BuildMode: manifester.ModeGenericTarball, 109 | DigestAlgo: cdig.Algo, 110 | DigestBits: int(cdig.Bits), 111 | } 112 | 113 | var envelopeBytes []byte 114 | 115 | // if we have an etag we can try a cache lookup 116 | if etag := res.Header.Get("Etag"); etag != "" { 117 | // we have an etag, we can try a cache lookup 118 | mReq.ETag = etag 119 | envelopeBytes, _ = s.p().mcread.Get(ctx, mReq.CacheKey(), nil) 120 | mReq.ETag = "" 121 | } 122 | 123 | // fall back to asking manifester 124 | if envelopeBytes == nil { 125 | envelopeBytes, err = s.getNewManifest(ctx, mReq, r.Shards) 126 | if err != nil { 127 | return nil, err 128 | } 129 | } 130 | 131 | // verify signature and params 132 | entry, _, err := common.VerifyMessageAsEntry(s.p().keys, common.ManifestContext, envelopeBytes) 133 | if err != nil { 134 | return nil, err 135 | } 136 | 137 | // TODO: we should check the envelope context against what we asked for, but we don't know 138 | // the sph of what we asked for, we either know an etag (md5sum) or nothing. so we can't 139 | // really check any more than the signature above. 140 | 141 | // get the narinfo that the manifester produced so we can add it to our fake cache. 142 | // try embedded meta in entry first (will be present on chunked manifests). 143 | mm := entry.ManifestMeta 144 | if mm == nil && len(entry.InlineData) > 0 { 145 | // manifest is inline, use that 146 | var m pb.Manifest 147 | err = proto.Unmarshal(entry.InlineData, &m) 148 | if err != nil { 149 | return nil, fmt.Errorf("manifest unmarshal error: %w", err) 150 | } 151 | mm = m.Meta 152 | } 153 | nipb := mm.GetNarinfo() 154 | if nipb == nil { 155 | return nil, fmt.Errorf("entry missing inline manifest metadata") 156 | } 157 | 158 | fh, err := hash.ParseNixBase32(nipb.FileHash) 159 | if err != nil { 160 | return nil, err 161 | } 162 | nh, err := hash.ParseNixBase32(nipb.NarHash) 163 | if err != nil { 164 | return nil, err 165 | } 166 | 167 | ni := narinfo.NarInfo{ 168 | StorePath: nipb.StorePath, 169 | URL: "nar/dummy.nar", // we'll use styx 170 | Compression: "none", 171 | FileHash: fh, 172 | FileSize: uint64(nipb.FileSize), 173 | NarHash: nh, 174 | NarSize: uint64(nipb.NarSize), 175 | // needed to make nix treat it as CA so doesn't it require a signature 176 | CA: "fixed:r:" + nh.NixString(), 177 | } 178 | 179 | sph := nipb.StorePath[11:43] 180 | b, err := proto.Marshal(&pb.FakeCacheData{ 181 | Narinfo: []byte(ni.String()), 182 | Upstream: mm.GenericTarballResolved, 183 | Updated: time.Now().Unix(), 184 | }) 185 | if err != nil { 186 | return nil, err 187 | } 188 | // TODO: prune this cache once in a while 189 | err = s.db.Update(func(tx *bbolt.Tx) error { 190 | return tx.Bucket(fakeCacheBucket).Put([]byte(sph), b) 191 | }) 192 | if err != nil { 193 | return nil, err 194 | } 195 | 196 | return &TarballResp{ 197 | ResolvedUrl: mm.GenericTarballResolved, 198 | StorePathHash: sph, 199 | Name: nipb.StorePath[44:], 200 | NarHash: hex.EncodeToString(nh.Digest()), 201 | NarHashAlgo: nh.HashTypeString(), 202 | }, nil 203 | } 204 | -------------------------------------------------------------------------------- /manifester/chunkstore.go: -------------------------------------------------------------------------------- 1 | package manifester 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "io/fs" 10 | "net/http" 11 | "os" 12 | "path" 13 | "strings" 14 | 15 | "github.com/aws/aws-sdk-go-v2/aws" 16 | awsconfig "github.com/aws/aws-sdk-go-v2/config" 17 | s3 "github.com/aws/aws-sdk-go-v2/service/s3" 18 | s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" 19 | 20 | "github.com/dnr/styx/common" 21 | ) 22 | 23 | type ( 24 | ChunkStoreWrite interface { 25 | PutIfNotExists(ctx context.Context, path, key string, data []byte) ([]byte, error) 26 | Get(ctx context.Context, path, key string, dst []byte) ([]byte, error) 27 | } 28 | 29 | ChunkStoreRead interface { 30 | // Data will be appended to dst and returned. 31 | // If dst is given, it must be big enough to hold the full chunk! 32 | Get(ctx context.Context, key string, dst []byte) ([]byte, error) 33 | } 34 | 35 | ChunkStoreWriteConfig struct { 36 | // One of these is required: 37 | ChunkBucket string 38 | ChunkLocalDir string 39 | ZstdEncoderLevel int 40 | } 41 | 42 | localChunkStoreWrite struct { 43 | dir string 44 | zp *common.ZstdCtxPool 45 | } 46 | 47 | s3ChunkStoreWrite struct { 48 | bucket string 49 | s3client *s3.Client 50 | zp *common.ZstdCtxPool 51 | zlevel int 52 | } 53 | 54 | urlChunkStoreRead struct { 55 | url string 56 | zp *common.ZstdCtxPool 57 | } 58 | ) 59 | 60 | func newLocalChunkStoreWrite(dir string) (*localChunkStoreWrite, error) { 61 | if err := os.MkdirAll(dir, 0755); err != nil { 62 | return nil, err 63 | } 64 | return &localChunkStoreWrite{dir: dir, zp: common.GetZstdCtxPool()}, nil 65 | } 66 | 67 | func (l *localChunkStoreWrite) PutIfNotExists(ctx context.Context, path_, key string, data []byte) ([]byte, error) { 68 | // ignore path! mix chunks and manifests in same directory 69 | z := l.zp.Get() 70 | defer l.zp.Put(z) 71 | fn := path.Join(l.dir, key) 72 | if _, err := os.Stat(fn); err == nil { 73 | return nil, nil 74 | } else if d, err := z.CompressLevel(nil, data, 1); err != nil { 75 | return nil, err 76 | } else if out, err := os.CreateTemp(l.dir, key+".tmp*"); err != nil { 77 | return nil, err 78 | } else if n, err := out.Write(d); err != nil || n != len(d) { 79 | _ = out.Close() 80 | _ = os.Remove(out.Name()) 81 | return nil, err 82 | } else if err := out.Close(); err != nil { 83 | _ = os.Remove(out.Name()) 84 | return nil, err 85 | } else if err := os.Rename(out.Name(), fn); err != nil { 86 | _ = os.Remove(out.Name()) 87 | return nil, err 88 | } else { 89 | return d, nil 90 | } 91 | } 92 | 93 | func (l *localChunkStoreWrite) Get(ctx context.Context, path_, key string, data []byte) ([]byte, error) { 94 | // ignore path! mix chunks and manifests in same directory 95 | b, err := os.ReadFile(path.Join(l.dir, key)) 96 | if err != nil { 97 | return nil, wrapNotFound(err) 98 | } 99 | z := l.zp.Get() 100 | defer l.zp.Put(z) 101 | return z.Decompress(data, b) 102 | } 103 | 104 | func newS3ChunkStoreWrite(bucket string, zlevel int) (*s3ChunkStoreWrite, error) { 105 | awscfg, err := awsconfig.LoadDefaultConfig(context.Background(), awsconfig.WithEC2IMDSRegion()) 106 | if err != nil { 107 | return nil, err 108 | } 109 | s3client := s3.NewFromConfig(awscfg, func(o *s3.Options) { 110 | o.EndpointOptions.DisableHTTPS = true 111 | o.RetryMaxAttempts = 15 112 | }) 113 | return &s3ChunkStoreWrite{ 114 | bucket: bucket, 115 | s3client: s3client, 116 | zp: common.GetZstdCtxPool(), 117 | zlevel: zlevel, 118 | }, nil 119 | } 120 | 121 | func (s *s3ChunkStoreWrite) PutIfNotExists(ctx context.Context, path, key string, data []byte) ([]byte, error) { 122 | if path != ChunkReadPath && path != ManifestCachePath && path != BuildRootPath { 123 | panic("path must be ChunkReadPath or ManifestCachePath") 124 | } 125 | key = path[1:] + key 126 | _, err := s.s3client.HeadObject(ctx, &s3.HeadObjectInput{ 127 | Bucket: &s.bucket, 128 | Key: &key, 129 | }) 130 | if err == nil || !IsS3NotFound(err) { 131 | return nil, err 132 | } 133 | z := s.zp.Get() 134 | defer s.zp.Put(z) 135 | // TODO: use buffer pool here (requires caller to return it?) 136 | d, err := z.CompressLevel(nil, data, s.zlevel) 137 | if err != nil { 138 | return nil, err 139 | } 140 | _, err = s.s3client.PutObject(ctx, &s3.PutObjectInput{ 141 | Bucket: &s.bucket, 142 | Key: &key, 143 | Body: bytes.NewReader(d), 144 | CacheControl: aws.String("public, max-age=31536000"), 145 | ContentType: aws.String("application/octet-stream"), 146 | ContentEncoding: aws.String("zstd"), 147 | }) 148 | return d, err 149 | } 150 | 151 | func (s *s3ChunkStoreWrite) Get(ctx context.Context, path, key string, data []byte) ([]byte, error) { 152 | key = path[1:] + key 153 | res, err := s.s3client.GetObject(ctx, &s3.GetObjectInput{ 154 | Bucket: &s.bucket, 155 | Key: &key, 156 | }) 157 | if err != nil { 158 | return nil, fmt.Errorf("get(%q): %w", key, wrapNotFound(err)) 159 | } 160 | defer res.Body.Close() 161 | b, err := io.ReadAll(res.Body) 162 | if err != nil { 163 | return nil, fmt.Errorf("read(%q): %w", key, err) 164 | } 165 | z := s.zp.Get() 166 | defer s.zp.Put(z) 167 | return z.Decompress(data, b) 168 | } 169 | 170 | func NewChunkStoreWrite(cfg ChunkStoreWriteConfig) (ChunkStoreWrite, error) { 171 | if len(cfg.ChunkLocalDir) > 0 { 172 | return newLocalChunkStoreWrite(cfg.ChunkLocalDir) 173 | } else if len(cfg.ChunkBucket) > 0 { 174 | return newS3ChunkStoreWrite(cfg.ChunkBucket, cfg.ZstdEncoderLevel) 175 | } 176 | return nil, errors.New("chunk store configuration is missing") 177 | } 178 | 179 | // path should be either ChunkReadPath or ManifestCachePath 180 | func NewChunkStoreReadUrl(url, path string) ChunkStoreRead { 181 | if path != ChunkReadPath && path != ManifestCachePath { 182 | panic("path must be ChunkReadPath or ManifestCachePath") 183 | } 184 | return &urlChunkStoreRead{ 185 | url: strings.TrimSuffix(url, "/") + path, 186 | zp: common.GetZstdCtxPool(), 187 | } 188 | } 189 | 190 | func (s *urlChunkStoreRead) Get(ctx context.Context, key string, dst []byte) ([]byte, error) { 191 | url := s.url + key 192 | res, err := common.RetryHttpRequest(ctx, http.MethodGet, url, "", nil) 193 | if err != nil { 194 | return nil, err 195 | } 196 | defer res.Body.Close() 197 | if res.StatusCode != http.StatusOK { 198 | return nil, common.HttpErrorFromRes(res) 199 | } 200 | b, err := io.ReadAll(res.Body) 201 | if err != nil { 202 | return nil, err 203 | } 204 | if res.Header.Get("Content-Encoding") == "zstd" { 205 | z := s.zp.Get() 206 | defer s.zp.Put(z) 207 | if dst == nil { 208 | return z.Decompress(nil, b) 209 | } else { 210 | // fast path, assume buffer is big enough 211 | n, err := z.DecompressInto(dst[len(dst):cap(dst)], b) 212 | if err != nil { 213 | return nil, err 214 | } 215 | return dst[:len(dst)+n], nil 216 | } 217 | } else if dst == nil { 218 | return b, nil 219 | } else { 220 | return append(dst, b...), nil 221 | } 222 | } 223 | 224 | type wrapNotFoundErr struct{ error } 225 | 226 | var _ common.NotFoundable = wrapNotFoundErr{} 227 | 228 | func (wrapNotFoundErr) IsNotFound() bool { return true } 229 | 230 | func wrapNotFound(err error) error { 231 | if errors.Is(err, fs.ErrNotExist) || IsS3NotFound(err) { 232 | return wrapNotFoundErr{err} 233 | } 234 | return err 235 | } 236 | 237 | func IsS3NotFound(err error) bool { 238 | // the S3 sdk is inconsistent about these, just check both 239 | var nsk *s3types.NoSuchKey 240 | var nf *s3types.NotFound 241 | return errors.As(err, &nsk) || errors.As(err, &nf) 242 | } 243 | --------------------------------------------------------------------------------