├── .envrc ├── .gitignore ├── docs ├── assets │ └── dev.gif └── vhs │ └── dev.tape ├── nixos ├── default.nix └── store.nix ├── pkg ├── util │ └── iterator.go ├── subject │ └── subject.go ├── test │ ├── grpc.go │ └── nats.go ├── cli │ ├── log.go │ └── nats.go ├── store │ ├── types.go │ ├── cache.go │ ├── nats_test.go │ ├── nats.go │ ├── cdc.go │ └── cdc_test.go ├── blob │ ├── helper_test.go │ ├── config.go │ ├── grpc.go │ └── grpc_test.go ├── directory │ ├── config.go │ ├── directory.go │ └── grpc.go └── pathinfo │ ├── config.go │ └── grpc.go ├── internal └── cli │ ├── store │ ├── store.go │ ├── init.go │ └── run.go │ └── cli.go ├── main.go ├── nix ├── default.nix ├── checks.nix ├── dev │ ├── default.nix │ ├── nvix.nix │ ├── tvix.nix │ └── nats.nix ├── docs.nix ├── formatter.nix ├── packages.nix └── devshell.nix ├── protos ├── blobstore.proto └── blobstore.pb.go ├── default.nix ├── shell.nix ├── LICENSE ├── .github └── workflows │ └── ci.yaml ├── flake.nix ├── go.mod ├── README.md ├── gomod2nix.toml ├── flake.lock └── go.sum /.envrc: -------------------------------------------------------------------------------- 1 | use flake 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .data 3 | result -------------------------------------------------------------------------------- /docs/assets/dev.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmcgee/nvix/HEAD/docs/assets/dev.gif -------------------------------------------------------------------------------- /nixos/default.nix: -------------------------------------------------------------------------------- 1 | { 2 | flake.nixosModules = { 3 | store = import ./store.nix; 4 | }; 5 | } 6 | -------------------------------------------------------------------------------- /pkg/util/iterator.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | type Iterator[T any] interface { 4 | Next() (T, error) 5 | Close() error 6 | } 7 | -------------------------------------------------------------------------------- /internal/cli/store/store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | type Cli struct { 4 | Run Run `cmd:"" default:""` 5 | Init Init `cmd:""` 6 | } 7 | -------------------------------------------------------------------------------- /internal/cli/cli.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "github.com/brianmcgee/nvix/internal/cli/store" 5 | ) 6 | 7 | var Cli struct { 8 | Store store.Cli `cmd:""` 9 | } 10 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/alecthomas/kong" 5 | "github.com/brianmcgee/nvix/internal/cli" 6 | ) 7 | 8 | func main() { 9 | ctx := kong.Parse(&cli.Cli) 10 | ctx.FatalIfErrorf(ctx.Run()) 11 | } 12 | -------------------------------------------------------------------------------- /nix/default.nix: -------------------------------------------------------------------------------- 1 | {inputs, ...}: { 2 | imports = [ 3 | inputs.flake-root.flakeModule 4 | ./checks.nix 5 | ./devshell.nix 6 | ./dev 7 | ./docs.nix 8 | ./formatter.nix 9 | ./packages.nix 10 | ]; 11 | } 12 | -------------------------------------------------------------------------------- /pkg/subject/subject.go: -------------------------------------------------------------------------------- 1 | package subject 2 | 3 | var subjectPrefix = "TVIX" 4 | 5 | func SetPrefix(prefix string) { 6 | subjectPrefix = prefix 7 | } 8 | 9 | func GetPrefix() string { 10 | return subjectPrefix 11 | } 12 | 13 | func WithPrefix(subj string) string { 14 | return subjectPrefix + "." + subj 15 | } 16 | -------------------------------------------------------------------------------- /protos/blobstore.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package nvix.blobstore.v1; 4 | 5 | option go_package = "github.com/brianmcgee/nvix/protos;protos"; 6 | 7 | message BlobMeta { 8 | repeated ChunkMeta chunks = 1; 9 | 10 | message ChunkMeta { 11 | bytes digest = 1; 12 | uint32 size = 2; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /docs/vhs/dev.tape: -------------------------------------------------------------------------------- 1 | Require dev 2 | 3 | Set FontSize 14 4 | 5 | Set Width 1440 6 | Set Height 800 7 | 8 | Type "# How to start local dev services" 9 | Sleep 1s 10 | Enter 11 | Enter 12 | 13 | Type "dev" 14 | Sleep 1s 15 | Enter 16 | 17 | Sleep 1s 18 | Down 19 | Sleep 1s 20 | Down 21 | 22 | Sleep 20s 23 | Ctrl+c 24 | Sleep 1s 25 | Enter 26 | Sleep 1s -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | ( 2 | import 3 | ( 4 | let 5 | lock = builtins.fromJSON (builtins.readFile ./flake.lock); 6 | in 7 | fetchTarball { 8 | url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; 9 | sha256 = lock.nodes.flake-compat.locked.narHash; 10 | } 11 | ) 12 | {src = ./.;} 13 | ) 14 | .defaultNix 15 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | ( 2 | import 3 | ( 4 | let 5 | lock = builtins.fromJSON (builtins.readFile ./flake.lock); 6 | in 7 | fetchTarball { 8 | url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; 9 | sha256 = lock.nodes.flake-compat.locked.narHash; 10 | } 11 | ) 12 | {src = ./.;} 13 | ) 14 | .defaultNix 15 | -------------------------------------------------------------------------------- /pkg/test/grpc.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "net" 5 | 6 | "google.golang.org/grpc" 7 | "google.golang.org/grpc/credentials/insecure" 8 | ) 9 | 10 | func GrpcConn(lis net.Listener, t TestingT) *grpc.ClientConn { 11 | t.Helper() 12 | conn, err := grpc.Dial(lis.Addr().String(), 13 | grpc.WithTransportCredentials(insecure.NewCredentials()), 14 | ) 15 | if err != nil { 16 | t.Fatalf("failed to create a grpc server connection: %v", err) 17 | } 18 | return conn 19 | } 20 | -------------------------------------------------------------------------------- /pkg/cli/log.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "github.com/charmbracelet/log" 5 | ) 6 | 7 | type LogOptions struct { 8 | Verbosity int `name:"verbose" short:"v" type:"counter" default:"0" env:"LOG_LEVEL" help:"Set the verbosity of logs e.g. -vv"` 9 | } 10 | 11 | func (lo *LogOptions) ConfigureLogger() { 12 | if lo.Verbosity == 0 { 13 | log.SetLevel(log.WarnLevel) 14 | } else if lo.Verbosity == 1 { 15 | log.SetLevel(log.InfoLevel) 16 | } else if lo.Verbosity >= 2 { 17 | log.SetLevel(log.DebugLevel) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /nix/checks.nix: -------------------------------------------------------------------------------- 1 | {lib, ...}: { 2 | perSystem = {self', ...}: { 3 | checks = 4 | # merge in the package derivations to force a build of all packages during a `nix flake check` 5 | with lib; mapAttrs' (n: nameValuePair "package-${n}") self'.packages; 6 | 7 | devshells.default = { 8 | commands = [ 9 | { 10 | name = "check"; 11 | help = "Run all linters and build all packages"; 12 | category = "checks"; 13 | command = "nix flake check"; 14 | } 15 | ]; 16 | }; 17 | }; 18 | } 19 | -------------------------------------------------------------------------------- /pkg/cli/nats.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "github.com/charmbracelet/log" 5 | "github.com/nats-io/nats.go" 6 | ) 7 | 8 | type NatsOptions struct { 9 | NatsUrl string `short:"n" env:"NATS_URL" default:"nats://localhost:4222"` 10 | NatsCredentials string `short:"c" env:"NATS_CREDENTIALS_FILE" required:"" type:"path"` 11 | } 12 | 13 | func (no *NatsOptions) Connect() *nats.Conn { 14 | conn, err := nats.Connect(no.NatsUrl, nats.UserCredentials(no.NatsCredentials)) 15 | if err != nil { 16 | log.Fatalf("failed to connect to nats: %v", err) 17 | } 18 | return conn 19 | } 20 | -------------------------------------------------------------------------------- /nix/dev/default.nix: -------------------------------------------------------------------------------- 1 | {...}: { 2 | imports = [ 3 | ./nats.nix 4 | ./nvix.nix 5 | ./tvix.nix 6 | ]; 7 | 8 | perSystem = {self', ...}: { 9 | config.process-compose.dev.settings = { 10 | log_location = "$PRJ_DATA_DIR/dev.log"; 11 | }; 12 | 13 | config.devshells.default = { 14 | commands = [ 15 | { 16 | category = "development"; 17 | help = "Run local dev services"; 18 | package = self'.packages.dev; 19 | } 20 | { 21 | category = "development"; 22 | help = "Re-initialise state for dev services"; 23 | name = "dev-init"; 24 | command = "rm -rf $PRJ_DATA_DIR && direnv reload"; 25 | } 26 | ]; 27 | }; 28 | }; 29 | } 30 | -------------------------------------------------------------------------------- /nix/docs.nix: -------------------------------------------------------------------------------- 1 | { 2 | perSystem = {pkgs, ...}: { 3 | config.devshells.default = { 4 | commands = let 5 | category = "docs"; 6 | in [ 7 | { 8 | inherit category; 9 | package = pkgs.vhs; 10 | } 11 | { 12 | inherit category; 13 | help = "Generate all gifs used in docs"; 14 | package = pkgs.writeShellApplication { 15 | name = "gifs"; 16 | runtimeInputs = [pkgs.vhs]; 17 | text = '' 18 | for tape in "$PRJ_ROOT"/docs/vhs/*; do 19 | vhs "$tape" -o "$PRJ_ROOT/docs/assets/$(basename "$tape" .tape).gif" 20 | done 21 | ''; 22 | }; 23 | } 24 | ]; 25 | }; 26 | }; 27 | } 28 | -------------------------------------------------------------------------------- /nix/formatter.nix: -------------------------------------------------------------------------------- 1 | {inputs, ...}: { 2 | imports = [ 3 | inputs.treefmt-nix.flakeModule 4 | ]; 5 | perSystem = { 6 | config, 7 | pkgs, 8 | ... 9 | }: { 10 | treefmt.config = { 11 | inherit (config.flake-root) projectRootFile; 12 | package = pkgs.treefmt; 13 | 14 | programs = { 15 | alejandra.enable = true; 16 | deadnix.enable = true; 17 | gofumpt.enable = true; 18 | prettier.enable = true; 19 | statix.enable = true; 20 | }; 21 | 22 | settings.formatter.prettier.options = ["--tab-width" "4"]; 23 | }; 24 | 25 | formatter = config.treefmt.build.wrapper; 26 | 27 | devshells.default = { 28 | commands = [ 29 | { 30 | category = "checks"; 31 | name = "fmt"; 32 | help = "Format the repo"; 33 | command = "nix fmt"; 34 | } 35 | ]; 36 | }; 37 | }; 38 | } 39 | -------------------------------------------------------------------------------- /pkg/store/types.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "io" 7 | 8 | "github.com/brianmcgee/nvix/pkg/util" 9 | 10 | "github.com/nats-io/nats.go" 11 | 12 | "github.com/juju/errors" 13 | ) 14 | 15 | const ( 16 | ErrKeyNotFound = errors.ConstError("key not found") 17 | ) 18 | 19 | type Digest [32]byte 20 | 21 | func (d Digest) String() string { 22 | return base64.StdEncoding.EncodeToString(d[:]) 23 | } 24 | 25 | type Store interface { 26 | Init(ctx context.Context) error 27 | Get(key string, ctx context.Context) (io.ReadCloser, error) 28 | Put(key string, reader io.ReadCloser, ctx context.Context) error 29 | PutAsync(key string, reader io.ReadCloser, ctx context.Context) (nats.PubAckFuture, error) 30 | List(ctx context.Context) (util.Iterator[io.ReadCloser], error) 31 | Stat(key string, ctx context.Context) (bool, error) 32 | Delete(key string, ctx context.Context) error 33 | } 34 | -------------------------------------------------------------------------------- /nix/packages.nix: -------------------------------------------------------------------------------- 1 | {inputs, ...}: { 2 | imports = [ 3 | inputs.flake-parts.flakeModules.easyOverlay 4 | ]; 5 | 6 | perSystem = { 7 | lib, 8 | pkgs, 9 | self', 10 | inputs', 11 | ... 12 | }: { 13 | packages = rec { 14 | nvix = inputs'.gomod2nix.legacyPackages.buildGoApplication rec { 15 | pname = "nvix"; 16 | version = "0.0.1+dev"; 17 | 18 | # ensure we are using the same version of go to build with 19 | inherit (pkgs) go; 20 | 21 | src = ../.; 22 | modules = ../gomod2nix.toml; 23 | 24 | ldflags = [ 25 | "-X 'build.Name=${pname}'" 26 | "-X 'build.Version=${version}'" 27 | ]; 28 | 29 | meta = with lib; { 30 | description = "NVIX: a NATS-based store for TVIX"; 31 | homepage = "https://github.com/brianmcgee/nvix"; 32 | license = licenses.mit; 33 | mainProgram = "nvix"; 34 | }; 35 | }; 36 | 37 | default = nvix; 38 | }; 39 | 40 | overlayAttrs = self'.packages; 41 | }; 42 | } 43 | -------------------------------------------------------------------------------- /nix/dev/nvix.nix: -------------------------------------------------------------------------------- 1 | {lib, ...}: { 2 | perSystem = {self', ...}: { 3 | config.process-compose = { 4 | dev.settings.processes = { 5 | nvix-store-init = { 6 | depends_on = { 7 | nats-server.condition = "process_healthy"; 8 | nsc-push.condition = "process_completed_successfully"; 9 | }; 10 | working_dir = "$PRJ_DATA_DIR"; 11 | environment = { 12 | NATS_CREDENTIALS_FILE = "./nsc/creds/Tvix/Store/Admin.creds"; 13 | }; 14 | command = "${lib.getExe self'.packages.nvix} store init -v"; 15 | }; 16 | nvix-store = { 17 | depends_on = { 18 | nvix-store-init.condition = "process_completed_successfully"; 19 | }; 20 | working_dir = "$PRJ_DATA_DIR"; 21 | environment = { 22 | NATS_CREDENTIALS_FILE = "./nsc/creds/Tvix/Store/Server.creds"; 23 | }; 24 | command = "${lib.getExe self'.packages.nvix} store run -v"; 25 | # TODO readiness probe 26 | }; 27 | }; 28 | }; 29 | }; 30 | } 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Brian McGee 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | 11 | build: 12 | strategy: 13 | matrix: 14 | os: [ 'ubuntu-latest', 'macos-latest', 'windows-latest' ] 15 | go: [ '1.20', '1.21' ] 16 | runs-on: ${{ matrix.os }} 17 | name: Build (Go ${{ matrix.go }}, OS ${{ matrix.os }}) 18 | steps: 19 | - uses: actions/checkout@v3 20 | - name: Set up Go 21 | uses: actions/setup-go@v4 22 | with: 23 | go-version: ${{ matrix.go }} 24 | - name: go test -v ./... 25 | run: go test -v ./... 26 | 27 | coverage: 28 | runs-on: 'ubuntu-latest' 29 | name: Coverage 30 | steps: 31 | - uses: actions/checkout@v3 32 | - name: Set up Go 33 | uses: actions/setup-go@v4 34 | with: 35 | go-version: '1.20' 36 | - name: Run coverage 37 | run: go test -covermode=atomic -coverprofile=coverage.out -v ./... 38 | - name: Convert coverage.out to coverage.lcov 39 | uses: jandelgado/gcov2lcov-action@v1 40 | - name: Coveralls 41 | uses: coverallsapp/github-action@v2 42 | with: 43 | github-token: ${{ secrets.github_token }} 44 | path-to-lcov: coverage.lcov -------------------------------------------------------------------------------- /nix/dev/tvix.nix: -------------------------------------------------------------------------------- 1 | {inputs, ...}: { 2 | perSystem = { 3 | pkgs, 4 | system, 5 | ... 6 | }: let 7 | depot = import inputs.depot { 8 | nixpkgsBisectPath = pkgs.path; 9 | localSystem = system; 10 | }; 11 | in { 12 | config.devshells.default = { 13 | env = [ 14 | { 15 | name = "TVIX_HOME"; 16 | eval = "$PRJ_DATA_DIR/tvix"; 17 | } 18 | { 19 | name = "BLOB_SERVICE_ADDR"; 20 | value = "grpc+http://localhost:5000"; 21 | } 22 | { 23 | name = "PATH_INFO_SERVICE_ADDR"; 24 | value = "grpc+http://localhost:5000"; 25 | } 26 | { 27 | name = "DIRECTORY_SERVICE_ADDR"; 28 | value = "grpc+http://localhost:5000"; 29 | } 30 | { 31 | name = "TVIX_MOUNT_DIR"; 32 | eval = "$PRJ_DATA_DIR/mount"; 33 | } 34 | ]; 35 | 36 | devshell.startup = { 37 | tvix-init.text = '' 38 | mkdir -p "$TVIX_HOME" 39 | mkdir -p "$TVIX_MOUNT_DIR" 40 | ''; 41 | }; 42 | 43 | commands = let 44 | category = "tvix"; 45 | in [ 46 | { 47 | inherit category; 48 | name = "tvix"; 49 | package = depot.tvix.cli; 50 | } 51 | { 52 | inherit category; 53 | name = "tvix-store"; 54 | package = depot.tvix.store; 55 | } 56 | ]; 57 | }; 58 | }; 59 | } 60 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "A NATS implementation of TVIX store"; 3 | 4 | nixConfig = { 5 | extra-substituters = [ 6 | "https://cache.garnix.io" 7 | "https://nix-community.cachix.org" 8 | ]; 9 | extra-trusted-public-keys = [ 10 | "cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g=" 11 | "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" 12 | ]; 13 | }; 14 | 15 | inputs = { 16 | nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; 17 | flake-parts.url = "github:hercules-ci/flake-parts"; 18 | flake-root.url = "github:srid/flake-root"; 19 | treefmt-nix = { 20 | url = "github:numtide/treefmt-nix"; 21 | inputs.nixpkgs.follows = "nixpkgs"; 22 | }; 23 | devshell = { 24 | url = "github:numtide/devshell"; 25 | inputs.nixpkgs.follows = "nixpkgs"; 26 | }; 27 | process-compose-flake.url = "github:Platonic-Systems/process-compose-flake"; 28 | depot = { 29 | url = "git+https://cl.tvl.fyi/depot"; 30 | flake = false; 31 | }; 32 | flake-compat = { 33 | url = "github:edolstra/flake-compat"; 34 | flake = false; 35 | }; 36 | gomod2nix.url = "github:nix-community/gomod2nix"; 37 | }; 38 | 39 | outputs = inputs @ {flake-parts, ...}: 40 | flake-parts.lib.mkFlake 41 | { 42 | inherit inputs; 43 | } { 44 | imports = [ 45 | ./nix 46 | ./nixos 47 | ]; 48 | systems = [ 49 | "x86_64-linux" 50 | "aarch64-linux" 51 | ]; 52 | }; 53 | } 54 | -------------------------------------------------------------------------------- /pkg/blob/helper_test.go: -------------------------------------------------------------------------------- 1 | package blob 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "testing" 8 | 9 | pb "code.tvl.fyi/tvix/castore/protos" 10 | 11 | "github.com/charmbracelet/log" 12 | ) 13 | 14 | func putBlob(c pb.BlobServiceClient, r io.Reader, chunkSize int, t *testing.T) (*pb.PutBlobResponse, error) { 15 | t.Helper() 16 | 17 | put, err := c.Put(context.Background()) 18 | if err != nil { 19 | t.Fatalf("failed to create put: %v", err) 20 | } 21 | 22 | chunk := make([]byte, chunkSize) 23 | 24 | for { 25 | n, err := r.Read(chunk) 26 | if err == io.EOF { 27 | break 28 | } else if err != nil { 29 | t.Fatalf("failed to read chunk: %v", err) 30 | } 31 | 32 | if err = put.Send(&pb.BlobChunk{ 33 | Data: chunk[:n], 34 | }); err != nil { 35 | t.Fatalf("failed to send blob chunk: %v", err) 36 | } 37 | } 38 | 39 | return put.CloseAndRecv() 40 | } 41 | 42 | func getBlob(c pb.BlobServiceClient, digest []byte, writer io.WriteCloser, t *testing.T) { 43 | get, err := c.Read(context.Background(), &pb.ReadBlobRequest{Digest: digest}) 44 | if err != nil { 45 | t.Fatalf("failed to open read request: %v", err) 46 | } 47 | 48 | for { 49 | chunk, err := get.Recv() 50 | if err == io.EOF { 51 | if err = writer.Close(); err != nil { 52 | log.Fatalf("failed to close writer: %v", err) 53 | } 54 | return 55 | } else if err != nil { 56 | t.Fatalf("failed to get next chunk: %v", err) 57 | } 58 | if _, err = io.Copy(writer, bytes.NewReader(chunk.Data)); err != nil { 59 | t.Fatalf("failed to write chunk into read buffer: %v", err) 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /nix/devshell.nix: -------------------------------------------------------------------------------- 1 | { 2 | inputs, 3 | lib, 4 | ... 5 | }: { 6 | imports = [ 7 | inputs.devshell.flakeModule 8 | inputs.process-compose-flake.flakeModule 9 | ]; 10 | 11 | config.perSystem = { 12 | pkgs, 13 | config, 14 | system, 15 | ... 16 | }: let 17 | inherit (pkgs.stdenv) isLinux isDarwin; 18 | in { 19 | config.devshells.default = { 20 | env = [ 21 | { 22 | name = "GOROOT"; 23 | value = pkgs.go + "/share/go"; 24 | } 25 | { 26 | name = "LD_LIBRARY_PATH"; 27 | value = "$DEVSHELL_DIR/lib"; 28 | } 29 | ]; 30 | 31 | packages = with lib; 32 | mkMerge [ 33 | [ 34 | # golang 35 | pkgs.go 36 | pkgs.gotools 37 | pkgs.pprof 38 | pkgs.rr 39 | pkgs.delve 40 | pkgs.golangci-lint 41 | pkgs.protobuf 42 | pkgs.protoc-gen-go 43 | 44 | pkgs.openssl 45 | 46 | pkgs.qemu-utils 47 | 48 | pkgs.statix 49 | ] 50 | # platform dependent CGO dependencies 51 | (mkIf isLinux [ 52 | pkgs.gcc 53 | ]) 54 | (mkIf isDarwin [ 55 | pkgs.darwin.cctools 56 | ]) 57 | ]; 58 | 59 | commands = [ 60 | { 61 | category = "development"; 62 | package = pkgs.evans; 63 | } 64 | { 65 | category = "development"; 66 | package = inputs.gomod2nix.packages.${system}.default; 67 | } 68 | ]; 69 | }; 70 | }; 71 | } 72 | -------------------------------------------------------------------------------- /internal/cli/store/init.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/brianmcgee/nvix/pkg/blob" 7 | "github.com/brianmcgee/nvix/pkg/cli" 8 | "github.com/brianmcgee/nvix/pkg/directory" 9 | "github.com/brianmcgee/nvix/pkg/pathinfo" 10 | "github.com/charmbracelet/log" 11 | ) 12 | 13 | type Init struct { 14 | Log cli.LogOptions `embed:""` 15 | Nats cli.NatsOptions `embed:""` 16 | } 17 | 18 | func (i *Init) Run() error { 19 | i.Log.ConfigureLogger() 20 | conn := i.Nats.Connect() 21 | 22 | ctx := context.Background() 23 | 24 | log.Info("initialising stores") 25 | 26 | log.Info("initialising blob chunk store") 27 | if err := blob.NewChunkStore(conn).Init(ctx); err != nil { 28 | log.Errorf("failed to initialise blob chunk store: %v", err) 29 | return err 30 | } 31 | 32 | log.Info("initialising blob meta store") 33 | if err := blob.NewMetaStore(conn).Init(ctx); err != nil { 34 | log.Errorf("failed to initialise blob meta store: %v", err) 35 | return err 36 | } 37 | 38 | log.Info("initialising directory store") 39 | if err := directory.NewDirectoryStore(conn).Init(ctx); err != nil { 40 | log.Errorf("failed to initialise directory store: %v", err) 41 | return err 42 | } 43 | 44 | log.Info("initialising path info store") 45 | if err := pathinfo.NewPathInfoStore(conn).Init(ctx); err != nil { 46 | log.Errorf("failed to initialise path info store: %v", err) 47 | return err 48 | } 49 | 50 | log.Info("initialising path info out idx store") 51 | if err := pathinfo.NewPathInfoOutIdxStore(conn).Init(ctx); err != nil { 52 | log.Errorf("failed to initialise path info out idx store: %v", err) 53 | return err 54 | } 55 | 56 | log.Info("initialising stores complete") 57 | 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /pkg/directory/config.go: -------------------------------------------------------------------------------- 1 | package directory 2 | 3 | import ( 4 | "github.com/brianmcgee/nvix/pkg/store" 5 | "github.com/brianmcgee/nvix/pkg/subject" 6 | "github.com/nats-io/nats.go" 7 | ) 8 | 9 | var ( 10 | DiskBasedStreamConfig = nats.StreamConfig{ 11 | Name: "directory_store", 12 | Subjects: []string{ 13 | subject.WithPrefix("STORE.DIRECTORY.*"), 14 | }, 15 | Replicas: 1, 16 | Discard: nats.DiscardOld, 17 | MaxMsgsPerSubject: 1, 18 | Storage: nats.FileStorage, 19 | AllowRollup: true, 20 | AllowDirect: true, 21 | Compression: nats.S2Compression, 22 | // automatically publish into the cache topic 23 | RePublish: &nats.RePublish{ 24 | Source: subject.WithPrefix("STORE.DIRECTORY.*"), 25 | Destination: subject.WithPrefix("CACHE.DIRECTORY.{{wildcard(1)}}"), 26 | }, 27 | } 28 | 29 | MemoryBasedStreamConfig = nats.StreamConfig{ 30 | Name: "directory_cache", 31 | Subjects: []string{ 32 | subject.WithPrefix("CACHE.DIRECTORY.*"), 33 | }, 34 | Replicas: 1, 35 | Discard: nats.DiscardOld, 36 | MaxBytes: 1024 * 1024 * 128, // todo make configurable from cli 37 | MaxMsgsPerSubject: 1, 38 | Storage: nats.MemoryStorage, 39 | AllowRollup: true, 40 | AllowDirect: true, 41 | } 42 | ) 43 | 44 | func NewDirectoryStore(conn *nats.Conn) store.Store { 45 | diskPrefix := DiskBasedStreamConfig.Subjects[0] 46 | diskPrefix = diskPrefix[:len(diskPrefix)-2] 47 | 48 | memoryPrefix := MemoryBasedStreamConfig.Subjects[0] 49 | memoryPrefix = memoryPrefix[:len(memoryPrefix)-2] 50 | 51 | disk := &store.NatsStore{ 52 | Conn: conn, 53 | StreamConfig: &DiskBasedStreamConfig, 54 | SubjectPrefix: diskPrefix, 55 | } 56 | 57 | memory := &store.NatsStore{ 58 | Conn: conn, 59 | StreamConfig: &MemoryBasedStreamConfig, 60 | SubjectPrefix: memoryPrefix, 61 | } 62 | 63 | return &store.CachingStore{ 64 | Disk: disk, 65 | Memory: memory, 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /nixos/store.nix: -------------------------------------------------------------------------------- 1 | { 2 | lib, 3 | pkgs, 4 | config, 5 | ... 6 | }: let 7 | cfg = config.services.nvix.store; 8 | in { 9 | options.services.nvix.store = with lib; { 10 | enable = mkEnableOption (mdDoc "Enable NVIX Store"); 11 | package = mkOption { 12 | type = types.package; 13 | default = pkgs.nvix; 14 | defaultText = literalExpression "pkgs.nvix"; 15 | description = mdDoc "Package to use for nits."; 16 | }; 17 | listen = { 18 | address = mkOption { 19 | type = types.str; 20 | default = "localhost:5000"; 21 | description = "interface and port to listen on"; 22 | }; 23 | }; 24 | metrics = { 25 | address = mkOption { 26 | type = types.str; 27 | default = "localhost:5050"; 28 | description = "interface and port to listen on"; 29 | }; 30 | }; 31 | nats = { 32 | url = mkOption { 33 | type = types.str; 34 | example = "nats://localhost:4222"; 35 | description = mdDoc "NATS server url."; 36 | }; 37 | credentialsFile = mkOption { 38 | type = types.path; 39 | example = "/mnt/shared/user.creds"; 40 | description = mdDoc "Path to a file containing a NATS credentials file"; 41 | }; 42 | }; 43 | verbosity = mkOption { 44 | type = types.int; 45 | default = 1; 46 | example = "2"; 47 | description = mdDoc "Selects the log verbosity."; 48 | }; 49 | }; 50 | 51 | config = lib.mkIf cfg.enable { 52 | systemd.services.nvix-store = { 53 | after = ["network.target"]; 54 | wantedBy = ["sysinit.target"]; 55 | 56 | description = "NVIX Store"; 57 | startLimitIntervalSec = 0; 58 | 59 | # the agent will restart itself after a successful deployment 60 | restartIfChanged = false; 61 | 62 | environment = lib.filterAttrs (_: v: v != null) { 63 | NATS_URL = cfg.nats.url; 64 | LISTEN_ADDRESS = cfg.listen.address; 65 | METRICS_ADDRESS = cfg.metrics.address; 66 | LOG_LEVEL = "${builtins.toString cfg.verbosity}"; 67 | }; 68 | 69 | serviceConfig = with lib; { 70 | Restart = mkDefault "on-failure"; 71 | RestartSec = 1; 72 | 73 | LoadCredential = [ 74 | "nats.creds:${cfg.nats.credentialsFile}" 75 | ]; 76 | 77 | User = "nvix-store"; 78 | DynamicUser = true; 79 | StateDirectory = "nvix-store"; 80 | ExecStart = "${cfg.package}/bin/nvix store run --nats-credentials %d/nats.creds"; 81 | }; 82 | }; 83 | }; 84 | } 85 | -------------------------------------------------------------------------------- /pkg/directory/directory.go: -------------------------------------------------------------------------------- 1 | package directory 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | 7 | capb "code.tvl.fyi/tvix/castore/protos" 8 | "github.com/juju/errors" 9 | ) 10 | 11 | const ( 12 | selfReference = "." 13 | parentReference = ".." 14 | 15 | ErrEmptyName = errors.ConstError("name cannot be an empty string") 16 | ErrNameWithSlash = errors.ConstError("name cannot contain slashes: '/'") 17 | ErrNameWithNullByte = errors.ConstError("name cannot contain null bytes") 18 | ErrNameWithSelfReference = errors.ConstError("name cannot be a self reference: '.'") 19 | ErrNameWithParentReference = errors.ConstError("name cannot be a parent reference: '..'") 20 | ErrNamesAreNotSorted = errors.ConstError("names must be lexicographically sorted") 21 | ) 22 | 23 | type DirEntry interface { 24 | GetName() []byte 25 | } 26 | 27 | func validateDirectory(directory *capb.Directory) error { 28 | cache := make(map[string]DirEntry) 29 | 30 | validate := func(name string, entry DirEntry) error { 31 | if err := validateName(name); err != nil { 32 | return err 33 | } else if _, ok := cache[name]; ok { 34 | return errors.Errorf("duplicate name: %v", name) 35 | } 36 | cache[name] = entry 37 | return nil 38 | } 39 | 40 | var lastName []byte 41 | for _, dir := range directory.Directories { 42 | if err := validate(string(dir.Name), dir); err != nil { 43 | return err 44 | } else if len(lastName) > 0 && bytes.Compare(lastName, dir.Name) > 0 { 45 | return ErrNamesAreNotSorted 46 | } 47 | lastName = dir.Name 48 | } 49 | 50 | lastName = nil 51 | for _, file := range directory.Files { 52 | if err := validate(string(file.Name), file); err != nil { 53 | return err 54 | } else if len(lastName) > 0 && bytes.Compare(lastName, file.Name) > 0 { 55 | return ErrNamesAreNotSorted 56 | } 57 | lastName = file.Name 58 | } 59 | 60 | lastName = nil 61 | for _, link := range directory.Symlinks { 62 | if err := validate(string(link.Name), link); err != nil { 63 | return err 64 | } else if len(lastName) > 0 && bytes.Compare(lastName, link.Name) > 0 { 65 | return ErrNamesAreNotSorted 66 | } 67 | lastName = link.Name 68 | } 69 | 70 | return nil 71 | } 72 | 73 | func validateName(name string) error { 74 | if name == "" { 75 | return ErrEmptyName 76 | } else if name == selfReference { 77 | return ErrNameWithSelfReference 78 | } else if name == parentReference { 79 | return ErrNameWithParentReference 80 | } else if strings.Contains(name, "/") { 81 | return ErrNameWithSlash 82 | } else if strings.IndexByte(name, '\x00') > -1 { 83 | return ErrNameWithNullByte 84 | } 85 | 86 | return nil 87 | } 88 | -------------------------------------------------------------------------------- /pkg/blob/config.go: -------------------------------------------------------------------------------- 1 | package blob 2 | 3 | import ( 4 | "github.com/brianmcgee/nvix/pkg/store" 5 | "github.com/brianmcgee/nvix/pkg/subject" 6 | "github.com/nats-io/nats.go" 7 | ) 8 | 9 | var ( 10 | DiskBasedStreamConfig = nats.StreamConfig{ 11 | Name: "blob_store", 12 | Subjects: []string{ 13 | subject.WithPrefix("STORE.BLOB.*"), 14 | subject.WithPrefix("STORE.CHUNK.*"), 15 | }, 16 | Replicas: 1, 17 | Discard: nats.DiscardOld, 18 | MaxMsgsPerSubject: 1, 19 | Storage: nats.FileStorage, 20 | AllowRollup: true, 21 | AllowDirect: true, 22 | Compression: nats.S2Compression, 23 | // automatically publish into the cache topic 24 | RePublish: &nats.RePublish{ 25 | Source: subject.WithPrefix("STORE.*.*"), 26 | Destination: subject.WithPrefix("CACHE.{{wildcard(1)}}.{{wildcard(2)}}"), 27 | }, 28 | } 29 | 30 | MemoryBasedStreamConfig = nats.StreamConfig{ 31 | Name: "blob_cache", 32 | Subjects: []string{ 33 | subject.WithPrefix("CACHE.BLOB.*"), 34 | subject.WithPrefix("CACHE.CHUNK.*"), 35 | }, 36 | Replicas: 1, 37 | Discard: nats.DiscardOld, 38 | MaxMsgsPerSubject: 1, 39 | MaxBytes: 1024 * 1024 * 128, // todo make configurable from cli 40 | Storage: nats.MemoryStorage, 41 | AllowRollup: true, 42 | AllowDirect: true, 43 | } 44 | ) 45 | 46 | func NewChunkStore(conn *nats.Conn) store.Store { 47 | diskPrefix := DiskBasedStreamConfig.Subjects[1] 48 | diskPrefix = diskPrefix[:len(diskPrefix)-2] 49 | 50 | memoryPrefix := MemoryBasedStreamConfig.Subjects[1] 51 | memoryPrefix = memoryPrefix[:len(memoryPrefix)-2] 52 | 53 | disk := &store.NatsStore{ 54 | Conn: conn, 55 | StreamConfig: &DiskBasedStreamConfig, 56 | SubjectPrefix: diskPrefix, 57 | } 58 | 59 | memory := &store.NatsStore{ 60 | Conn: conn, 61 | StreamConfig: &MemoryBasedStreamConfig, 62 | SubjectPrefix: memoryPrefix, 63 | } 64 | 65 | return &store.CachingStore{ 66 | Disk: disk, 67 | Memory: memory, 68 | } 69 | } 70 | 71 | func NewMetaStore(conn *nats.Conn) store.Store { 72 | diskPrefix := DiskBasedStreamConfig.Subjects[0] 73 | diskPrefix = diskPrefix[:len(diskPrefix)-2] 74 | 75 | memoryPrefix := MemoryBasedStreamConfig.Subjects[0] 76 | memoryPrefix = memoryPrefix[:len(memoryPrefix)-2] 77 | 78 | disk := &store.NatsStore{ 79 | Conn: conn, 80 | StreamConfig: &DiskBasedStreamConfig, 81 | SubjectPrefix: diskPrefix, 82 | } 83 | 84 | memory := &store.NatsStore{ 85 | Conn: conn, 86 | StreamConfig: &MemoryBasedStreamConfig, 87 | SubjectPrefix: memoryPrefix, 88 | } 89 | 90 | return &store.CachingStore{ 91 | Disk: disk, 92 | Memory: memory, 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /pkg/pathinfo/config.go: -------------------------------------------------------------------------------- 1 | package pathinfo 2 | 3 | import ( 4 | "github.com/brianmcgee/nvix/pkg/store" 5 | "github.com/brianmcgee/nvix/pkg/subject" 6 | "github.com/nats-io/nats.go" 7 | ) 8 | 9 | var ( 10 | DiskBasedStreamConfig = nats.StreamConfig{ 11 | Name: "path_info_store", 12 | Subjects: []string{ 13 | subject.WithPrefix("STORE.PATH_INFO.*"), 14 | subject.WithPrefix("STORE.PATH_INFO_OUT_IDX.*"), 15 | }, 16 | Replicas: 1, 17 | Discard: nats.DiscardOld, 18 | MaxMsgsPerSubject: 1, 19 | Storage: nats.FileStorage, 20 | AllowRollup: true, 21 | AllowDirect: true, 22 | Compression: nats.S2Compression, 23 | // automatically publish into the cache topic 24 | RePublish: &nats.RePublish{ 25 | Source: subject.WithPrefix("STORE.*.*"), 26 | Destination: subject.WithPrefix("CACHE.{{wildcard(1)}}.{{wildcard(2)}}"), 27 | }, 28 | } 29 | 30 | MemoryBasedStreamConfig = nats.StreamConfig{ 31 | Name: "path_info_cache", 32 | Subjects: []string{ 33 | subject.WithPrefix("CACHE.PATH_INFO.*"), 34 | subject.WithPrefix("CACHE.PATH_INFO_OUT_IDX.*"), 35 | }, 36 | Replicas: 1, 37 | Discard: nats.DiscardOld, 38 | MaxMsgsPerSubject: 1, 39 | MaxBytes: 1024 * 1024 * 128, // todo make configurable from cli 40 | Storage: nats.MemoryStorage, 41 | AllowRollup: true, 42 | AllowDirect: true, 43 | } 44 | ) 45 | 46 | func NewPathInfoStore(conn *nats.Conn) store.Store { 47 | diskPrefix := DiskBasedStreamConfig.Subjects[0] 48 | diskPrefix = diskPrefix[:len(diskPrefix)-2] 49 | 50 | memoryPrefix := MemoryBasedStreamConfig.Subjects[0] 51 | memoryPrefix = memoryPrefix[:len(memoryPrefix)-2] 52 | 53 | disk := &store.NatsStore{ 54 | Conn: conn, 55 | StreamConfig: &DiskBasedStreamConfig, 56 | SubjectPrefix: diskPrefix, 57 | } 58 | 59 | memory := &store.NatsStore{ 60 | Conn: conn, 61 | StreamConfig: &MemoryBasedStreamConfig, 62 | SubjectPrefix: memoryPrefix, 63 | } 64 | 65 | return &store.CachingStore{ 66 | Disk: disk, 67 | Memory: memory, 68 | } 69 | } 70 | 71 | func NewPathInfoOutIdxStore(conn *nats.Conn) store.Store { 72 | diskPrefix := DiskBasedStreamConfig.Subjects[1] 73 | diskPrefix = diskPrefix[:len(diskPrefix)-2] 74 | 75 | memoryPrefix := MemoryBasedStreamConfig.Subjects[1] 76 | memoryPrefix = memoryPrefix[:len(memoryPrefix)-2] 77 | 78 | disk := &store.NatsStore{ 79 | Conn: conn, 80 | StreamConfig: &DiskBasedStreamConfig, 81 | SubjectPrefix: diskPrefix, 82 | } 83 | 84 | memory := &store.NatsStore{ 85 | Conn: conn, 86 | StreamConfig: &MemoryBasedStreamConfig, 87 | SubjectPrefix: memoryPrefix, 88 | } 89 | 90 | return &store.CachingStore{ 91 | Disk: disk, 92 | Memory: memory, 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/brianmcgee/nvix 2 | 3 | go 1.20 4 | 5 | require ( 6 | code.tvl.fyi/tvix/castore/protos v0.0.0-20231014122118-3fc2ade7dfb2 7 | code.tvl.fyi/tvix/store/protos v0.0.0-20231014142132-b2dfae6a1028 8 | github.com/SaveTheRbtz/fastcdc-go v0.3.0 9 | github.com/alecthomas/kong v0.8.1 10 | github.com/charmbracelet/log v0.2.5 11 | github.com/golang/protobuf v1.5.3 12 | github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 13 | github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 14 | github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf 15 | github.com/juju/errors v1.0.0 16 | github.com/multiformats/go-multihash v0.2.3 17 | github.com/nats-io/nats-server/v2 v2.10.2 18 | github.com/nats-io/nats.go v1.31.0 19 | github.com/nats-io/nuid v1.0.1 20 | github.com/nix-community/go-nix v0.0.0-20231012070617-9b176785e54d 21 | github.com/prometheus/client_golang v1.17.0 22 | github.com/stretchr/testify v1.8.4 23 | github.com/ztrue/shutdown v0.1.1 24 | golang.org/x/sync v0.4.0 25 | google.golang.org/grpc v1.58.3 26 | google.golang.org/protobuf v1.31.0 27 | lukechampine.com/blake3 v1.2.1 28 | ) 29 | 30 | require ( 31 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect 32 | github.com/beorn7/perks v1.0.1 // indirect 33 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 34 | github.com/charmbracelet/lipgloss v0.9.1 // indirect 35 | github.com/davecgh/go-spew v1.1.1 // indirect 36 | github.com/go-logfmt/logfmt v0.6.0 // indirect 37 | github.com/klauspost/compress v1.17.1 // indirect 38 | github.com/klauspost/cpuid/v2 v2.2.5 // indirect 39 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 40 | github.com/mattn/go-isatty v0.0.19 // indirect 41 | github.com/mattn/go-runewidth v0.0.15 // indirect 42 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 43 | github.com/minio/highwayhash v1.0.2 // indirect 44 | github.com/minio/sha256-simd v1.0.1 // indirect 45 | github.com/mr-tron/base58 v1.2.0 // indirect 46 | github.com/muesli/reflow v0.3.0 // indirect 47 | github.com/muesli/termenv v0.15.2 // indirect 48 | github.com/multiformats/go-varint v0.0.7 // indirect 49 | github.com/nats-io/jwt/v2 v2.5.2 // indirect 50 | github.com/nats-io/nkeys v0.4.5 // indirect 51 | github.com/pmezard/go-difflib v1.0.0 // indirect 52 | github.com/prometheus/client_model v0.5.0 // indirect 53 | github.com/prometheus/common v0.44.0 // indirect 54 | github.com/prometheus/procfs v0.12.0 // indirect 55 | github.com/rivo/uniseg v0.4.4 // indirect 56 | github.com/spaolacci/murmur3 v1.1.0 // indirect 57 | golang.org/x/crypto v0.14.0 // indirect 58 | golang.org/x/net v0.17.0 // indirect 59 | golang.org/x/sys v0.13.0 // indirect 60 | golang.org/x/text v0.13.0 // indirect 61 | golang.org/x/time v0.3.0 // indirect 62 | google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a // indirect 63 | gopkg.in/yaml.v3 v3.0.1 // indirect 64 | ) 65 | 66 | //replace code.tvl.fyi/tvix/store/protos => /home/brian/Development/tvl.fyi/depot/tvix/store/protos 67 | -------------------------------------------------------------------------------- /pkg/test/nats.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "os" 5 | "time" 6 | 7 | "github.com/charmbracelet/log" 8 | "github.com/nats-io/nats-server/v2/server" 9 | "github.com/nats-io/nats-server/v2/test" 10 | "github.com/nats-io/nats.go" 11 | ) 12 | 13 | const ( 14 | _EMPTY_ = "" 15 | ) 16 | 17 | type TestingT interface { 18 | Helper() 19 | Fatal(args ...any) 20 | Fatalf(msg string, args ...any) 21 | Errorf(msg string, args ...any) 22 | TempDir() string 23 | } 24 | 25 | type LogAdapter struct{} 26 | 27 | func (l *LogAdapter) Noticef(format string, v ...interface{}) { 28 | log.Infof(format, v...) 29 | } 30 | 31 | func (l *LogAdapter) Warnf(format string, v ...interface{}) { 32 | log.Warnf(format, v...) 33 | } 34 | 35 | func (l *LogAdapter) Fatalf(format string, v ...interface{}) { 36 | log.Fatalf(format, v...) 37 | } 38 | 39 | func (l *LogAdapter) Errorf(format string, v ...interface{}) { 40 | log.Errorf(format, v...) 41 | } 42 | 43 | func (l *LogAdapter) Debugf(format string, v ...interface{}) { 44 | log.Debugf(format, v...) 45 | } 46 | 47 | func (l *LogAdapter) Tracef(format string, v ...interface{}) { 48 | log.Debugf(format, v...) 49 | } 50 | 51 | func RunBasicServer(t TestingT) *server.Server { 52 | t.Helper() 53 | opts := test.DefaultTestOptions 54 | opts.Port = -1 55 | opts.JetStream = false 56 | opts.Debug = false 57 | opts.MaxPayload = 8 * 1024 * 1024 58 | srv := test.RunServer(&opts) 59 | srv.SetLoggerV2(&LogAdapter{}, opts.Debug, opts.Trace, false) 60 | return srv 61 | } 62 | 63 | func RunBasicJetStreamServer(t TestingT) *server.Server { 64 | t.Helper() 65 | opts := test.DefaultTestOptions 66 | opts.Port = -1 67 | opts.JetStream = true 68 | opts.StoreDir = t.TempDir() 69 | opts.Debug = false 70 | opts.MaxPayload = 8 * 1024 * 1024 71 | srv := test.RunServer(&opts) 72 | srv.SetLoggerV2(&LogAdapter{}, opts.Debug, opts.Trace, false) 73 | return srv 74 | } 75 | 76 | func NatsConn(t TestingT, s *server.Server, opts ...nats.Option) *nats.Conn { 77 | t.Helper() 78 | nc, err := nats.Connect(s.ClientURL(), opts...) 79 | if err != nil { 80 | t.Fatalf("Unexpected error creating Nats connection: %v", err) 81 | } 82 | return nc 83 | } 84 | 85 | func JsClient(t TestingT, s *server.Server, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) { 86 | t.Helper() 87 | nc := NatsConn(t, s, opts...) 88 | js, err := nc.JetStream(nats.MaxWait(10 * time.Second)) 89 | if err != nil { 90 | t.Fatalf("Unexpected error getting JetStream context: %v", err) 91 | } 92 | return nc, js 93 | } 94 | 95 | func ShutdownServer(t TestingT, s *server.Server) { 96 | t.Helper() 97 | s.Shutdown() 98 | s.WaitForShutdown() 99 | } 100 | 101 | func ShutdownJSServerAndRemoveStorage(t TestingT, s *server.Server) { 102 | t.Helper() 103 | var sd string 104 | if config := s.JetStreamConfig(); config != nil { 105 | sd = config.StoreDir 106 | } 107 | s.Shutdown() 108 | if sd != _EMPTY_ { 109 | if err := os.RemoveAll(sd); err != nil { 110 | t.Fatalf("Unable to remove storage %q: %v", sd, err) 111 | } 112 | } 113 | s.WaitForShutdown() 114 | } 115 | -------------------------------------------------------------------------------- /pkg/store/cache.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | 8 | "github.com/brianmcgee/nvix/pkg/util" 9 | 10 | "github.com/nats-io/nats.go" 11 | 12 | "github.com/charmbracelet/log" 13 | ) 14 | 15 | type CachingStore struct { 16 | Disk Store 17 | Memory Store 18 | } 19 | 20 | func (c *CachingStore) Init(ctx context.Context) error { 21 | if err := c.Disk.Init(ctx); err != nil { 22 | return err 23 | } 24 | return c.Memory.Init(ctx) 25 | } 26 | 27 | func (c *CachingStore) List(ctx context.Context) (util.Iterator[io.ReadCloser], error) { 28 | return c.Disk.List(ctx) 29 | } 30 | 31 | func (c *CachingStore) Stat(key string, ctx context.Context) (ok bool, err error) { 32 | ok, err = c.Memory.Stat(key, ctx) 33 | if err == nil { 34 | return 35 | } else if !ok { 36 | // check disk 37 | ok, err = c.Disk.Stat(key, ctx) 38 | } 39 | return 40 | } 41 | 42 | func (c *CachingStore) Get(key string, ctx context.Context) (reader io.ReadCloser, err error) { 43 | // try in Memory store first 44 | reader, err = c.Memory.Get(key, ctx) 45 | if err == nil { 46 | reader = &cacheReader{ 47 | key: key, 48 | disk: c.Disk, 49 | memory: c.Memory, 50 | ctx: ctx, 51 | reader: reader, 52 | } 53 | } 54 | return 55 | } 56 | 57 | func (c *CachingStore) Put(key string, reader io.ReadCloser, ctx context.Context) error { 58 | return c.Disk.Put(key, reader, ctx) 59 | } 60 | 61 | func (c *CachingStore) PutAsync(key string, reader io.ReadCloser, ctx context.Context) (nats.PubAckFuture, error) { 62 | return c.Disk.PutAsync(key, reader, ctx) 63 | } 64 | 65 | func (c *CachingStore) Delete(key string, ctx context.Context) error { 66 | if err := c.Memory.Delete(key, ctx); err != nil { 67 | return err 68 | } 69 | return c.Disk.Delete(key, ctx) 70 | } 71 | 72 | type cacheWriter struct { 73 | store Store 74 | key string 75 | reader io.Reader 76 | buf *bytes.Buffer 77 | } 78 | 79 | func (c cacheWriter) Read(p []byte) (n int, err error) { 80 | if n, err = c.reader.Read(p); err != nil { 81 | return 82 | } 83 | _, err = c.buf.Write(p[:n]) 84 | return 85 | } 86 | 87 | func (c cacheWriter) Close() (err error) { 88 | log.Debug("populating cache", "key", c.key) 89 | if err = c.store.Put(c.key, io.NopCloser(bytes.NewReader(c.buf.Bytes())), context.Background()); err != nil { 90 | log.Error("failed to populate cache", "key", c.key, "error", err) 91 | } 92 | return 93 | } 94 | 95 | type cacheReader struct { 96 | key string 97 | disk Store 98 | memory Store 99 | ctx context.Context 100 | 101 | reader io.ReadCloser 102 | faulted bool 103 | } 104 | 105 | func (c *cacheReader) Read(p []byte) (n int, err error) { 106 | n, err = c.reader.Read(p) 107 | if err == ErrKeyNotFound && !c.faulted { 108 | _ = c.reader.Close() 109 | 110 | log.Debug("cache miss", "key", c.key) 111 | // fallback and try the Disk based store last 112 | c.reader, err = c.disk.Get(c.key, c.ctx) 113 | if err == nil { 114 | c.reader = cacheWriter{ 115 | store: c.memory, 116 | key: c.key, 117 | reader: c.reader, 118 | buf: bytes.NewBuffer(nil), 119 | } 120 | } 121 | 122 | c.faulted = true 123 | n, err = c.reader.Read(p) 124 | } 125 | return 126 | } 127 | 128 | func (c *cacheReader) Close() error { 129 | return c.reader.Close() 130 | } 131 | -------------------------------------------------------------------------------- /pkg/blob/grpc.go: -------------------------------------------------------------------------------- 1 | package blob 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | capb "code.tvl.fyi/tvix/castore/protos" 8 | 9 | "github.com/brianmcgee/nvix/pkg/store" 10 | "github.com/charmbracelet/log" 11 | "github.com/nats-io/nats.go" 12 | "google.golang.org/grpc/codes" 13 | "google.golang.org/grpc/status" 14 | ) 15 | 16 | func NewServer(conn *nats.Conn) (*Server, error) { 17 | return &Server{ 18 | conn: conn, 19 | store: &store.CdcStore{ 20 | Meta: NewMetaStore(conn), 21 | Chunks: NewChunkStore(conn), 22 | }, 23 | }, nil 24 | } 25 | 26 | type Server struct { 27 | capb.UnimplementedBlobServiceServer 28 | conn *nats.Conn 29 | 30 | store *store.CdcStore 31 | } 32 | 33 | func (s *Server) Stat(ctx context.Context, request *capb.StatBlobRequest) (*capb.BlobMeta, error) { 34 | l := log.WithPrefix("blob.stat") 35 | l.Debug("executing", "digest", store.Digest(request.GetDigest())) 36 | 37 | digest := store.Digest(request.Digest) 38 | ok, err := s.store.Stat(digest, ctx) 39 | if err != nil { 40 | return nil, err 41 | } 42 | if !ok { 43 | return nil, status.Errorf(codes.NotFound, "blob not found: %v", digest) 44 | } 45 | // castore blob meta is empty for now 46 | return &capb.BlobMeta{}, nil 47 | } 48 | 49 | func (s *Server) Read(request *capb.ReadBlobRequest, server capb.BlobService_ReadServer) error { 50 | l := log.WithPrefix("blob.read") 51 | 52 | ctx, cancel := context.WithCancel(server.Context()) 53 | defer cancel() 54 | 55 | digest := store.Digest(request.Digest) 56 | reader, err := s.store.Get(digest, ctx) 57 | 58 | if err == store.ErrKeyNotFound { 59 | return status.Errorf(codes.NotFound, "blob not found: %v", digest) 60 | } else if err != nil { 61 | l.Error("failed to get blob", "digest", digest, "error", err) 62 | return status.Error(codes.Internal, "internal error") 63 | } 64 | 65 | // we want to stay just under the 4MB max size restriction in gRPC 66 | sendBuf := make([]byte, (4*1024*1024)-5) 67 | 68 | for { 69 | n, err := reader.Read(sendBuf) 70 | if err == io.EOF { 71 | _ = reader.Close() 72 | break 73 | } else if err != nil { 74 | l.Errorf("failed to read next chunk: %v", err) 75 | return status.Error(codes.Internal, "internal error") 76 | } 77 | 78 | if err = server.Send(&capb.BlobChunk{ 79 | Data: sendBuf[:n], 80 | }); err != nil { 81 | l.Errorf("failed to send blob chunk to client: %v", err) 82 | return err 83 | } 84 | } 85 | 86 | return nil 87 | } 88 | 89 | func (s *Server) GetByDigest(digest store.Digest, ctx context.Context) (io.ReadCloser, error) { 90 | return s.store.Get(digest, ctx) 91 | } 92 | 93 | func (s *Server) Put(server capb.BlobService_PutServer) (err error) { 94 | l := log.WithPrefix("blob.put") 95 | 96 | ctx, cancel := context.WithCancel(server.Context()) 97 | defer cancel() 98 | 99 | reader := blobReader{server: server} 100 | 101 | digest, err := s.store.Put(&reader, ctx) 102 | if err != nil { 103 | l.Error("failed to put blob", "error", err) 104 | return status.Error(codes.Internal, "internal error") 105 | } 106 | 107 | return server.SendAndClose(&capb.PutBlobResponse{ 108 | Digest: digest[:], 109 | }) 110 | } 111 | 112 | type blobReader struct { 113 | server capb.BlobService_PutServer 114 | chunk []byte 115 | } 116 | 117 | func (b *blobReader) Read(p []byte) (n int, err error) { 118 | if b.chunk == nil { 119 | var chunk *capb.BlobChunk 120 | if chunk, err = b.server.Recv(); err != nil { 121 | return 122 | } 123 | b.chunk = chunk.Data 124 | } 125 | 126 | n = copy(p, b.chunk) 127 | if n == len(b.chunk) { 128 | b.chunk = nil 129 | } else { 130 | b.chunk = b.chunk[n:] 131 | } 132 | 133 | return 134 | } 135 | 136 | func (b *blobReader) Close() error { 137 | return nil 138 | } 139 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NVIX - TVIX services backed by NATS 2 | 3 | [![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/brianmcgee/nvix) 4 | ![Build](https://github.com/brianmcgee/nvix/actions/workflows/ci.yaml/badge.svg) 5 | [![Coverage Status](https://coveralls.io/repos/github/brianmcgee/nvix/badge.svg)](https://coveralls.io/github/brianmcgee/nvix) 6 | 7 | Status: _ACTIVE_ 8 | 9 | This project came into this world as part of the Hackday at [NixCon 2023](https://2023.nixcon.org/). 10 | 11 | It's primary focus is implementing [TVIX](https://cs.tvl.fyi/depot/-/tree/tvix) services using [NATS](https://nats.io). 12 | 13 | ## Roadmap 14 | 15 | - [x] Store 16 | - [x] Blob Service 17 | - [x] Path Info Service 18 | - [x] Directory Service 19 | - [ ] Improve test coverage 20 | - [ ] Add more metrics 21 | 22 | ## Requirements 23 | 24 | You must be familiar with and have the following installed: 25 | 26 | - [Direnv](https://direnv.net) 27 | - [Nix](https://nixos.org) 28 | 29 | ## Quick Start 30 | 31 | After cloning the repository cd into the root directory and run: 32 | 33 | ```terminal 34 | direnv allow 35 | ``` 36 | 37 | You will be asked to accept config for some additional substituters to which you should select yes. 38 | 39 | Once the dev shell has finished initialising you will then be met with the following: 40 | 41 | ```terminal 42 | 🔨 Welcome to devshell 43 | 44 | [checks] 45 | 46 | check - Run all linters and build all packages 47 | fix - Remove unused nix code 48 | fmt - Format the repo 49 | 50 | [development] 51 | 52 | dev - Run local dev services 53 | dev-init - Re-initialise state for dev services 54 | enumer - Go tool to auto generate methods for enums 55 | evans - More expressive universal gRPC client 56 | 57 | [docs] 58 | 59 | gifs - generate all gifs used in docs 60 | vhs - A tool for generating terminal GIFs with code 61 | 62 | [general commands] 63 | 64 | menu - prints this menu 65 | 66 | [nats] 67 | 68 | nats - NATS Server and JetStream administration 69 | nsc - Creates NATS operators, accounts, users, and manage their permissions 70 | 71 | [tvix] 72 | 73 | tvix 74 | tvix-store 75 | 76 | direnv: export +BLOB_SERVICE_ADDR +DEVSHELL_DIR +DIRECTORY_SERVICE_ADDR +GOROOT +IN_NIX_SHELL +NATS_HOME +NATS_JWT_DIR +NIXPKGS_PATH +NKEYS_PATH +NSC_HOME +PATH_INFO_SERVICE_ADDR +PRJ_DATA_DIR +PRJ_ROOT +TVIX_HOME +TVIX_MOUNT_DIR +name -NIX_BUILD_CORES -NIX_BUILD_TOP -NIX_STORE -TEMP -TEMPDIR -TMP -TMPDIR -builder -out -outputs -stdenv -system ~LD_LIBRARY_PATH ~PATH ~XDG_DATA_DIRS 77 | ``` 78 | 79 | To start the local dev services type `dev`. 80 | 81 | ![](./docs/assets/dev.gif) 82 | 83 | You must wait until `nsc-push` has completed successfully and `nats-server` and `nvix-store` are in the running state. 84 | 85 | With the dev services up and running you can now import paths and explore the TVIX store with the following: 86 | 87 | ```terminal 88 | ❯ tvix-store import pkg # imports the pkg source directory 89 | 2023-09-12T09:16:07.507131Z INFO tvix_store: import successful, path: "pkg", name: b"hr72zqz792b8s20aqm02lkmzidnxli77-pkg", digest: "sYpXtNHyUKRRRu4nnl0/o9bx2qk5DA0HNQgxsB4Hb4o=" 90 | at src/bin/tvix-store.rs:308 91 | 92 | ❯ tvix-store mount -l $TVIX_MOUNT_DIR # mounts the store into a directory within .data 93 | 2023-09-12T09:16:18.571878Z INFO tvix_store: mounting tvix-store on "/home/brian/Development/github.com/brianmcgee/nvix/.data/mount" 94 | at src/bin/tvix-store.rs:294 95 | ``` 96 | 97 | In a separate terminal you can explore the contents of the TVIX store: 98 | 99 | ```terminal 100 | ❯ tree $TVIX_MOUNT_DIR 101 | /home/brian/Development/github.com/brianmcgee/nvix/.data/mount 102 | └── hr72zqz792b8s20aqm02lkmzidnxli77-pkg 103 | └── store 104 | ├── blob 105 | │   ├── blob.go 106 | │   ├── blob_test.go 107 | │   └── helper_test.go 108 | └── subject 109 | └── blob.go 110 | ``` 111 | 112 | ## License 113 | 114 | This software is provided free under the [MIT Licence](https://opensource.org/licenses/MIT). 115 | 116 | ## Contact 117 | 118 | There are a few different ways to reach me, all of which are listed on my [website](https://bmcgee.ie/). 119 | -------------------------------------------------------------------------------- /nix/dev/nats.nix: -------------------------------------------------------------------------------- 1 | {lib, ...}: { 2 | perSystem = {pkgs, ...}: let 3 | config = pkgs.writeTextFile { 4 | name = "nats.conf"; 5 | text = '' 6 | ## Default NATS server configuration (see: https://docs.nats.io/running-a-nats-service/configuration) 7 | 8 | ## Host for client connections. 9 | host: "127.0.0.1" 10 | 11 | ## Port for client connections. 12 | port: 4222 13 | 14 | ## Port for monitoring 15 | http_port: 8222 16 | 17 | ## Increase max payload to 8MB 18 | max_payload: 8388608 19 | 20 | ## Configuration map for JetStream. 21 | ## see: https://docs.nats.io/running-a-nats-service/configuration#jetstream 22 | jetstream {} 23 | 24 | # include paths must be relative so for simplicity we just read in the auth.conf file 25 | include './auth.conf' 26 | ''; 27 | }; 28 | 29 | # we need to wrap nsc and nats to ensure they pick up key related stated from the data directory 30 | nscWrapped = pkgs.writeShellScriptBin "nsc" '' 31 | XDG_CONFIG_HOME="$PRJ_DATA_DIR" ${pkgs.nsc}/bin/nsc -H $NSC_HOME "$@" 32 | ''; 33 | 34 | natsWrapped = pkgs.writeShellScriptBin "nats" '' 35 | XDG_CONFIG_HOME="$PRJ_DATA_DIR" ${pkgs.natscli}/bin/nats "$@" 36 | ''; 37 | in { 38 | config.process-compose = { 39 | dev.settings.processes = { 40 | nats-server = { 41 | working_dir = "$NATS_HOME"; 42 | command = ''${lib.getExe pkgs.nats-server} -c ./nats.conf -sd ./''; 43 | readiness_probe = { 44 | http_get = { 45 | host = "127.0.0.1"; 46 | port = 8222; 47 | path = "/healthz"; 48 | }; 49 | initial_delay_seconds = 2; 50 | }; 51 | }; 52 | 53 | nsc-push = { 54 | depends_on = { 55 | nats-server.condition = "process_healthy"; 56 | }; 57 | environment = { 58 | XDG_CONFIG_HOME = "$PRJ_DATA_DIR"; 59 | }; 60 | command = pkgs.writeShellApplication { 61 | name = "nsc-push"; 62 | runtimeInputs = [nscWrapped]; 63 | text = ''nsc push''; 64 | }; 65 | }; 66 | }; 67 | }; 68 | 69 | config.devshells.default = { 70 | env = [ 71 | { 72 | name = "NATS_HOME"; 73 | eval = "$PRJ_DATA_DIR/nats"; 74 | } 75 | { 76 | name = "NSC_HOME"; 77 | eval = "$PRJ_DATA_DIR/nsc"; 78 | } 79 | { 80 | name = "NKEYS_PATH"; 81 | eval = "$NSC_HOME"; 82 | } 83 | { 84 | name = "NATS_JWT_DIR"; 85 | eval = "$PRJ_DATA_DIR/nats/jwt"; 86 | } 87 | ]; 88 | 89 | devshell.startup = { 90 | setup-nats.text = '' 91 | set -euo pipefail 92 | 93 | # we only setup the data dir if it doesn't exist 94 | # to refresh simply delete the directory and run `direnv reload` 95 | [ -d $NSC_HOME ] && exit 0 96 | 97 | mkdir -p $NSC_HOME 98 | 99 | # initialise nsc state 100 | 101 | nsc init -n Tvix --dir $NSC_HOME 102 | nsc edit operator \ 103 | --service-url nats://localhost:4222 \ 104 | --account-jwt-server-url nats://localhost:4222 105 | 106 | # setup server config 107 | 108 | mkdir -p "$NATS_HOME" 109 | 110 | # disable cow if underlying filesystem is btrfs 111 | chattr +C "$NATS_HOME" 112 | 113 | cp ${config} "$NATS_HOME/nats.conf" 114 | nsc generate config --nats-resolver --config-file "$NATS_HOME/auth.conf" 115 | 116 | nsc add account -n Store 117 | nsc edit account -n Store \ 118 | --js-mem-storage -1 \ 119 | --js-disk-storage -1 \ 120 | --js-streams -1 \ 121 | --js-consumer -1 122 | 123 | nsc add user -a Store -n Admin 124 | nsc add user -a Store -n Server 125 | ''; 126 | }; 127 | 128 | packages = [ 129 | pkgs.nkeys 130 | pkgs.nats-top 131 | ]; 132 | 133 | commands = let 134 | category = "nats"; 135 | in [ 136 | { 137 | inherit category; 138 | help = "Creates NATS operators, accounts, users, and manage their permissions"; 139 | package = nscWrapped; 140 | } 141 | { 142 | inherit category; 143 | help = "NATS Server and JetStream administration"; 144 | package = natsWrapped; 145 | } 146 | ]; 147 | }; 148 | }; 149 | } 150 | -------------------------------------------------------------------------------- /pkg/store/nats_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "math/rand" 9 | "testing" 10 | 11 | "github.com/nats-io/nuid" 12 | 13 | "github.com/brianmcgee/nvix/pkg/test" 14 | "github.com/inhies/go-bytesize" 15 | "github.com/nats-io/nats.go" 16 | ) 17 | 18 | var natsStoreSizes = []bytesize.ByteSize{ 19 | 1 << 10, 20 | 4 << 10, 21 | 16 << 10, 22 | 32 << 10, 23 | 64 << 10, 24 | 128 << 10, 25 | 256 << 10, 26 | 512 << 10, 27 | 1 << 20, 28 | 4 << 20, 29 | (8 << 20) - 1024, // stay just under max msg size 30 | } 31 | 32 | func BenchmarkNatsStore_Put(b *testing.B) { 33 | s := test.RunBasicJetStreamServer(b) 34 | defer test.ShutdownJSServerAndRemoveStorage(b, s) 35 | 36 | conn, js := test.JsClient(b, s) 37 | 38 | js, err := conn.JetStream() 39 | if err != nil { 40 | b.Fatal(err) 41 | } 42 | 43 | storageTypes := []nats.StorageType{ 44 | nats.FileStorage, 45 | nats.MemoryStorage, 46 | } 47 | 48 | streamConfig := nats.StreamConfig{ 49 | Replicas: 1, 50 | Discard: nats.DiscardOld, 51 | MaxMsgsPerSubject: 1, 52 | Storage: nats.FileStorage, 53 | AllowRollup: true, 54 | AllowDirect: true, 55 | } 56 | 57 | for _, storage := range storageTypes { 58 | 59 | subjectPrefix := fmt.Sprintf("STORE.%v", storage) 60 | 61 | streamConfig.Name = storage.String() 62 | streamConfig.Subjects = []string{subjectPrefix + ".*"} 63 | streamConfig.Storage = storage 64 | 65 | if _, err := js.AddStream(&streamConfig); err != nil { 66 | b.Fatal(err) 67 | } 68 | 69 | store := NatsStore{ 70 | Conn: conn, 71 | StreamConfig: &streamConfig, 72 | SubjectPrefix: subjectPrefix, 73 | } 74 | 75 | for _, size := range natsStoreSizes { 76 | size := size 77 | b.Run(fmt.Sprintf("%s-%v", streamConfig.Name, size), func(b *testing.B) { 78 | b.SetBytes(int64(size)) 79 | b.ReportAllocs() 80 | b.ResetTimer() 81 | 82 | b.RunParallel(func(pb *testing.PB) { 83 | rng := rand.New(rand.NewSource(1)) 84 | data := make([]byte, size) 85 | rng.Read(data) 86 | 87 | r := bytes.NewReader(data) 88 | 89 | for pb.Next() { 90 | r.Reset(data) 91 | if err := store.Put(nuid.Next(), io.NopCloser(r), context.Background()); err != nil { 92 | b.Fatal(err) 93 | } 94 | } 95 | }) 96 | }) 97 | } 98 | } 99 | } 100 | 101 | func BenchmarkNatsStore_Get(b *testing.B) { 102 | s := test.RunBasicJetStreamServer(b) 103 | defer test.ShutdownJSServerAndRemoveStorage(b, s) 104 | 105 | conn, js := test.JsClient(b, s) 106 | 107 | js, err := conn.JetStream() 108 | if err != nil { 109 | b.Fatal(err) 110 | } 111 | 112 | storageTypes := []nats.StorageType{ 113 | nats.FileStorage, 114 | nats.MemoryStorage, 115 | } 116 | 117 | streamConfig := nats.StreamConfig{ 118 | Replicas: 1, 119 | Discard: nats.DiscardOld, 120 | MaxMsgsPerSubject: 1, 121 | Storage: nats.FileStorage, 122 | AllowRollup: true, 123 | AllowDirect: true, 124 | } 125 | 126 | for _, storage := range storageTypes { 127 | 128 | subjectPrefix := fmt.Sprintf("STORE.%v", storage) 129 | 130 | streamConfig.Name = storage.String() 131 | streamConfig.Subjects = []string{subjectPrefix + ".*"} 132 | streamConfig.Storage = storage 133 | 134 | if _, err := js.AddStream(&streamConfig); err != nil { 135 | b.Fatal(err) 136 | } 137 | 138 | store := NatsStore{ 139 | Conn: conn, 140 | StreamConfig: &streamConfig, 141 | SubjectPrefix: subjectPrefix, 142 | } 143 | 144 | for _, size := range natsStoreSizes { 145 | size := size 146 | 147 | rng := rand.New(rand.NewSource(1)) 148 | data := make([]byte, size) 149 | rng.Read(data) 150 | 151 | r := bytes.NewReader(data) 152 | 153 | key := fmt.Sprintf("key-%d", int(size)) 154 | if err := store.Put(key, io.NopCloser(r), context.Background()); err != nil { 155 | b.Fatal(err) 156 | } 157 | 158 | b.Run(fmt.Sprintf("%s-%v", streamConfig.Name, size), func(b *testing.B) { 159 | b.SetBytes(int64(size)) 160 | b.ReportAllocs() 161 | b.ResetTimer() 162 | 163 | b.RunParallel(func(pb *testing.PB) { 164 | for pb.Next() { 165 | reader, err := store.Get(key, context.Background()) 166 | if err != nil { 167 | b.Fatal(err) 168 | } 169 | 170 | getData, err := io.ReadAll(reader) 171 | if err != nil { 172 | b.Fatal(err) 173 | } 174 | 175 | if len(getData) != len(data) { 176 | b.Fatalf("expected %d bytes, received %b", len(data), len(getData)) 177 | } 178 | } 179 | }) 180 | }) 181 | } 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /internal/cli/store/run.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "net" 5 | "net/http" 6 | "runtime/debug" 7 | "syscall" 8 | 9 | "google.golang.org/grpc/reflection" 10 | 11 | "github.com/brianmcgee/nvix/pkg/cli" 12 | 13 | tvpb "code.tvl.fyi/tvix/store/protos" 14 | "github.com/brianmcgee/nvix/pkg/pathinfo" 15 | 16 | "github.com/brianmcgee/nvix/pkg/directory" 17 | 18 | "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery" 19 | "github.com/prometheus/client_golang/prometheus" 20 | "github.com/prometheus/client_golang/prometheus/promauto" 21 | "github.com/prometheus/client_golang/prometheus/promhttp" 22 | "google.golang.org/grpc/codes" 23 | "google.golang.org/grpc/status" 24 | 25 | "github.com/brianmcgee/nvix/pkg/blob" 26 | 27 | pb "code.tvl.fyi/tvix/castore/protos" 28 | grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" 29 | 30 | "github.com/charmbracelet/log" 31 | "github.com/ztrue/shutdown" 32 | "golang.org/x/sync/errgroup" 33 | "google.golang.org/grpc" 34 | ) 35 | 36 | type Run struct { 37 | Log cli.LogOptions `embed:""` 38 | Nats cli.NatsOptions `embed:""` 39 | 40 | ListenAddr string `short:"l" env:"LISTEN_ADDR" default:"localhost:5000"` 41 | MetricsAddr string `short:"m" env:"METRICS_ADDR" default:"localhost:5050"` 42 | } 43 | 44 | func (r *Run) Run() error { 45 | r.Log.ConfigureLogger() 46 | 47 | conn := r.Nats.Connect() 48 | 49 | blobServer, err := blob.NewServer(conn) 50 | if err != nil { 51 | log.Fatalf("failed to create blob service: %v", err) 52 | } 53 | 54 | directoryServer, err := directory.NewServer(conn) 55 | if err != nil { 56 | log.Fatalf("failed to create directory service: %v", err) 57 | } 58 | 59 | pathInfoServer, err := pathinfo.NewServer(conn, blobServer, directoryServer) 60 | if err != nil { 61 | log.Fatalf("failed to create path info service: %v", err) 62 | } 63 | 64 | // setup metrics 65 | srvMetrics := grpcprom.NewServerMetrics( 66 | grpcprom.WithServerHandlingTimeHistogram( 67 | grpcprom.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}), 68 | ), 69 | ) 70 | reg := prometheus.NewRegistry() 71 | reg.MustRegister(srvMetrics) 72 | 73 | // setup logging 74 | rpcLogger := log.With("service", "gRPC/server") 75 | 76 | // Setup metric for panic recoveries. 77 | panicsTotal := promauto.With(reg).NewCounter(prometheus.CounterOpts{ 78 | Name: "grpc_req_panics_recovered_total", 79 | Help: "Total number of gRPC requests recovered from internal panic.", 80 | }) 81 | grpcPanicRecoveryHandler := func(p any) (err error) { 82 | panicsTotal.Inc() 83 | rpcLogger.Error("recovered from panic", "panic", p, "stack", string(debug.Stack())) 84 | return status.Errorf(codes.Internal, "%s", p) 85 | } 86 | 87 | // 88 | opts := []grpc.ServerOption{ 89 | grpc.MaxRecvMsgSize(16 * 1024 * 1024), 90 | grpc.ChainUnaryInterceptor( 91 | srvMetrics.UnaryServerInterceptor(), 92 | // todo add logging 93 | recovery.UnaryServerInterceptor(recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), 94 | ), 95 | grpc.ChainStreamInterceptor( 96 | srvMetrics.StreamServerInterceptor(), 97 | // todo add logging 98 | recovery.StreamServerInterceptor(recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), 99 | ), 100 | } 101 | 102 | grpcServer := grpc.NewServer(opts...) 103 | pb.RegisterBlobServiceServer(grpcServer, blobServer) 104 | pb.RegisterDirectoryServiceServer(grpcServer, directoryServer) 105 | tvpb.RegisterPathInfoServiceServer(grpcServer, pathInfoServer) 106 | 107 | // register for reflection to help with cli tools 108 | reflection.Register(grpcServer) 109 | 110 | srvMetrics.InitializeMetrics(grpcServer) 111 | 112 | rpcListener, err := net.Listen("tcp", r.ListenAddr) 113 | if err != nil { 114 | log.Fatalf("failed to listen: %v", err) 115 | } 116 | 117 | shutdown.Add(func() { 118 | grpcServer.GracefulStop() 119 | grpcServer.Stop() 120 | }) 121 | 122 | eg := errgroup.Group{} 123 | eg.Go(func() error { 124 | return grpcServer.Serve(rpcListener) 125 | }) 126 | 127 | metricsListener, err := net.Listen("tcp", r.MetricsAddr) 128 | if err != nil { 129 | log.Fatalf("failed to listen: %v", err) 130 | } 131 | 132 | shutdown.Add(func() { 133 | _ = metricsListener.Close() 134 | }) 135 | 136 | httpSrv := &http.Server{} 137 | 138 | eg.Go(func() error { 139 | m := http.NewServeMux() 140 | // Create HTTP handler for Prometheus metrics. 141 | m.Handle("/metrics", promhttp.HandlerFor( 142 | reg, 143 | promhttp.HandlerOpts{ 144 | // Opt into OpenMetrics e.g. to support exemplars. 145 | EnableOpenMetrics: true, 146 | }, 147 | )) 148 | httpSrv.Handler = m 149 | return httpSrv.Serve(metricsListener) 150 | }) 151 | 152 | log.Info("listening", "type", "rpc", "addr", rpcListener.Addr()) 153 | log.Info("listening", "type", "metrics", "addr", metricsListener.Addr()) 154 | 155 | shutdown.Listen(syscall.SIGINT, syscall.SIGTERM) 156 | 157 | return eg.Wait() 158 | } 159 | -------------------------------------------------------------------------------- /pkg/store/nats.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "time" 8 | 9 | "github.com/brianmcgee/nvix/pkg/util" 10 | 11 | "github.com/juju/errors" 12 | "github.com/nats-io/nats.go" 13 | ) 14 | 15 | type NatsStore struct { 16 | Conn *nats.Conn 17 | StreamConfig *nats.StreamConfig 18 | SubjectPrefix string 19 | } 20 | 21 | func (n *NatsStore) Init(ctx context.Context) error { 22 | js, err := n.js(ctx) 23 | if err != nil { 24 | return err 25 | } 26 | _, err = js.AddStream(n.StreamConfig) 27 | return err 28 | } 29 | 30 | func (n *NatsStore) Stat(key string, ctx context.Context) (ok bool, err error) { 31 | var reader io.ReadCloser 32 | reader, err = n.Get(key, ctx) 33 | if err != nil { 34 | return 35 | } 36 | defer func() { 37 | _ = reader.Close() 38 | }() 39 | 40 | // try to read, forcing an error if the entry doesn't exist 41 | b := make([]byte, 0) 42 | _, err = reader.Read(b) 43 | ok = err == nil 44 | 45 | return 46 | } 47 | 48 | func (n *NatsStore) subject(key string) string { 49 | return n.SubjectPrefix + "." + key 50 | } 51 | 52 | func (n *NatsStore) Get(key string, ctx context.Context) (io.ReadCloser, error) { 53 | js, err := n.js(ctx) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | reader := natsMsgReader{ 59 | js: js, 60 | stream: n.StreamConfig.Name, 61 | subject: n.subject(key), 62 | } 63 | 64 | return &reader, nil 65 | } 66 | 67 | func (n *NatsStore) Put(key string, reader io.ReadCloser, ctx context.Context) error { 68 | future, err := n.PutAsync(key, reader, ctx) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | select { 74 | case <-future.Ok(): 75 | return nil 76 | case err := <-future.Err(): 77 | return err 78 | } 79 | } 80 | 81 | func (n *NatsStore) PutAsync(key string, reader io.ReadCloser, ctx context.Context) (nats.PubAckFuture, error) { 82 | js, err := n.js(ctx) 83 | if err != nil { 84 | return nil, err 85 | } 86 | 87 | msg := nats.NewMsg(n.subject(key)) 88 | msg.Data, err = io.ReadAll(reader) 89 | if err != nil { 90 | return nil, err 91 | } else if err = reader.Close(); err != nil { 92 | return nil, err 93 | } 94 | 95 | // overwrite the last msg for this subject 96 | msg.Header.Set(nats.MsgRollup, nats.MsgRollupSubject) 97 | 98 | return js.PublishMsgAsync(msg) 99 | } 100 | 101 | func (n *NatsStore) Delete(key string, ctx context.Context) error { 102 | js, err := n.js(ctx) 103 | if err != nil { 104 | return err 105 | } 106 | return js.PurgeStream(n.StreamConfig.Name, &nats.StreamPurgeRequest{ 107 | Subject: n.subject(key), 108 | }) 109 | } 110 | 111 | func (n *NatsStore) List(ctx context.Context) (util.Iterator[io.ReadCloser], error) { 112 | js, err := n.js(ctx) 113 | if err != nil { 114 | return nil, err 115 | } 116 | sub, err := js.SubscribeSync(n.subject("*"), nats.DeliverAll()) 117 | if err != nil { 118 | return nil, err 119 | } 120 | return &natsIterator{ 121 | ctx: ctx, 122 | sub: sub, 123 | fetchTimeout: 5 * time.Second, 124 | numPending: 1, 125 | }, nil 126 | } 127 | 128 | func (n *NatsStore) js(_ context.Context) (nats.JetStreamContext, error) { 129 | // todo potentially extract js from ctx 130 | js, err := n.Conn.JetStream(nats.DirectGet()) 131 | if err != nil { 132 | err = errors.Annotate(err, "failed to create js context") 133 | } 134 | return js, err 135 | } 136 | 137 | type natsIterator struct { 138 | ctx context.Context 139 | sub *nats.Subscription 140 | fetchTimeout time.Duration 141 | 142 | numPending uint64 143 | } 144 | 145 | func (n *natsIterator) Next() (io.ReadCloser, error) { 146 | if n.numPending == 0 { 147 | // we have caught up 148 | return nil, io.EOF 149 | } 150 | 151 | select { 152 | case <-n.ctx.Done(): 153 | return nil, n.ctx.Err() 154 | default: 155 | ctx, cancel := context.WithTimeout(n.ctx, n.fetchTimeout) 156 | defer cancel() 157 | 158 | msg, err := n.sub.NextMsgWithContext(ctx) 159 | if err != nil { 160 | return nil, err 161 | } 162 | 163 | meta, err := msg.Metadata() 164 | if err != nil { 165 | return nil, errors.Annotate(err, "failed to get msg metadata") 166 | } 167 | 168 | n.numPending = meta.NumPending 169 | 170 | return &natsMsgReader{ 171 | reader: bytes.NewReader(msg.Data), 172 | }, nil 173 | } 174 | } 175 | 176 | func (n *natsIterator) Close() error { 177 | return n.sub.Unsubscribe() 178 | } 179 | 180 | type natsMsgReader struct { 181 | js nats.JetStreamContext 182 | stream string 183 | subject string 184 | 185 | msg *nats.RawStreamMsg 186 | reader io.Reader 187 | } 188 | 189 | func (r *natsMsgReader) Read(p []byte) (n int, err error) { 190 | if r.reader == nil && r.msg == nil { 191 | r.msg, err = r.js.GetLastMsg(r.stream, r.subject) 192 | if err == nats.ErrMsgNotFound { 193 | return 0, ErrKeyNotFound 194 | } else if err != nil { 195 | return 196 | } 197 | r.reader = bytes.NewReader(r.msg.Data) 198 | } 199 | 200 | return r.reader.Read(p) 201 | } 202 | 203 | func (r *natsMsgReader) Close() error { 204 | return nil 205 | } 206 | -------------------------------------------------------------------------------- /pkg/directory/grpc.go: -------------------------------------------------------------------------------- 1 | package directory 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/base64" 7 | "io" 8 | 9 | capb "code.tvl.fyi/tvix/castore/protos" 10 | 11 | "github.com/brianmcgee/nvix/pkg/store" 12 | "github.com/charmbracelet/log" 13 | "github.com/golang/protobuf/proto" 14 | "google.golang.org/grpc/codes" 15 | "google.golang.org/grpc/status" 16 | 17 | "github.com/nats-io/nats.go" 18 | ) 19 | 20 | func NewServer(conn *nats.Conn) (*Server, error) { 21 | return &Server{ 22 | conn: conn, 23 | store: NewDirectoryStore(conn), 24 | }, nil 25 | } 26 | 27 | type Server struct { 28 | capb.UnimplementedDirectoryServiceServer 29 | conn *nats.Conn 30 | store store.Store 31 | } 32 | 33 | func (s *Server) Get(req *capb.GetDirectoryRequest, server capb.DirectoryService_GetServer) error { 34 | l := log.WithPrefix("directory.get") 35 | 36 | rootDigest := store.Digest(req.GetDigest()) 37 | l.Debug("request", "digest", rootDigest) 38 | 39 | ctx, cancel := context.WithCancel(server.Context()) 40 | defer cancel() 41 | 42 | // todo handle get by what 43 | 44 | rootDirectory, err := s.GetByDigest(rootDigest, ctx) 45 | if err != nil { 46 | l.Errorf("failure: %v", err) 47 | return status.Errorf(codes.NotFound, "directory not found: %v", rootDigest) 48 | } 49 | 50 | dirs := []*capb.Directory{rootDirectory} 51 | 52 | iterateDirs := func(directory *capb.Directory) error { 53 | for _, dir := range directory.Directories { 54 | digest := store.Digest(dir.Digest) 55 | if d, err := s.GetByDigest(digest, ctx); err != nil { 56 | return err 57 | } else if req.Recursive { 58 | dirs = append(dirs, d) 59 | } 60 | } 61 | return nil 62 | } 63 | 64 | for _, dir := range dirs { 65 | if err = iterateDirs(dir); err != nil { 66 | l.Errorf("failure: %v", err) 67 | return status.Error(codes.Internal, "failed to iterate directories") 68 | } else if err = server.Send(dir); err != nil { 69 | l.Errorf("failure: %v", err) 70 | return status.Error(codes.Internal, "failed to send directory") 71 | } 72 | // remove from head 73 | dirs = dirs[1:] 74 | } 75 | 76 | return nil 77 | } 78 | 79 | func (s *Server) GetByDigest(digest store.Digest, ctx context.Context) (*capb.Directory, error) { 80 | l := log.WithPrefix("directory.getByDigest") 81 | 82 | reader, err := s.store.Get(digest.String(), ctx) 83 | if err != nil { 84 | l.Errorf("failure: %v", err) 85 | return nil, status.Errorf(codes.NotFound, "digest not found: %v", digest) 86 | } 87 | defer func() { 88 | _ = reader.Close() 89 | }() 90 | 91 | b, err := io.ReadAll(reader) 92 | if err != nil { 93 | l.Errorf("failure: %v", err) 94 | return nil, status.Error(codes.Internal, "failed to read directory entry from store") 95 | } 96 | var dir capb.Directory 97 | if err = proto.Unmarshal(b, &dir); err != nil { 98 | l.Errorf("failure: %v", err) 99 | return nil, status.Error(codes.Internal, "failed to unmarshal directory entry from store") 100 | } 101 | return &dir, nil 102 | } 103 | 104 | func (s *Server) Put(server capb.DirectoryService_PutServer) error { 105 | l := log.WithPrefix("directory.put") 106 | 107 | ctx, cancel := context.WithCancel(server.Context()) 108 | defer cancel() 109 | 110 | var rootDigest []byte 111 | var futures []nats.PubAckFuture 112 | 113 | cache := make(map[string]*capb.Directory) 114 | 115 | for { 116 | directory, err := server.Recv() 117 | if err == io.EOF { 118 | break 119 | } else if err != nil { 120 | l.Error("failed to receive directory", "err", err) 121 | return status.Errorf(codes.Unknown, "failed to receive directory") 122 | } 123 | 124 | if err := validateDirectory(directory); err != nil { 125 | return status.Errorf(codes.InvalidArgument, "bad request: %v", err) 126 | } 127 | 128 | digest, err := directory.Digest() 129 | if err != nil { 130 | return status.Error(codes.Unknown, "failed to generate directory digest") 131 | } 132 | 133 | digestStr := base64.StdEncoding.EncodeToString(digest) 134 | cache[digestStr] = directory 135 | 136 | for _, node := range directory.Directories { 137 | target := base64.StdEncoding.EncodeToString(node.Digest) 138 | if _, ok := cache[target]; !ok { 139 | return status.Errorf(codes.InvalidArgument, "directory node refers to unknown directory digest: %v", target) 140 | } 141 | } 142 | 143 | b, err := proto.Marshal(directory) 144 | if err != nil { 145 | l.Error("failed to marshal directory", "err", err) 146 | return status.Error(codes.Internal, "failed to marshal directory") 147 | } 148 | 149 | future, err := s.store.PutAsync(digestStr, io.NopCloser(bytes.NewReader(b)), ctx) 150 | if err != nil { 151 | l.Error("failed to put directory in store", "err", err) 152 | return status.Errorf(codes.Internal, "failed to put directory in store") 153 | } 154 | 155 | rootDigest = digest 156 | futures = append(futures, future) 157 | } 158 | 159 | for _, f := range futures { 160 | select { 161 | case <-ctx.Done(): 162 | return ctx.Err() 163 | case err := <-f.Err(): 164 | // TODO how to handle partial writes due to failure? 165 | l.Error(codes.Internal, "put future has returned an error", "err", err) 166 | return status.Errorf(codes.Internal, "failed to put directory in store") 167 | case ack := <-f.Ok(): 168 | l.Debug("put acknowledged", "ack", ack) 169 | } 170 | } 171 | 172 | l.Debug("all puts complete") 173 | 174 | resp := &capb.PutDirectoryResponse{ 175 | RootDigest: rootDigest, 176 | } 177 | if err := server.SendAndClose(resp); err != nil { 178 | l.Error("failed to send put response", "err", err) 179 | } 180 | 181 | l.Debug("finished", "digest", store.Digest(rootDigest).String()) 182 | return nil 183 | } 184 | -------------------------------------------------------------------------------- /pkg/pathinfo/grpc.go: -------------------------------------------------------------------------------- 1 | package pathinfo 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | 8 | capb "code.tvl.fyi/tvix/castore/protos" 9 | tvpb "code.tvl.fyi/tvix/store/protos" 10 | 11 | "github.com/brianmcgee/nvix/pkg/blob" 12 | "github.com/brianmcgee/nvix/pkg/directory" 13 | "github.com/brianmcgee/nvix/pkg/store" 14 | "github.com/charmbracelet/log" 15 | "github.com/golang/protobuf/proto" 16 | multihash "github.com/multiformats/go-multihash/core" 17 | "github.com/nats-io/nats.go" 18 | "github.com/nix-community/go-nix/pkg/hash" 19 | "github.com/nix-community/go-nix/pkg/nixbase32" 20 | "google.golang.org/grpc/codes" 21 | "google.golang.org/grpc/status" 22 | ) 23 | 24 | func NewServer(conn *nats.Conn, blob *blob.Server, directory *directory.Server) (*Service, error) { 25 | return &Service{ 26 | conn: conn, 27 | store: NewPathInfoStore(conn), 28 | outIdx: NewPathInfoOutIdxStore(conn), 29 | blob: blob, 30 | directory: directory, 31 | }, nil 32 | } 33 | 34 | type Service struct { 35 | tvpb.UnimplementedPathInfoServiceServer 36 | conn *nats.Conn 37 | store store.Store 38 | outIdx store.Store 39 | blob *blob.Server 40 | directory *directory.Server 41 | } 42 | 43 | func pathInfoDigest(node *capb.Node) (*store.Digest, error) { 44 | var digest store.Digest 45 | if node.GetDirectory() != nil { 46 | digest = store.Digest(node.GetDirectory().Digest) 47 | // todo check directory exists 48 | } else if node.GetFile() != nil { 49 | digest = store.Digest(node.GetFile().Digest) 50 | } else if node.GetSymlink() != nil { 51 | digest = store.Digest(node.GetSymlink().Target) 52 | } else { 53 | return nil, status.Error(codes.Internal, "unexpected node type") 54 | } 55 | 56 | return &digest, nil 57 | } 58 | 59 | func (s *Service) Get(ctx context.Context, req *tvpb.GetPathInfoRequest) (*tvpb.PathInfo, error) { 60 | // only supporting get by output hash 61 | l := log.WithPrefix("path_info.get") 62 | 63 | outHash := nixbase32.EncodeToString(req.GetByOutputHash()) 64 | l.Debug("executing", "outHash", outHash) 65 | 66 | reader, err := s.outIdx.Get(outHash, ctx) 67 | if err != nil { 68 | return nil, err 69 | } 70 | defer func() { 71 | _ = reader.Close() 72 | }() 73 | 74 | digest, err := io.ReadAll(reader) 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | reader, err = s.store.Get(store.Digest(digest).String(), ctx) 80 | if err != nil { 81 | return nil, status.Error(codes.Internal, "failed to get path info") 82 | } 83 | 84 | b, err := io.ReadAll(reader) 85 | if err != nil { 86 | return nil, status.Error(codes.Internal, "failed to read path info") 87 | } 88 | 89 | var pathInfo tvpb.PathInfo 90 | if err = proto.Unmarshal(b, &pathInfo); err != nil { 91 | return nil, status.Error(codes.Internal, "failed to unmarshal path info") 92 | } 93 | 94 | return &pathInfo, nil 95 | } 96 | 97 | func (s *Service) Put(ctx context.Context, pathInfo *tvpb.PathInfo) (*tvpb.PathInfo, error) { 98 | digest, err := pathInfoDigest(pathInfo.Node) 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | l := log.WithPrefix("path_info.put") 104 | l.Debug("executing", "digest", digest.String()) 105 | 106 | b, err := proto.Marshal(pathInfo) 107 | if err != nil { 108 | return nil, status.Error(codes.Internal, "failed to marshal path info") 109 | } 110 | 111 | err = s.store.Put(digest.String(), io.NopCloser(bytes.NewReader(b)), ctx) 112 | if err != nil { 113 | return nil, status.Error(codes.Internal, "failed to put path info") 114 | } 115 | 116 | // we need to maintain an index of outHash => digest for lookup in Get 117 | outputHash := nixbase32.EncodeToString(pathInfo.Narinfo.NarSha256) 118 | 119 | err = s.outIdx.Put( 120 | outputHash, 121 | io.NopCloser(bytes.NewReader(digest[:])), 122 | ctx, 123 | ) 124 | 125 | // no signatures to add so just return the same path info 126 | return pathInfo, err 127 | } 128 | 129 | func (s *Service) CalculateNAR(ctx context.Context, node *capb.Node) (*tvpb.CalculateNARResponse, error) { 130 | r, w := io.Pipe() 131 | 132 | go func() { 133 | err := tvpb.Export(w, node, 134 | func(digest []byte) (*capb.Directory, error) { 135 | return s.directory.GetByDigest(store.Digest(digest), ctx) 136 | }, func(digest []byte) (io.ReadCloser, error) { 137 | return s.blob.GetByDigest(store.Digest(digest), ctx) 138 | }) 139 | _ = w.CloseWithError(err) 140 | }() 141 | 142 | hasher, err := hash.New(multihash.SHA2_256) 143 | if err != nil { 144 | return nil, status.Error(codes.Internal, "failed to create a SHA256 hasher") 145 | } 146 | 147 | if _, err = io.Copy(hasher, r); err != nil { 148 | return nil, status.Error(codes.Internal, "failed to write NAR to SHA256 hasher") 149 | } 150 | 151 | return &tvpb.CalculateNARResponse{ 152 | NarSize: hasher.BytesWritten(), 153 | NarSha256: hasher.Digest(), 154 | }, nil 155 | } 156 | 157 | func (s *Service) List(_ *tvpb.ListPathInfoRequest, server tvpb.PathInfoService_ListServer) error { 158 | // only list all currently to implement 159 | ctx, cancel := context.WithCancel(server.Context()) 160 | defer cancel() 161 | 162 | l := log.WithPrefix("path_info.list") 163 | l.Debug("executing") 164 | 165 | iter, err := s.store.List(ctx) 166 | if err != nil { 167 | return status.Error(codes.Internal, "failed to create store iterator") 168 | } 169 | 170 | defer func() { 171 | _ = iter.Close() 172 | }() 173 | 174 | for { 175 | reader, err := iter.Next() 176 | if err == io.EOF { 177 | break 178 | } else if err != nil { 179 | return status.Error(codes.Internal, "failed to read next path info") 180 | } 181 | 182 | b, err := io.ReadAll(reader) 183 | _ = reader.Close() 184 | if err != nil { 185 | return status.Error(codes.Internal, "failed to read path info") 186 | } 187 | 188 | var pathInfo tvpb.PathInfo 189 | if err = proto.Unmarshal(b, &pathInfo); err != nil { 190 | return status.Error(codes.Internal, "failed to unmarshal path info") 191 | } 192 | 193 | if err = server.Send(&pathInfo); err != nil { 194 | return err 195 | } 196 | } 197 | 198 | return nil 199 | } 200 | -------------------------------------------------------------------------------- /pkg/blob/grpc_test.go: -------------------------------------------------------------------------------- 1 | package blob 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "crypto/rand" 7 | "io" 8 | "net" 9 | "testing" 10 | 11 | "github.com/inhies/go-bytesize" 12 | 13 | "github.com/brianmcgee/nvix/pkg/test" 14 | 15 | pb "code.tvl.fyi/tvix/castore/protos" 16 | 17 | "github.com/charmbracelet/log" 18 | "github.com/nats-io/nats-server/v2/server" 19 | "github.com/stretchr/testify/assert" 20 | "google.golang.org/grpc" 21 | ) 22 | 23 | var sizes = []bytesize.ByteSize{ 24 | 1 << 10, 25 | 4 << 10, 26 | 16 << 10, 27 | 32 << 10, 28 | 64 << 10, 29 | 128 << 10, 30 | 256 << 10, 31 | 512 << 10, 32 | 1 << 20, 33 | 4 << 20, 34 | 8 << 20, 35 | 16 << 20, 36 | 32 << 20, 37 | 64 << 20, 38 | 128 << 20, 39 | 256 << 20, 40 | 512 << 20, 41 | } 42 | 43 | func blobServer(s *server.Server, t test.TestingT) (*grpc.Server, net.Listener) { 44 | t.Helper() 45 | 46 | conn := test.NatsConn(t, s) 47 | 48 | ctx := context.Background() 49 | if err := NewMetaStore(conn).Init(ctx); err != nil { 50 | t.Fatal(err) 51 | } else if err := NewChunkStore(conn).Init(ctx); err != nil { 52 | t.Fatal(err) 53 | } 54 | 55 | blobService, err := NewServer(conn) 56 | if err != nil { 57 | t.Fatalf("failed to create blob service: %v", err) 58 | } 59 | 60 | srv := grpc.NewServer(grpc.MaxRecvMsgSize(16 * 1024 * 1024)) 61 | pb.RegisterBlobServiceServer(srv, blobService) 62 | 63 | lis, err := net.Listen("tcp", ":0") 64 | if err != nil { 65 | log.Fatalf("failed to listen: %v", err) 66 | } 67 | 68 | go func() { 69 | if err := srv.Serve(lis); err != nil { 70 | t.Errorf("error serving server: %v", err) 71 | } 72 | }() 73 | 74 | return srv, lis 75 | } 76 | 77 | func BenchmarkBlobService_Put(b *testing.B) { 78 | s := test.RunBasicJetStreamServer(b) 79 | defer test.ShutdownJSServerAndRemoveStorage(b, s) 80 | 81 | srv, lis := blobServer(s, b) 82 | defer srv.Stop() 83 | 84 | conn := test.GrpcConn(lis, b) 85 | client := pb.NewBlobServiceClient(conn) 86 | 87 | for _, size := range sizes { 88 | size := size 89 | b.Run(size.String(), func(b *testing.B) { 90 | b.SetBytes(int64(size)) 91 | b.ReportAllocs() 92 | b.ResetTimer() 93 | 94 | b.RunParallel(func(p *testing.PB) { 95 | data := make([]byte, size) 96 | _, err := rand.Read(data) 97 | if err != nil { 98 | b.Fatal(err) 99 | } 100 | 101 | r := bytes.NewReader(data) 102 | 103 | sendBuf := make([]byte, (16*1024*1024)-5) 104 | 105 | for p.Next() { 106 | r.Reset(data) 107 | 108 | put, err := client.Put(context.Background()) 109 | if err != nil { 110 | b.Fatal(err) 111 | } 112 | 113 | for { 114 | if n, err := r.Read(sendBuf); err != nil { 115 | if err == io.EOF { 116 | break 117 | } else { 118 | b.Fatal(err) 119 | } 120 | } else if err = put.Send(&pb.BlobChunk{Data: sendBuf[:n]}); err != nil { 121 | b.Fatal(err) 122 | } 123 | } 124 | 125 | if _, err = put.CloseAndRecv(); err != nil { 126 | b.Fatal(err) 127 | } 128 | } 129 | }) 130 | }) 131 | } 132 | } 133 | 134 | func BenchmarkBlobService_Read(b *testing.B) { 135 | s := test.RunBasicJetStreamServer(b) 136 | defer test.ShutdownJSServerAndRemoveStorage(b, s) 137 | 138 | srv, lis := blobServer(s, b) 139 | defer srv.Stop() 140 | 141 | conn := test.GrpcConn(lis, b) 142 | client := pb.NewBlobServiceClient(conn) 143 | 144 | for _, size := range sizes { 145 | size := size 146 | 147 | data := make([]byte, size) 148 | _, err := rand.Read(data) 149 | if err != nil { 150 | b.Fatal(err) 151 | } 152 | 153 | r := bytes.NewReader(data) 154 | 155 | put, err := client.Put(context.Background()) 156 | if err != nil { 157 | b.Fatal(err) 158 | } 159 | 160 | sendBuf := make([]byte, (16*1024*1024)-5) 161 | for { 162 | if n, err := r.Read(sendBuf); err != nil { 163 | if err == io.EOF { 164 | break 165 | } else { 166 | b.Fatal(err) 167 | } 168 | } else if err = put.Send(&pb.BlobChunk{Data: sendBuf[:n]}); err != nil { 169 | b.Fatal(err) 170 | } 171 | } 172 | 173 | resp, err := put.CloseAndRecv() 174 | if err != nil { 175 | b.Fatal(err) 176 | } 177 | 178 | b.Run(size.String(), func(b *testing.B) { 179 | b.SetBytes(int64(size)) 180 | b.ReportAllocs() 181 | b.ResetTimer() 182 | 183 | b.RunParallel(func(p *testing.PB) { 184 | for p.Next() { 185 | buf := bytes.NewBuffer(nil) 186 | 187 | read, err := client.Read(context.Background(), &pb.ReadBlobRequest{Digest: resp.Digest}) 188 | if err != nil { 189 | b.Fatal(err) 190 | } 191 | 192 | for { 193 | chunk, err := read.Recv() 194 | if err == io.EOF { 195 | break 196 | } else if err != nil { 197 | b.Fatal(err) 198 | } 199 | _, err = buf.Write(chunk.Data) 200 | if err != nil { 201 | b.Fatal(err) 202 | } 203 | } 204 | 205 | if buf.Len() != len(data) { 206 | b.Fatalf("Received %v bytes, expected %v", buf.Len(), len(data)) 207 | } 208 | } 209 | }) 210 | }) 211 | } 212 | } 213 | 214 | func TestBlobService_Put(t *testing.T) { 215 | s := test.RunBasicJetStreamServer(t) 216 | defer test.ShutdownJSServerAndRemoveStorage(t, s) 217 | 218 | as := assert.New(t) 219 | 220 | srv, lis := blobServer(s, t) 221 | defer srv.Stop() 222 | 223 | conn := test.GrpcConn(lis, t) 224 | client := pb.NewBlobServiceClient(conn) 225 | 226 | payload := make([]byte, 16*1024*1024) 227 | _, err := rand.Read(payload) 228 | if err != nil { 229 | t.Fatalf("failed to generate random bytes: %v", err) 230 | } 231 | 232 | chunkSize := (4 * 1024 * 1024) - 1024 // stay just under 4MB grpc limit 233 | resp, err := putBlob(client, bytes.NewReader(payload), chunkSize, t) 234 | if err != nil { 235 | t.Fatalf("failed to received a response: %v", err) 236 | } 237 | 238 | meta, err := client.Stat(context.Background(), &pb.StatBlobRequest{Digest: resp.Digest}) 239 | as.Nil(err) 240 | as.NotNil(meta) 241 | 242 | reader, writer := io.Pipe() 243 | go func() { 244 | getBlob(client, resp.Digest, writer, t) 245 | }() 246 | 247 | payload2, err := io.ReadAll(reader) 248 | if err != nil { 249 | log.Fatalf("failed to read blob: %v", err) 250 | } 251 | 252 | assert.Equal(t, payload, payload2) 253 | } 254 | -------------------------------------------------------------------------------- /pkg/store/cdc.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | 8 | "github.com/brianmcgee/nvix/pkg/util" 9 | "golang.org/x/sync/errgroup" 10 | 11 | "github.com/nats-io/nats.go" 12 | 13 | "github.com/SaveTheRbtz/fastcdc-go" 14 | pb "github.com/brianmcgee/nvix/protos" 15 | "github.com/charmbracelet/log" 16 | "github.com/golang/protobuf/proto" 17 | "github.com/juju/errors" 18 | 19 | "lukechampine.com/blake3" 20 | ) 21 | 22 | var ChunkOptions = fastcdc.Options{ 23 | MinSize: 1 * 1024 * 1024, 24 | AverageSize: 4 * 1024 * 1024, 25 | MaxSize: (8 * 1024 * 1024) - 1024, // we allow 1kb for headers to avoid max message size 26 | } 27 | 28 | type CdcStore struct { 29 | Meta Store 30 | Chunks Store 31 | } 32 | 33 | func (c *CdcStore) Init(ctx context.Context) error { 34 | if err := c.Meta.Init(ctx); err != nil { 35 | return err 36 | } 37 | return c.Chunks.Init(ctx) 38 | } 39 | 40 | func (c *CdcStore) getMeta(key string, ctx context.Context) (*pb.BlobMeta, error) { 41 | reader, err := c.Meta.Get(key, ctx) 42 | if err != nil { 43 | return nil, err 44 | } 45 | defer func() { 46 | _ = reader.Close() 47 | }() 48 | 49 | b, err := io.ReadAll(reader) 50 | 51 | if err == ErrKeyNotFound { 52 | return nil, err 53 | } else if err != nil { 54 | return nil, errors.Annotate(err, "failed to read bytes") 55 | } 56 | 57 | var meta pb.BlobMeta 58 | if err = proto.Unmarshal(b, &meta); err != nil { 59 | return nil, errors.Annotate(err, "failed to unmarshal blob metadata") 60 | } 61 | return &meta, nil 62 | } 63 | 64 | func (c *CdcStore) List(ctx context.Context) (util.Iterator[io.ReadCloser], error) { 65 | metaIterator, err := c.Meta.List(ctx) 66 | if err != nil { 67 | return nil, err 68 | } 69 | return &blobIterator{ 70 | ctx: ctx, 71 | chunks: c.Chunks, 72 | metaIterator: metaIterator, 73 | }, nil 74 | } 75 | 76 | func (c *CdcStore) Stat(digest Digest, ctx context.Context) (ok bool, err error) { 77 | return c.Meta.Stat(digest.String(), ctx) 78 | } 79 | 80 | func (c *CdcStore) Get(digest Digest, ctx context.Context) (io.ReadCloser, error) { 81 | meta, err := c.getMeta(digest.String(), ctx) 82 | if err != nil { 83 | return nil, err 84 | } 85 | 86 | return &blobReader{blob: meta, store: c.Chunks, ctx: ctx}, nil 87 | } 88 | 89 | func (c *CdcStore) Put(reader io.ReadCloser, ctx context.Context) (*Digest, error) { 90 | hasher := blake3.New(32, nil) 91 | chunkHasher := blake3.New(32, nil) 92 | 93 | chunker, err := fastcdc.NewChunker(io.TeeReader(reader, hasher), ChunkOptions) 94 | if err != nil { 95 | return nil, errors.Annotate(err, "failed to create a chunker") 96 | } 97 | 98 | var blobDigest Digest 99 | blobMeta := pb.BlobMeta{} 100 | 101 | var futures []nats.PubAckFuture 102 | 103 | for { 104 | chunk, err := chunker.Next() 105 | if err == io.EOF { 106 | // no more chunks 107 | blobDigest = Digest(hasher.Sum(nil)) 108 | break 109 | } else if err != nil { 110 | return nil, errors.Annotate(err, "failed to read next chunk") 111 | } 112 | 113 | _, err = io.Copy(chunkHasher, bytes.NewReader(chunk.Data)) 114 | if err != nil { 115 | return nil, errors.Annotate(err, "failed to write into chunk hasher") 116 | } 117 | 118 | chunkDigest := Digest(chunkHasher.Sum(nil)) 119 | 120 | future, err := c.Chunks.PutAsync(chunkDigest.String(), io.NopCloser(bytes.NewReader(chunk.Data)), ctx) 121 | if err != nil { 122 | return nil, errors.Annotate(err, "failed to put chunk") 123 | } 124 | 125 | futures = append(futures, future) 126 | 127 | blobMeta.Chunks = append(blobMeta.Chunks, &pb.BlobMeta_ChunkMeta{ 128 | Digest: chunkDigest[:], 129 | Size: uint32(len(chunk.Data)), 130 | }) 131 | 132 | chunkHasher.Reset() 133 | } 134 | 135 | for _, future := range futures { 136 | select { 137 | case <-future.Ok(): 138 | // do nothing 139 | case err := <-future.Err(): 140 | return nil, errors.Annotate(err, "failed to put chunk") 141 | } 142 | } 143 | 144 | b, err := proto.Marshal(&blobMeta) 145 | if err != nil { 146 | return nil, errors.Annotate(err, "failed to marshal blob meta") 147 | } 148 | 149 | if err = c.Meta.Put(blobDigest.String(), io.NopCloser(bytes.NewReader(b)), ctx); err != nil { 150 | return nil, errors.Annotate(err, "failed to put blob meta") 151 | } 152 | 153 | log.Debug("put complete", "digest", blobDigest, "chunks", len(blobMeta.Chunks)) 154 | 155 | return &blobDigest, nil 156 | } 157 | 158 | func (c *CdcStore) Delete(digest Digest, ctx context.Context) error { 159 | meta, err := c.getMeta(digest.String(), ctx) 160 | if err != nil { 161 | return err 162 | } 163 | 164 | if err = c.Meta.Delete(digest.String(), ctx); err != nil { 165 | return errors.Annotate(err, "failed to delete metadata entry") 166 | } 167 | 168 | // delete all the chunks 169 | for _, chunk := range meta.Chunks { 170 | digest := Digest(chunk.Digest) 171 | if err = c.Chunks.Delete(digest.String(), ctx); err != nil { 172 | return errors.Annotatef(err, "failed to delete chunk: %v", digest) 173 | } 174 | } 175 | 176 | return nil 177 | } 178 | 179 | type blobIterator struct { 180 | ctx context.Context 181 | chunks Store 182 | metaIterator util.Iterator[io.ReadCloser] 183 | } 184 | 185 | func (b *blobIterator) Next() (io.ReadCloser, error) { 186 | metaReader, err := b.metaIterator.Next() 187 | if err != nil { 188 | return nil, err 189 | } 190 | defer func() { 191 | _ = metaReader.Close() 192 | }() 193 | 194 | metaBytes, err := io.ReadAll(metaReader) 195 | if err != nil { 196 | return nil, errors.Annotate(err, "failed to read blob metadata") 197 | } 198 | 199 | var meta pb.BlobMeta 200 | if err = proto.Unmarshal(metaBytes, &meta); err != nil { 201 | return nil, err 202 | } 203 | 204 | return &blobReader{ 205 | blob: &meta, 206 | store: b.chunks, 207 | ctx: b.ctx, 208 | }, nil 209 | } 210 | 211 | func (b *blobIterator) Close() error { 212 | return b.metaIterator.Close() 213 | } 214 | 215 | type blobReader struct { 216 | blob *pb.BlobMeta 217 | store Store 218 | 219 | eg *errgroup.Group 220 | ctx context.Context 221 | readers chan io.ReadCloser 222 | 223 | reader io.ReadCloser 224 | } 225 | 226 | func (c *blobReader) Read(p []byte) (n int, err error) { 227 | if c.eg == nil { 228 | var ctx context.Context 229 | c.eg, ctx = errgroup.WithContext(c.ctx) 230 | 231 | c.readers = make(chan io.ReadCloser, 2) 232 | 233 | c.eg.Go(func() error { 234 | // close channel on return 235 | defer close(c.readers) 236 | 237 | b := make([]byte, 0) 238 | 239 | for _, chunk := range c.blob.Chunks { 240 | r, err := c.store.Get(Digest(chunk.Digest).String(), ctx) 241 | if err != nil { 242 | return err 243 | } 244 | 245 | // tickle the reader to force it to fetch the underlying message and is ready for reading 246 | _, err = r.Read(b) 247 | if err != nil { 248 | return err 249 | } 250 | c.readers <- r 251 | } 252 | return nil 253 | }) 254 | } 255 | 256 | for { 257 | if c.reader == nil { 258 | var ok bool 259 | c.reader, ok = <-c.readers 260 | if !ok { 261 | // channel has been closed 262 | err = c.eg.Wait() 263 | if err == nil { 264 | err = io.EOF 265 | } 266 | return 267 | } 268 | } 269 | 270 | n, err = c.reader.Read(p) 271 | if err == io.EOF { 272 | if err = c.Close(); err != nil { 273 | return 274 | } 275 | c.reader = nil 276 | } else { 277 | return 278 | } 279 | } 280 | } 281 | 282 | func (c *blobReader) Close() error { 283 | // do nothing 284 | return nil 285 | } 286 | -------------------------------------------------------------------------------- /gomod2nix.toml: -------------------------------------------------------------------------------- 1 | schema = 3 2 | 3 | [mod] 4 | [mod."code.tvl.fyi/tvix/castore/protos"] 5 | version = "v0.0.0-20231014122118-3fc2ade7dfb2" 6 | hash = "sha256-IdGAfqSlgK41TrzQEEtj16WERSvSTXQQ/5OVjPkVNYY=" 7 | [mod."code.tvl.fyi/tvix/store/protos"] 8 | version = "v0.0.0-20231014142132-b2dfae6a1028" 9 | hash = "sha256-aUoFjwGZZNGY8z3X4JzfyKaizXxHy4UB2Iu+PHv7XZw=" 10 | [mod."github.com/SaveTheRbtz/fastcdc-go"] 11 | version = "v0.3.0" 12 | hash = "sha256-Xh1g+8UqfT6NT73GnpHyd2qHCLAFEIlgRpSYgWodXUQ=" 13 | [mod."github.com/alecthomas/kong"] 14 | version = "v0.8.1" 15 | hash = "sha256-170mjSrLNC+0W1KhXltaa+YWYgt5gJQEcfssepcyh4E=" 16 | [mod."github.com/aymanbagabas/go-osc52/v2"] 17 | version = "v2.0.1" 18 | hash = "sha256-6Bp0jBZ6npvsYcKZGHHIUSVSTAMEyieweAX2YAKDjjg=" 19 | [mod."github.com/beorn7/perks"] 20 | version = "v1.0.1" 21 | hash = "sha256-h75GUqfwJKngCJQVE5Ao5wnO3cfKD9lSIteoLp/3xJ4=" 22 | [mod."github.com/cespare/xxhash/v2"] 23 | version = "v2.2.0" 24 | hash = "sha256-nPufwYQfTkyrEkbBrpqM3C2vnMxfIz6tAaBmiUP7vd4=" 25 | [mod."github.com/charmbracelet/lipgloss"] 26 | version = "v0.9.1" 27 | hash = "sha256-AHbabOymgDRIXsMBgJHS25/GgBWT54oGbd15EBWKeZc=" 28 | [mod."github.com/charmbracelet/log"] 29 | version = "v0.2.5" 30 | hash = "sha256-h2DkVx/fgmuXPgX4UTjglUmvqwQVhe7q4nJRu90NdO8=" 31 | [mod."github.com/davecgh/go-spew"] 32 | version = "v1.1.1" 33 | hash = "sha256-nhzSUrE1fCkN0+RL04N4h8jWmRFPPPWbCuDc7Ss0akI=" 34 | [mod."github.com/go-logfmt/logfmt"] 35 | version = "v0.6.0" 36 | hash = "sha256-RtIG2qARd5sT10WQ7F3LR8YJhS8exs+KiuUiVf75bWg=" 37 | [mod."github.com/golang/protobuf"] 38 | version = "v1.5.3" 39 | hash = "sha256-svogITcP4orUIsJFjMtp+Uv1+fKJv2Q5Zwf2dMqnpOQ=" 40 | [mod."github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"] 41 | version = "v1.0.0" 42 | hash = "sha256-80PPlslIcKUVndOvGeAOPezJxKnvEYyYvDm2zlUi5E0=" 43 | [mod."github.com/grpc-ecosystem/go-grpc-middleware/v2"] 44 | version = "v2.0.1" 45 | hash = "sha256-/pSCJhjhKmpuWM3zF9QugH7zucCrenMnGpmjalpV2EI=" 46 | [mod."github.com/inhies/go-bytesize"] 47 | version = "v0.0.0-20220417184213-4913239db9cf" 48 | hash = "sha256-1aqhD/x1I5JfRrMx+nGWdRj3a67O1p3Tf10Wc6wxQj0=" 49 | [mod."github.com/juju/errors"] 50 | version = "v1.0.0" 51 | hash = "sha256-9uZ0wNf44ilzLsvXqOsmFUpNOBFAVadj6+ZH8+QMDMk=" 52 | [mod."github.com/klauspost/compress"] 53 | version = "v1.17.1" 54 | hash = "sha256-SO/uAUJ7thMmGhPdC4Df6sBtZ6IFK+hgIgK+Y2fw/80=" 55 | [mod."github.com/klauspost/cpuid/v2"] 56 | version = "v2.2.5" 57 | hash = "sha256-/M8CHNah2/EPr0va44r1Sx+3H6E+jN8bGFi5jQkLBrM=" 58 | [mod."github.com/lucasb-eyer/go-colorful"] 59 | version = "v1.2.0" 60 | hash = "sha256-Gg9dDJFCTaHrKHRR1SrJgZ8fWieJkybljybkI9x0gyE=" 61 | [mod."github.com/mattn/go-isatty"] 62 | version = "v0.0.19" 63 | hash = "sha256-wYQqGxeqV3Elkmn26Md8mKZ/viw598R4Ych3vtt72YE=" 64 | [mod."github.com/mattn/go-runewidth"] 65 | version = "v0.0.15" 66 | hash = "sha256-WP39EU2UrQbByYfnwrkBDoKN7xzXsBssDq3pNryBGm0=" 67 | [mod."github.com/matttproud/golang_protobuf_extensions"] 68 | version = "v1.0.4" 69 | hash = "sha256-uovu7OycdeZ2oYQ7FhVxLey5ZX3T0FzShaRldndyGvc=" 70 | [mod."github.com/minio/highwayhash"] 71 | version = "v1.0.2" 72 | hash = "sha256-UeHeepKtToyA5e/w3KdmpbCn+4medesZG0cAcU6P2cY=" 73 | [mod."github.com/minio/sha256-simd"] 74 | version = "v1.0.1" 75 | hash = "sha256-4hfGDIQaWq8fvtGzHDhoK9v2IocXnJY7OAL6saMJbmA=" 76 | [mod."github.com/mr-tron/base58"] 77 | version = "v1.2.0" 78 | hash = "sha256-8FzMu3kHUbBX10pUdtGf59Ag7BNupx8ZHeUaodR1/Vk=" 79 | [mod."github.com/muesli/reflow"] 80 | version = "v0.3.0" 81 | hash = "sha256-Pou2ybE9SFSZG6YfZLVV1Eyfm+X4FuVpDPLxhpn47Cc=" 82 | [mod."github.com/muesli/termenv"] 83 | version = "v0.15.2" 84 | hash = "sha256-Eum/SpyytcNIchANPkG4bYGBgcezLgej7j/+6IhqoMU=" 85 | [mod."github.com/multiformats/go-multihash"] 86 | version = "v0.2.3" 87 | hash = "sha256-zqIIE5jMFzm+qhUrouSF+WdXGeHUEYIQvVnKWWU6mRs=" 88 | [mod."github.com/multiformats/go-varint"] 89 | version = "v0.0.7" 90 | hash = "sha256-To3Uuv7uSUJEr5OTwxE1LEIpA62xY3M/KKMNlscHmlA=" 91 | [mod."github.com/nats-io/jwt/v2"] 92 | version = "v2.5.2" 93 | hash = "sha256-DYqG0MZFLrz70pZFYYhsHPBYo5tlG3bL4NZ+6pETqZY=" 94 | [mod."github.com/nats-io/nats-server/v2"] 95 | version = "v2.10.2" 96 | hash = "sha256-c+v1Ton3loHBOqQn4BZ1gsnY2depFAZyZtfJMJVsLDc=" 97 | [mod."github.com/nats-io/nats.go"] 98 | version = "v1.31.0" 99 | hash = "sha256-n8l5DIuDqZjrDVXK2deLOuoqZySzSrSbWaVxWl/BERQ=" 100 | [mod."github.com/nats-io/nkeys"] 101 | version = "v0.4.5" 102 | hash = "sha256-txPd4Q/ApaNutt2Ik5E2478tHAQmpTJQKYnHA9niz3E=" 103 | [mod."github.com/nats-io/nuid"] 104 | version = "v1.0.1" 105 | hash = "sha256-7wddxVz3hnFg/Pf+61+MtQJJL/l8EaC8brHoNsmD64c=" 106 | [mod."github.com/nix-community/go-nix"] 107 | version = "v0.0.0-20231012070617-9b176785e54d" 108 | hash = "sha256-CuGARFZPJw8JrBnTIuDLSaQKKbcuVnnkhAamwEohOHk=" 109 | [mod."github.com/pmezard/go-difflib"] 110 | version = "v1.0.0" 111 | hash = "sha256-/FtmHnaGjdvEIKAJtrUfEhV7EVo5A/eYrtdnUkuxLDA=" 112 | [mod."github.com/prometheus/client_golang"] 113 | version = "v1.17.0" 114 | hash = "sha256-FIIzCuNqHdVzpbyH7yAp7Tcu+1tPxEMS5g6KfsGQBGE=" 115 | [mod."github.com/prometheus/client_model"] 116 | version = "v0.5.0" 117 | hash = "sha256-/sXlngf8AoEIeLIiaLg6Y7uYPVq7tI0qnLt0mUyKid4=" 118 | [mod."github.com/prometheus/common"] 119 | version = "v0.44.0" 120 | hash = "sha256-8n3gSWKDSJtGfOQgxsiCGyTnUjb5hvSxJi/hPcrE5Oo=" 121 | [mod."github.com/prometheus/procfs"] 122 | version = "v0.12.0" 123 | hash = "sha256-Y4ZZmxIpVCO67zN3pGwSk2TcI88zvmGJkgwq9DRTwFw=" 124 | [mod."github.com/rivo/uniseg"] 125 | version = "v0.4.4" 126 | hash = "sha256-B8tbL9K6ICLdm0lEhs9+h4cpjAfvFtNiFMGvQZmw0bM=" 127 | [mod."github.com/spaolacci/murmur3"] 128 | version = "v1.1.0" 129 | hash = "sha256-RWD4PPrlAsZZ8Xy356MBxpj+/NZI7w2XOU14Ob7/Y9M=" 130 | [mod."github.com/stretchr/testify"] 131 | version = "v1.8.4" 132 | hash = "sha256-MoOmRzbz9QgiJ+OOBo5h5/LbilhJfRUryvzHJmXAWjo=" 133 | [mod."github.com/ztrue/shutdown"] 134 | version = "v0.1.1" 135 | hash = "sha256-+ygx5THHu9g+vBAn6b63tV35bvQGdRyto4pLhkontJI=" 136 | [mod."golang.org/x/crypto"] 137 | version = "v0.14.0" 138 | hash = "sha256-UUSt3X/i34r1K0mU+Y5IzljX5HYy07JcHh39Pm1MU+o=" 139 | [mod."golang.org/x/net"] 140 | version = "v0.17.0" 141 | hash = "sha256-qRawHWLSsJ06QNbLhUWPXGVSO1eaioeC9xZlUEWN8J8=" 142 | [mod."golang.org/x/sync"] 143 | version = "v0.4.0" 144 | hash = "sha256-VCl5IerUva6XZqGXHa0J/r/ewsbOIIP7EBqyh1JGsXY=" 145 | [mod."golang.org/x/sys"] 146 | version = "v0.13.0" 147 | hash = "sha256-/+RDZ0a0oEfJ0k304VqpJpdrl2ZXa3yFlOxy4mjW7w0=" 148 | [mod."golang.org/x/text"] 149 | version = "v0.13.0" 150 | hash = "sha256-J34dbc8UNVIdRJUZP7jPt11oxuwG8VvrOOylxE7V3oA=" 151 | [mod."golang.org/x/time"] 152 | version = "v0.3.0" 153 | hash = "sha256-/hmc9skIswMYbivxNS7R8A6vCTUF9k2/7tr/ACkcEaM=" 154 | [mod."google.golang.org/genproto/googleapis/rpc"] 155 | version = "v0.0.0-20231012201019-e917dd12ba7a" 156 | hash = "sha256-VAMLHlDPnzbg5bNlNKDQ++pGTMRraIG1Eb5uLPJy+KA=" 157 | [mod."google.golang.org/grpc"] 158 | version = "v1.58.3" 159 | hash = "sha256-YxXO1UAc/+4E0bsSsGSiFNrY3yyR6AIml/1sVY2QJjQ=" 160 | [mod."google.golang.org/protobuf"] 161 | version = "v1.31.0" 162 | hash = "sha256-UdIk+xRaMfdhVICvKRk1THe3R1VU+lWD8hqoW/y8jT0=" 163 | [mod."gopkg.in/yaml.v3"] 164 | version = "v3.0.1" 165 | hash = "sha256-FqL9TKYJ0XkNwJFnq9j0VvJ5ZUU1RvH/52h/f5bkYAU=" 166 | [mod."lukechampine.com/blake3"] 167 | version = "v1.2.1" 168 | hash = "sha256-x9rHXfq7mAfx8J7ya7uL8tIXhlqDfaRhoFRL66rDVGk=" 169 | -------------------------------------------------------------------------------- /protos/blobstore.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.31.0 4 | // protoc v4.24.2 5 | // source: protos/blobstore.proto 6 | 7 | package protos 8 | 9 | import ( 10 | reflect "reflect" 11 | sync "sync" 12 | 13 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 14 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 15 | ) 16 | 17 | const ( 18 | // Verify that this generated code is sufficiently up-to-date. 19 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 20 | // Verify that runtime/protoimpl is sufficiently up-to-date. 21 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 22 | ) 23 | 24 | type BlobMeta struct { 25 | state protoimpl.MessageState 26 | sizeCache protoimpl.SizeCache 27 | unknownFields protoimpl.UnknownFields 28 | 29 | Chunks []*BlobMeta_ChunkMeta `protobuf:"bytes,1,rep,name=chunks,proto3" json:"chunks,omitempty"` 30 | } 31 | 32 | func (x *BlobMeta) Reset() { 33 | *x = BlobMeta{} 34 | if protoimpl.UnsafeEnabled { 35 | mi := &file_protos_blobstore_proto_msgTypes[0] 36 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 37 | ms.StoreMessageInfo(mi) 38 | } 39 | } 40 | 41 | func (x *BlobMeta) String() string { 42 | return protoimpl.X.MessageStringOf(x) 43 | } 44 | 45 | func (*BlobMeta) ProtoMessage() {} 46 | 47 | func (x *BlobMeta) ProtoReflect() protoreflect.Message { 48 | mi := &file_protos_blobstore_proto_msgTypes[0] 49 | if protoimpl.UnsafeEnabled && x != nil { 50 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 51 | if ms.LoadMessageInfo() == nil { 52 | ms.StoreMessageInfo(mi) 53 | } 54 | return ms 55 | } 56 | return mi.MessageOf(x) 57 | } 58 | 59 | // Deprecated: Use BlobMeta.ProtoReflect.Descriptor instead. 60 | func (*BlobMeta) Descriptor() ([]byte, []int) { 61 | return file_protos_blobstore_proto_rawDescGZIP(), []int{0} 62 | } 63 | 64 | func (x *BlobMeta) GetChunks() []*BlobMeta_ChunkMeta { 65 | if x != nil { 66 | return x.Chunks 67 | } 68 | return nil 69 | } 70 | 71 | type BlobMeta_ChunkMeta struct { 72 | state protoimpl.MessageState 73 | sizeCache protoimpl.SizeCache 74 | unknownFields protoimpl.UnknownFields 75 | 76 | Digest []byte `protobuf:"bytes,1,opt,name=digest,proto3" json:"digest,omitempty"` 77 | Size uint32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` 78 | } 79 | 80 | func (x *BlobMeta_ChunkMeta) Reset() { 81 | *x = BlobMeta_ChunkMeta{} 82 | if protoimpl.UnsafeEnabled { 83 | mi := &file_protos_blobstore_proto_msgTypes[1] 84 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 85 | ms.StoreMessageInfo(mi) 86 | } 87 | } 88 | 89 | func (x *BlobMeta_ChunkMeta) String() string { 90 | return protoimpl.X.MessageStringOf(x) 91 | } 92 | 93 | func (*BlobMeta_ChunkMeta) ProtoMessage() {} 94 | 95 | func (x *BlobMeta_ChunkMeta) ProtoReflect() protoreflect.Message { 96 | mi := &file_protos_blobstore_proto_msgTypes[1] 97 | if protoimpl.UnsafeEnabled && x != nil { 98 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 99 | if ms.LoadMessageInfo() == nil { 100 | ms.StoreMessageInfo(mi) 101 | } 102 | return ms 103 | } 104 | return mi.MessageOf(x) 105 | } 106 | 107 | // Deprecated: Use BlobMeta_ChunkMeta.ProtoReflect.Descriptor instead. 108 | func (*BlobMeta_ChunkMeta) Descriptor() ([]byte, []int) { 109 | return file_protos_blobstore_proto_rawDescGZIP(), []int{0, 0} 110 | } 111 | 112 | func (x *BlobMeta_ChunkMeta) GetDigest() []byte { 113 | if x != nil { 114 | return x.Digest 115 | } 116 | return nil 117 | } 118 | 119 | func (x *BlobMeta_ChunkMeta) GetSize() uint32 { 120 | if x != nil { 121 | return x.Size 122 | } 123 | return 0 124 | } 125 | 126 | var File_protos_blobstore_proto protoreflect.FileDescriptor 127 | 128 | var file_protos_blobstore_proto_rawDesc = []byte{ 129 | 0x0a, 0x16, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x62, 0x6c, 0x6f, 0x62, 0x73, 0x74, 0x6f, 130 | 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x11, 0x6e, 0x76, 0x69, 0x78, 0x2e, 0x62, 131 | 0x6c, 0x6f, 0x62, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x82, 0x01, 0x0a, 0x08, 132 | 0x42, 0x6c, 0x6f, 0x62, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x3d, 0x0a, 0x06, 0x63, 0x68, 0x75, 0x6e, 133 | 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6e, 0x76, 0x69, 0x78, 0x2e, 134 | 0x62, 0x6c, 0x6f, 0x62, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 135 | 0x62, 0x4d, 0x65, 0x74, 0x61, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x65, 0x74, 0x61, 0x52, 136 | 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x43, 0x68, 0x75, 0x6e, 0x6b, 137 | 0x4d, 0x65, 0x74, 0x61, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 138 | 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 139 | 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 140 | 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x62, 141 | 0x72, 0x69, 0x61, 0x6e, 0x6d, 0x63, 0x67, 0x65, 0x65, 0x2f, 0x6e, 0x76, 0x69, 0x78, 0x2f, 0x70, 142 | 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x62, 0x06, 0x70, 0x72, 143 | 0x6f, 0x74, 0x6f, 0x33, 144 | } 145 | 146 | var ( 147 | file_protos_blobstore_proto_rawDescOnce sync.Once 148 | file_protos_blobstore_proto_rawDescData = file_protos_blobstore_proto_rawDesc 149 | ) 150 | 151 | func file_protos_blobstore_proto_rawDescGZIP() []byte { 152 | file_protos_blobstore_proto_rawDescOnce.Do(func() { 153 | file_protos_blobstore_proto_rawDescData = protoimpl.X.CompressGZIP(file_protos_blobstore_proto_rawDescData) 154 | }) 155 | return file_protos_blobstore_proto_rawDescData 156 | } 157 | 158 | var ( 159 | file_protos_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 2) 160 | file_protos_blobstore_proto_goTypes = []interface{}{ 161 | (*BlobMeta)(nil), // 0: nvix.blobstore.v1.BlobMeta 162 | (*BlobMeta_ChunkMeta)(nil), // 1: nvix.blobstore.v1.BlobMeta.ChunkMeta 163 | } 164 | ) 165 | 166 | var file_protos_blobstore_proto_depIdxs = []int32{ 167 | 1, // 0: nvix.blobstore.v1.BlobMeta.chunks:type_name -> nvix.blobstore.v1.BlobMeta.ChunkMeta 168 | 1, // [1:1] is the sub-list for method output_type 169 | 1, // [1:1] is the sub-list for method input_type 170 | 1, // [1:1] is the sub-list for extension type_name 171 | 1, // [1:1] is the sub-list for extension extendee 172 | 0, // [0:1] is the sub-list for field type_name 173 | } 174 | 175 | func init() { file_protos_blobstore_proto_init() } 176 | func file_protos_blobstore_proto_init() { 177 | if File_protos_blobstore_proto != nil { 178 | return 179 | } 180 | if !protoimpl.UnsafeEnabled { 181 | file_protos_blobstore_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 182 | switch v := v.(*BlobMeta); i { 183 | case 0: 184 | return &v.state 185 | case 1: 186 | return &v.sizeCache 187 | case 2: 188 | return &v.unknownFields 189 | default: 190 | return nil 191 | } 192 | } 193 | file_protos_blobstore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { 194 | switch v := v.(*BlobMeta_ChunkMeta); i { 195 | case 0: 196 | return &v.state 197 | case 1: 198 | return &v.sizeCache 199 | case 2: 200 | return &v.unknownFields 201 | default: 202 | return nil 203 | } 204 | } 205 | } 206 | type x struct{} 207 | out := protoimpl.TypeBuilder{ 208 | File: protoimpl.DescBuilder{ 209 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 210 | RawDescriptor: file_protos_blobstore_proto_rawDesc, 211 | NumEnums: 0, 212 | NumMessages: 2, 213 | NumExtensions: 0, 214 | NumServices: 0, 215 | }, 216 | GoTypes: file_protos_blobstore_proto_goTypes, 217 | DependencyIndexes: file_protos_blobstore_proto_depIdxs, 218 | MessageInfos: file_protos_blobstore_proto_msgTypes, 219 | }.Build() 220 | File_protos_blobstore_proto = out.File 221 | file_protos_blobstore_proto_rawDesc = nil 222 | file_protos_blobstore_proto_goTypes = nil 223 | file_protos_blobstore_proto_depIdxs = nil 224 | } 225 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "depot": { 4 | "flake": false, 5 | "locked": { 6 | "lastModified": 1697459196, 7 | "narHash": "sha256-G01UQh0sbLFpPM14qdgmoJOwV5KNmgol/IKOt/fRYv8=", 8 | "ref": "refs/heads/canon", 9 | "rev": "652afd21b39fe7aabe81bb948ba4ba490422070d", 10 | "revCount": 19000, 11 | "type": "git", 12 | "url": "https://cl.tvl.fyi/depot" 13 | }, 14 | "original": { 15 | "type": "git", 16 | "url": "https://cl.tvl.fyi/depot" 17 | } 18 | }, 19 | "devshell": { 20 | "inputs": { 21 | "nixpkgs": [ 22 | "nixpkgs" 23 | ], 24 | "systems": "systems" 25 | }, 26 | "locked": { 27 | "lastModified": 1695973661, 28 | "narHash": "sha256-BP2H4c42GThPIhERtTpV1yCtwQHYHEKdRu7pjrmQAwo=", 29 | "owner": "numtide", 30 | "repo": "devshell", 31 | "rev": "cd4e2fda3150dd2f689caeac07b7f47df5197c31", 32 | "type": "github" 33 | }, 34 | "original": { 35 | "owner": "numtide", 36 | "repo": "devshell", 37 | "type": "github" 38 | } 39 | }, 40 | "flake-compat": { 41 | "flake": false, 42 | "locked": { 43 | "lastModified": 1696426674, 44 | "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", 45 | "owner": "edolstra", 46 | "repo": "flake-compat", 47 | "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", 48 | "type": "github" 49 | }, 50 | "original": { 51 | "owner": "edolstra", 52 | "repo": "flake-compat", 53 | "type": "github" 54 | } 55 | }, 56 | "flake-parts": { 57 | "inputs": { 58 | "nixpkgs-lib": "nixpkgs-lib" 59 | }, 60 | "locked": { 61 | "lastModified": 1696343447, 62 | "narHash": "sha256-B2xAZKLkkeRFG5XcHHSXXcP7To9Xzr59KXeZiRf4vdQ=", 63 | "owner": "hercules-ci", 64 | "repo": "flake-parts", 65 | "rev": "c9afaba3dfa4085dbd2ccb38dfade5141e33d9d4", 66 | "type": "github" 67 | }, 68 | "original": { 69 | "owner": "hercules-ci", 70 | "repo": "flake-parts", 71 | "type": "github" 72 | } 73 | }, 74 | "flake-root": { 75 | "locked": { 76 | "lastModified": 1692742795, 77 | "narHash": "sha256-f+Y0YhVCIJ06LemO+3Xx00lIcqQxSKJHXT/yk1RTKxw=", 78 | "owner": "srid", 79 | "repo": "flake-root", 80 | "rev": "d9a70d9c7a5fd7f3258ccf48da9335e9b47c3937", 81 | "type": "github" 82 | }, 83 | "original": { 84 | "owner": "srid", 85 | "repo": "flake-root", 86 | "type": "github" 87 | } 88 | }, 89 | "flake-utils": { 90 | "inputs": { 91 | "systems": "systems_2" 92 | }, 93 | "locked": { 94 | "lastModified": 1694529238, 95 | "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", 96 | "owner": "numtide", 97 | "repo": "flake-utils", 98 | "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", 99 | "type": "github" 100 | }, 101 | "original": { 102 | "owner": "numtide", 103 | "repo": "flake-utils", 104 | "type": "github" 105 | } 106 | }, 107 | "gomod2nix": { 108 | "inputs": { 109 | "flake-utils": "flake-utils", 110 | "nixpkgs": "nixpkgs" 111 | }, 112 | "locked": { 113 | "lastModified": 1694616124, 114 | "narHash": "sha256-c49BVhQKw3XDRgt+y+uPAbArtgUlMXCET6VxEBmzHXE=", 115 | "owner": "nix-community", 116 | "repo": "gomod2nix", 117 | "rev": "f95720e89af6165c8c0aa77f180461fe786f3c21", 118 | "type": "github" 119 | }, 120 | "original": { 121 | "owner": "nix-community", 122 | "repo": "gomod2nix", 123 | "type": "github" 124 | } 125 | }, 126 | "nixpkgs": { 127 | "locked": { 128 | "lastModified": 1658285632, 129 | "narHash": "sha256-zRS5S/hoeDGUbO+L95wXG9vJNwsSYcl93XiD0HQBXLk=", 130 | "owner": "NixOS", 131 | "repo": "nixpkgs", 132 | "rev": "5342fc6fb59d0595d26883c3cadff16ce58e44f3", 133 | "type": "github" 134 | }, 135 | "original": { 136 | "owner": "NixOS", 137 | "ref": "master", 138 | "repo": "nixpkgs", 139 | "type": "github" 140 | } 141 | }, 142 | "nixpkgs-lib": { 143 | "locked": { 144 | "dir": "lib", 145 | "lastModified": 1696019113, 146 | "narHash": "sha256-X3+DKYWJm93DRSdC5M6K5hLqzSya9BjibtBsuARoPco=", 147 | "owner": "NixOS", 148 | "repo": "nixpkgs", 149 | "rev": "f5892ddac112a1e9b3612c39af1b72987ee5783a", 150 | "type": "github" 151 | }, 152 | "original": { 153 | "dir": "lib", 154 | "owner": "NixOS", 155 | "ref": "nixos-unstable", 156 | "repo": "nixpkgs", 157 | "type": "github" 158 | } 159 | }, 160 | "nixpkgs_2": { 161 | "locked": { 162 | "lastModified": 1696725822, 163 | "narHash": "sha256-B7uAOS7TkLlOg1aX01rQlYbydcyB6ZnLJSfaYbKVww8=", 164 | "owner": "NixOS", 165 | "repo": "nixpkgs", 166 | "rev": "5aabb5780a11c500981993d49ee93cfa6df9307b", 167 | "type": "github" 168 | }, 169 | "original": { 170 | "owner": "NixOS", 171 | "ref": "nixpkgs-unstable", 172 | "repo": "nixpkgs", 173 | "type": "github" 174 | } 175 | }, 176 | "process-compose-flake": { 177 | "locked": { 178 | "lastModified": 1695992918, 179 | "narHash": "sha256-5tHNbk0ldLUjAqKRZog/3asiVvkD51VGK9TvwzUBs38=", 180 | "owner": "Platonic-Systems", 181 | "repo": "process-compose-flake", 182 | "rev": "1ebecb83f15736f5d4ae3feb01a8391977dd71da", 183 | "type": "github" 184 | }, 185 | "original": { 186 | "owner": "Platonic-Systems", 187 | "repo": "process-compose-flake", 188 | "type": "github" 189 | } 190 | }, 191 | "root": { 192 | "inputs": { 193 | "depot": "depot", 194 | "devshell": "devshell", 195 | "flake-compat": "flake-compat", 196 | "flake-parts": "flake-parts", 197 | "flake-root": "flake-root", 198 | "gomod2nix": "gomod2nix", 199 | "nixpkgs": "nixpkgs_2", 200 | "process-compose-flake": "process-compose-flake", 201 | "treefmt-nix": "treefmt-nix" 202 | } 203 | }, 204 | "systems": { 205 | "locked": { 206 | "lastModified": 1681028828, 207 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 208 | "owner": "nix-systems", 209 | "repo": "default", 210 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 211 | "type": "github" 212 | }, 213 | "original": { 214 | "owner": "nix-systems", 215 | "repo": "default", 216 | "type": "github" 217 | } 218 | }, 219 | "systems_2": { 220 | "locked": { 221 | "lastModified": 1681028828, 222 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 223 | "owner": "nix-systems", 224 | "repo": "default", 225 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 226 | "type": "github" 227 | }, 228 | "original": { 229 | "owner": "nix-systems", 230 | "repo": "default", 231 | "type": "github" 232 | } 233 | }, 234 | "treefmt-nix": { 235 | "inputs": { 236 | "nixpkgs": [ 237 | "nixpkgs" 238 | ] 239 | }, 240 | "locked": { 241 | "lastModified": 1695822946, 242 | "narHash": "sha256-IQU3fYo0H+oGlqX5YrgZU3VRhbt2Oqe6KmslQKUO4II=", 243 | "owner": "numtide", 244 | "repo": "treefmt-nix", 245 | "rev": "720bd006d855b08e60664e4683ccddb7a9ff614a", 246 | "type": "github" 247 | }, 248 | "original": { 249 | "owner": "numtide", 250 | "repo": "treefmt-nix", 251 | "type": "github" 252 | } 253 | } 254 | }, 255 | "root": "root", 256 | "version": 7 257 | } 258 | -------------------------------------------------------------------------------- /pkg/store/cdc_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "math/rand" 8 | "testing" 9 | 10 | "github.com/brianmcgee/nvix/pkg/subject" 11 | 12 | "github.com/nats-io/nats.go" 13 | "github.com/stretchr/testify/assert" 14 | 15 | "lukechampine.com/blake3" 16 | 17 | "github.com/brianmcgee/nvix/pkg/test" 18 | "github.com/inhies/go-bytesize" 19 | ) 20 | 21 | var ( 22 | DiskBasedStreamConfig = nats.StreamConfig{ 23 | Name: "blob_store", 24 | Subjects: []string{ 25 | subject.WithPrefix("STORE.BLOB.*"), 26 | subject.WithPrefix("STORE.CHUNK.*"), 27 | }, 28 | Replicas: 1, 29 | Discard: nats.DiscardOld, 30 | MaxMsgsPerSubject: 1, 31 | Storage: nats.FileStorage, 32 | AllowRollup: true, 33 | AllowDirect: true, 34 | Compression: nats.S2Compression, 35 | // automatically publish into the cache topic 36 | RePublish: &nats.RePublish{ 37 | Source: subject.WithPrefix("STORE.*.*"), 38 | Destination: subject.WithPrefix("CACHE.{{wildcard(1)}}.{{wildcard(2)}}"), 39 | }, 40 | } 41 | 42 | MemoryBasedStreamConfig = nats.StreamConfig{ 43 | Name: "blob_cache", 44 | Subjects: []string{ 45 | subject.WithPrefix("CACHE.BLOB.*"), 46 | subject.WithPrefix("CACHE.CHUNK.*"), 47 | }, 48 | Replicas: 1, 49 | Discard: nats.DiscardOld, 50 | MaxMsgsPerSubject: 1, 51 | Storage: nats.MemoryStorage, 52 | AllowRollup: true, 53 | AllowDirect: true, 54 | } 55 | ) 56 | 57 | func newChunkStore(conn *nats.Conn) Store { 58 | diskPrefix := DiskBasedStreamConfig.Subjects[1] 59 | diskPrefix = diskPrefix[:len(diskPrefix)-2] 60 | 61 | memoryPrefix := MemoryBasedStreamConfig.Subjects[1] 62 | memoryPrefix = memoryPrefix[:len(memoryPrefix)-2] 63 | 64 | disk := &NatsStore{ 65 | Conn: conn, 66 | StreamConfig: &DiskBasedStreamConfig, 67 | SubjectPrefix: diskPrefix, 68 | } 69 | 70 | memory := &NatsStore{ 71 | Conn: conn, 72 | StreamConfig: &MemoryBasedStreamConfig, 73 | SubjectPrefix: memoryPrefix, 74 | } 75 | 76 | return &CachingStore{ 77 | Disk: disk, 78 | Memory: memory, 79 | } 80 | } 81 | 82 | func newMetaStore(conn *nats.Conn) Store { 83 | diskPrefix := DiskBasedStreamConfig.Subjects[0] 84 | diskPrefix = diskPrefix[:len(diskPrefix)-2] 85 | 86 | memoryPrefix := MemoryBasedStreamConfig.Subjects[0] 87 | memoryPrefix = memoryPrefix[:len(memoryPrefix)-2] 88 | 89 | disk := &NatsStore{ 90 | Conn: conn, 91 | StreamConfig: &DiskBasedStreamConfig, 92 | SubjectPrefix: diskPrefix, 93 | } 94 | 95 | memory := &NatsStore{ 96 | Conn: conn, 97 | StreamConfig: &MemoryBasedStreamConfig, 98 | SubjectPrefix: memoryPrefix, 99 | } 100 | 101 | return &CachingStore{ 102 | Disk: disk, 103 | Memory: memory, 104 | } 105 | } 106 | 107 | func newCdcStore(t test.TestingT, conn *nats.Conn, js nats.JetStreamContext) *CdcStore { 108 | if _, err := js.AddStream(&DiskBasedStreamConfig); err != nil { 109 | t.Fatal(err) 110 | } 111 | 112 | if _, err := js.AddStream(&MemoryBasedStreamConfig); err != nil { 113 | t.Fatal(err) 114 | } 115 | 116 | return &CdcStore{ 117 | Meta: newMetaStore(conn), 118 | Chunks: newChunkStore(conn), 119 | } 120 | } 121 | 122 | func TestCdcStore_List(t *testing.T) { 123 | as := assert.New(t) 124 | 125 | s := test.RunBasicJetStreamServer(t) 126 | defer test.ShutdownJSServerAndRemoveStorage(t, s) 127 | 128 | conn, js := test.JsClient(t, s) 129 | 130 | js, err := conn.JetStream() 131 | if err != nil { 132 | t.Fatal(err) 133 | } 134 | 135 | store := newCdcStore(t, conn, js) 136 | rng := rand.New(rand.NewSource(1)) 137 | 138 | digests := make(map[string]bool) 139 | 140 | writeCount := 10 141 | for i := 0; i < writeCount; i++ { 142 | data := make([]byte, 8*1024*1024) 143 | rng.Read(data) 144 | 145 | digest, err := store.Put(io.NopCloser(bytes.NewReader(data)), context.Background()) 146 | as.Nil(err) 147 | 148 | digests[digest.String()] = true 149 | } 150 | 151 | iter, err := store.List(context.Background()) 152 | as.Nil(err) 153 | 154 | readCount := 0 155 | for { 156 | reader, err := iter.Next() 157 | if err == io.EOF { 158 | break 159 | } 160 | 161 | hasher := blake3.New(32, nil) 162 | as.Nil(err) 163 | 164 | _, err = io.Copy(hasher, reader) 165 | as.Nil(err) 166 | 167 | blobDigest := Digest(hasher.Sum(nil)) 168 | as.True(digests[blobDigest.String()]) 169 | readCount += 1 170 | } 171 | 172 | as.Equal(writeCount, readCount) 173 | } 174 | 175 | func TestCdcStore_PutAndGet(t *testing.T) { 176 | as := assert.New(t) 177 | 178 | s := test.RunBasicJetStreamServer(t) 179 | defer test.ShutdownJSServerAndRemoveStorage(t, s) 180 | 181 | conn, js := test.JsClient(t, s) 182 | 183 | js, err := conn.JetStream() 184 | if err != nil { 185 | t.Fatal(err) 186 | } 187 | 188 | store := newCdcStore(t, conn, js) 189 | 190 | rng := rand.New(rand.NewSource(1)) 191 | data := make([]byte, 20*1024*1024) 192 | rng.Read(data) 193 | 194 | r := bytes.NewReader(data) 195 | 196 | digest, err := store.Put(io.NopCloser(r), context.Background()) 197 | as.Nil(err) 198 | 199 | hasher := blake3.New(32, nil) 200 | _, err = hasher.Write(data) 201 | as.Nil(err) 202 | as.Equal(Digest(hasher.Sum(nil)), *digest) 203 | 204 | ok, err := store.Stat(*digest, context.Background()) 205 | as.Nil(err) 206 | as.True(ok) 207 | 208 | reader, err := store.Get(*digest, context.Background()) 209 | as.Nil(err) 210 | getData, err := io.ReadAll(reader) 211 | as.Nil(err) 212 | as.Equal(data, getData) 213 | as.Nil(reader.Close()) 214 | 215 | as.Nil(store.Delete(*digest, context.Background())) 216 | 217 | _, err = store.Get(*digest, context.Background()) 218 | as.ErrorIs(err, ErrKeyNotFound) 219 | 220 | ok, err = store.Stat(*digest, context.Background()) 221 | as.ErrorIs(err, ErrKeyNotFound) 222 | } 223 | 224 | var sizes = []bytesize.ByteSize{ 225 | 1 << 10, 226 | 4 << 10, 227 | 16 << 10, 228 | 32 << 10, 229 | 64 << 10, 230 | 128 << 10, 231 | 256 << 10, 232 | 512 << 10, 233 | 1 << 20, 234 | 4 << 20, 235 | 8 << 20, 236 | 16 << 20, 237 | 32 << 20, 238 | 64 << 20, 239 | 128 << 20, 240 | 256 << 20, 241 | } 242 | 243 | func BenchmarkCdcStore_Put(b *testing.B) { 244 | s := test.RunBasicJetStreamServer(b) 245 | defer test.ShutdownJSServerAndRemoveStorage(b, s) 246 | 247 | conn, js := test.JsClient(b, s) 248 | 249 | js, err := conn.JetStream() 250 | if err != nil { 251 | b.Fatal(err) 252 | } 253 | 254 | store := newCdcStore(b, conn, js) 255 | 256 | for _, size := range sizes { 257 | size := size 258 | b.Run(size.String(), func(b *testing.B) { 259 | b.SetBytes(int64(size)) 260 | b.ReportAllocs() 261 | b.ResetTimer() 262 | 263 | b.RunParallel(func(pb *testing.PB) { 264 | rng := rand.New(rand.NewSource(1)) 265 | data := make([]byte, size) 266 | rng.Read(data) 267 | 268 | r := bytes.NewReader(data) 269 | 270 | for pb.Next() { 271 | r.Reset(data) 272 | if _, err := store.Put(io.NopCloser(r), context.Background()); err != nil { 273 | b.Fatal(err) 274 | } 275 | } 276 | }) 277 | }) 278 | } 279 | } 280 | 281 | func BenchmarkCdcStore_Get(b *testing.B) { 282 | s := test.RunBasicJetStreamServer(b) 283 | defer test.ShutdownJSServerAndRemoveStorage(b, s) 284 | 285 | conn, js := test.JsClient(b, s) 286 | 287 | js, err := conn.JetStream() 288 | if err != nil { 289 | b.Fatal(err) 290 | } 291 | 292 | store := newCdcStore(b, conn, js) 293 | 294 | for _, size := range sizes { 295 | size := size 296 | 297 | rng := rand.New(rand.NewSource(1)) 298 | data := make([]byte, size) 299 | rng.Read(data) 300 | 301 | r := bytes.NewReader(data) 302 | 303 | digest, err := store.Put(io.NopCloser(r), context.Background()) 304 | if err != nil { 305 | b.Fatal(err) 306 | } 307 | 308 | b.Run(size.String(), func(b *testing.B) { 309 | b.SetBytes(int64(size)) 310 | b.ReportAllocs() 311 | b.ResetTimer() 312 | 313 | b.RunParallel(func(pb *testing.PB) { 314 | for pb.Next() { 315 | reader, err := store.Get(*digest, context.Background()) 316 | if err != nil { 317 | b.Fatal(err) 318 | } 319 | 320 | getData, err := io.ReadAll(reader) 321 | if err != nil { 322 | b.Fatal(err) 323 | } 324 | 325 | err = reader.Close() 326 | if err != nil { 327 | b.Fatal(err) 328 | } 329 | 330 | if len(getData) != len(data) { 331 | b.Fatalf("expected %v bytes, received %v", len(data), len(getData)) 332 | } 333 | } 334 | }) 335 | }) 336 | } 337 | } 338 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | code.tvl.fyi/tvix/castore/protos v0.0.0-20231014122118-3fc2ade7dfb2 h1:Z5GS8OUe7L/hKDbb1amArY7QgX0DSD5xaBwWxmh4H3Y= 2 | code.tvl.fyi/tvix/castore/protos v0.0.0-20231014122118-3fc2ade7dfb2/go.mod h1:hj0y8RPthqn1QPj8u2jFe2vzH7NouUoclrwo1/CSbuc= 3 | code.tvl.fyi/tvix/store/protos v0.0.0-20231014142132-b2dfae6a1028 h1:WBNA1OKC3JRW9dn8+YNSP1J5G+Bo3e3GTo23pi8dxO8= 4 | code.tvl.fyi/tvix/store/protos v0.0.0-20231014142132-b2dfae6a1028/go.mod h1:dgHQu5Swf/ImS6u6y9e+9SaUNJMWdFVhwS5iQxJ5NOM= 5 | github.com/SaveTheRbtz/fastcdc-go v0.3.0 h1:JdHvLlnijDuisYIwpRDcHZEjbxvCqtEmJ3gf35VJBgA= 6 | github.com/SaveTheRbtz/fastcdc-go v0.3.0/go.mod h1:2kMKqvBv1h9wCaUfETqsVkSESsCiFhp4YyEHyz7/SfE= 7 | github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0= 8 | github.com/alecthomas/kong v0.8.1 h1:acZdn3m4lLRobeh3Zi2S2EpnXTd1mOL6U7xVml+vfkY= 9 | github.com/alecthomas/kong v0.8.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= 10 | github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE= 11 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= 12 | github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= 13 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 14 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 15 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 16 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 17 | github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= 18 | github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= 19 | github.com/charmbracelet/log v0.2.5 h1:1yVvyKCKVV639RR4LIq1iy1Cs1AKxuNO+Hx2LJtk7Wc= 20 | github.com/charmbracelet/log v0.2.5/go.mod h1:nQGK8tvc4pS9cvVEH/pWJiZ50eUq1aoXUOjGpXvdD0k= 21 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 22 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 23 | github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= 24 | github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= 25 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 26 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 27 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 28 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 29 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 30 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 31 | github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 h1:f4tggROQKKcnh4eItay6z/HbHLqghBxS8g7pyMhmDio= 32 | github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0/go.mod h1:hKAkSgNkL0FII46ZkJcpVEAai4KV+swlIWCKfekd1pA= 33 | github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 h1:HcUWd006luQPljE73d5sk+/VgYPGUReEVz2y1/qylwY= 34 | github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1/go.mod h1:w9Y7gY31krpLmrVU5ZPG9H7l9fZuRu5/3R3S3FMtVQ4= 35 | github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= 36 | github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s= 37 | github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4= 38 | github.com/juju/errors v1.0.0 h1:yiq7kjCLll1BiaRuNY53MGI0+EQ3rF6GB+wvboZDefM= 39 | github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5Qe8= 40 | github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g= 41 | github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= 42 | github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= 43 | github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= 44 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 45 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 46 | github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= 47 | github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= 48 | github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= 49 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 50 | github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= 51 | github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= 52 | github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 53 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 54 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 55 | github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= 56 | github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= 57 | github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= 58 | github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= 59 | github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= 60 | github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= 61 | github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= 62 | github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= 63 | github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= 64 | github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= 65 | github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= 66 | github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= 67 | github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= 68 | github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= 69 | github.com/nats-io/jwt/v2 v2.5.2 h1:DhGH+nKt+wIkDxM6qnVSKjokq5t59AZV5HRcFW0zJwU= 70 | github.com/nats-io/jwt/v2 v2.5.2/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= 71 | github.com/nats-io/nats-server/v2 v2.10.2 h1:2o/OOyc/dxeMCQtrF1V/9er0SU0A3LKhDlv/+rqreBM= 72 | github.com/nats-io/nats-server/v2 v2.10.2/go.mod h1:lzrskZ/4gyMAh+/66cCd+q74c6v7muBypzfWhP/MAaM= 73 | github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E= 74 | github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8= 75 | github.com/nats-io/nkeys v0.4.5 h1:Zdz2BUlFm4fJlierwvGK+yl20IAKUm7eV6AAZXEhkPk= 76 | github.com/nats-io/nkeys v0.4.5/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= 77 | github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= 78 | github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= 79 | github.com/nix-community/go-nix v0.0.0-20231012070617-9b176785e54d h1:kwc1ivTuStqa3iBC2M/ojWPor88+YeIbZGeD2SlMYZ0= 80 | github.com/nix-community/go-nix v0.0.0-20231012070617-9b176785e54d/go.mod h1:4ZJah5sYrUSsWXIOJIsQ6iVOQyLO+ffhWXU3gblcO+E= 81 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 82 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 83 | github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= 84 | github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= 85 | github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= 86 | github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= 87 | github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= 88 | github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= 89 | github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= 90 | github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= 91 | github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 92 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 93 | github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= 94 | github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 95 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 96 | github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= 97 | github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 98 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 99 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 100 | github.com/ztrue/shutdown v0.1.1 h1:GKR2ye2OSQlq1GNVE/s2NbrIMsFdmL+NdR6z6t1k+Tg= 101 | github.com/ztrue/shutdown v0.1.1/go.mod h1:hcMWcM2SwIsQk7Wb49aYme4tX66x6iLzs07w1OYAQLw= 102 | golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= 103 | golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= 104 | golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= 105 | golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= 106 | golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= 107 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 108 | golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= 109 | golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= 110 | golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 111 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 112 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 113 | golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= 114 | golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 115 | golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= 116 | golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= 117 | golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= 118 | golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 119 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 120 | google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a h1:a2MQQVoTo96JC9PMGtGBymLp7+/RzpFc2yX/9WfFg1c= 121 | google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= 122 | google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= 123 | google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= 124 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 125 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 126 | google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= 127 | google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 128 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 129 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 130 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 131 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 132 | lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= 133 | lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= 134 | --------------------------------------------------------------------------------