├── .gitignore ├── img ├── nocacheprog.png ├── proxymode.png └── withcacheprog.png ├── functests ├── env │ ├── race.go │ ├── norace.go │ ├── recordingserver.go │ └── env.go ├── scripts_test.go ├── testscripts │ ├── dce-compliance-test.txtar │ ├── disable-get-test.txtar │ ├── metrics-test.txtar │ ├── plain-test.txtar │ ├── cgo-test.txtar │ ├── golangci-lint-v1-test.txtar │ ├── golangci-lint-v2-test.txtar │ ├── golangci-lint-v2-cgo-test.txtar │ └── proxy-mode-test.txtar ├── README.md └── go.mod ├── deployments └── compose │ ├── toxiproxy.json │ └── docker-compose.yml ├── .github ├── dependabot.yml ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── pull_request_template.md └── workflows │ ├── release.yml │ └── pull_request.yml ├── cmd └── cacheprog │ └── main.go ├── internal ├── infra │ ├── logging │ │ ├── smithy.go │ │ ├── args.go │ │ └── handler.go │ ├── cacheproto │ │ ├── writer.go │ │ ├── reader.go │ │ ├── models.go │ │ ├── mocks_world_test.go │ │ ├── server_test.go │ │ ├── server.go │ │ └── reader_test.go │ ├── storage │ │ ├── interfaces.go │ │ ├── disk_root.go │ │ ├── http.go │ │ ├── http_test.go │ │ ├── http_contract.go │ │ ├── disk_test.go │ │ ├── mocks_world_test.go │ │ └── s3_test.go │ ├── metrics │ │ └── metrics.go │ ├── compression │ │ └── codec.go │ └── debugging │ │ └── debugging.go └── app │ ├── app_proxy.go │ ├── proxy │ ├── handler_test.go │ └── handler.go │ ├── cacheprog │ ├── observers.go │ └── mocks_world_test.go │ ├── app.go │ └── app_cacheprog.go ├── .golangci.yml ├── .goreleaser.yml ├── pkg └── httpstorage │ ├── mocks_world_test.go │ ├── server.go │ └── server_test.go ├── go.mod ├── go.sum └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | /bin 2 | /.cachetest 3 | /coverage 4 | *.out 5 | *-tests.json 6 | dist/ 7 | -------------------------------------------------------------------------------- /img/nocacheprog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/platacard/cacheprog/HEAD/img/nocacheprog.png -------------------------------------------------------------------------------- /img/proxymode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/platacard/cacheprog/HEAD/img/proxymode.png -------------------------------------------------------------------------------- /functests/env/race.go: -------------------------------------------------------------------------------- 1 | //go:build race 2 | 3 | package env 4 | 5 | const RaceEnabled = true 6 | -------------------------------------------------------------------------------- /img/withcacheprog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/platacard/cacheprog/HEAD/img/withcacheprog.png -------------------------------------------------------------------------------- /functests/env/norace.go: -------------------------------------------------------------------------------- 1 | //go:build !race 2 | 3 | package env 4 | 5 | const RaceEnabled = false 6 | -------------------------------------------------------------------------------- /deployments/compose/toxiproxy.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "minio_master", 4 | "listen": "[::]:9000", 5 | "upstream": "minio:9000", 6 | "enabled": true 7 | } 8 | ] -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | schedule: 6 | interval: "monthly" 7 | groups: 8 | aws_sdk: 9 | applies-to: version-updates 10 | patterns: 11 | - "github.com/aws/aws-sdk-go-v2*" 12 | - "github.com/aws/smithy-go*" 13 | cooldown: 14 | default-days: 7 15 | 16 | -------------------------------------------------------------------------------- /functests/scripts_test.go: -------------------------------------------------------------------------------- 1 | package functests_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/platacard/cacheprog/functests/env" 7 | "github.com/rogpeppe/go-internal/testscript" 8 | ) 9 | 10 | func TestCacheprog_Functional(t *testing.T) { 11 | testEnv := env.NewEnv(t) 12 | t.Run("scripts", func(t *testing.T) { 13 | testscript.Run(t, testEnv.GetTestScriptParams(t)) 14 | }) 15 | } 16 | -------------------------------------------------------------------------------- /cmd/cacheprog/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | 9 | "github.com/platacard/cacheprog/internal/app" 10 | ) 11 | 12 | var Version = "development" 13 | 14 | func main() { 15 | sigCtx, sigCancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) 16 | exitCode := app.RunApp(sigCtx, Version, os.Args[1:]...) 17 | sigCancel() 18 | os.Exit(exitCode) 19 | } 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Submit a feature request to improve Terratest. 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the solution you'd like** 11 | A clear and concise description of what you want to happen. 12 | 13 | **Describe alternatives you've considered** 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the feature request here. 18 | 19 | -------------------------------------------------------------------------------- /functests/testscripts/dce-compliance-test.txtar: -------------------------------------------------------------------------------- 1 | # This test checks that we compiled code doesn't contain 'MethodByName' 2 | # or similar calls that may disable dead-code-elimination (dce). 3 | # We really need it functioning because AWS SDK without dce almost doubles executable size. 4 | # We want our executable to be small to reduce build times on CI environments. 5 | 6 | set-cacheprog COMPILED_BINARY 7 | go tool nm ${COMPILED_BINARY} 8 | ! stdout 'reflect\.(Value|Type)\.MethodByName' 9 | ! stdout 'reflect\.(Value|Type)\.Method' 10 | 11 | # this check needed to ensure that test is working correctly 12 | ! stdout 'no symbol' 13 | -------------------------------------------------------------------------------- /functests/README.md: -------------------------------------------------------------------------------- 1 | E2E tests for cacheprog 2 | ======== 3 | 4 | To run tests you need to have docker or any compatible container runtime on your machine. 5 | 6 | To run tests: 7 | 8 | ```bash 9 | go test -count=1 ./... 10 | ``` 11 | 12 | `-count=1` flag is used to disable caching of test results. 13 | 14 | `-short` flag also supported. If specified, tests will be run without starting docker compose stack. 15 | 16 | To run tests with race detector and coverage: 17 | 18 | ```bash 19 | GOCOVERDIR="" go test -race -count=1 -covermode=atomic ./... 20 | ``` 21 | And then run following command to get text coverage report: 22 | 23 | ```bash 24 | go tool covdata textfmt -i= -o= 25 | ``` 26 | -------------------------------------------------------------------------------- /internal/infra/logging/smithy.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "cmp" 5 | "context" 6 | "log/slog" 7 | 8 | "github.com/aws/smithy-go/logging" 9 | ) 10 | 11 | type SmithyLogger struct { 12 | *slog.Logger 13 | 14 | ctx context.Context 15 | } 16 | 17 | func (s *SmithyLogger) Logf(classification logging.Classification, format string, v ...any) { 18 | level := slog.LevelInfo 19 | switch classification { 20 | case logging.Debug: 21 | level = slog.LevelDebug 22 | case logging.Warn: 23 | level = slog.LevelWarn 24 | } 25 | 26 | s.Log(cmp.Or(s.ctx, context.Background()), level, format, v...) 27 | } 28 | 29 | func (s *SmithyLogger) WithContext(ctx context.Context) logging.Logger { 30 | return &SmithyLogger{ 31 | Logger: s.Logger, 32 | ctx: ctx, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /internal/infra/cacheproto/writer.go: -------------------------------------------------------------------------------- 1 | package cacheproto 2 | 3 | import ( 4 | "cmp" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "log/slog" 10 | ) 11 | 12 | type Writer struct { 13 | enc *json.Encoder 14 | } 15 | 16 | func NewWriter(w io.Writer) *Writer { 17 | return &Writer{enc: json.NewEncoder(w)} 18 | } 19 | 20 | func (w *Writer) WriteResponse(resp *Response) error { 21 | if err := w.enc.Encode(resp); err != nil { 22 | return fmt.Errorf("encode response: %w", err) 23 | } 24 | return nil 25 | } 26 | 27 | type noopWriter struct { 28 | ctx context.Context 29 | } 30 | 31 | func (w *noopWriter) WriteResponse(response *Response) error { 32 | if response.Err != "" { 33 | slog.ErrorContext(cmp.Or(w.ctx, context.Background()), "Response error", "error", response.Err) 34 | } 35 | return nil 36 | } 37 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a bug report to help us improve Terratest. 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior, code snippets and examples which can be used to reproduce the issue. 15 | 16 | ```go 17 | // paste code snippets here 18 | ``` 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Nice to have** 24 | - [ ] Terminal output 25 | - [ ] Screenshots 26 | 27 | **Versions** 28 | - Terratest version: 29 | - Environment details (Ubuntu 22.04, Windows 10, etc.): 30 | 31 | **Additional context** 32 | Add any other context about the problem here. 33 | 34 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Description 4 | 5 | Fixes #000. 6 | 7 | 8 | 9 | ## TODOs 10 | 11 | Read the [Contribution guidelines](/.github/CODE_OF_CONDUCT.md). 12 | 13 | - [ ] Generate the docs. 14 | - [ ] Run the relevant tests successfully. 15 | - [ ] Include release notes. If this PR is backward incompatible, include a migration guide. 16 | 17 | ## Release Notes (draft) 18 | 19 | 20 | Added / Removed / Updated [X]. 21 | 22 | ### Migration Guide 23 | 24 | 25 | -------------------------------------------------------------------------------- /internal/infra/logging/args.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "log/slog" 7 | ) 8 | 9 | func Error(err error) slog.Attr { 10 | if err == nil { 11 | return slog.Any("error", nil) 12 | } 13 | return slog.String("error", err.Error()) 14 | } 15 | 16 | type argsContextKey struct{} 17 | 18 | func AttachArgs(ctx context.Context, args ...any) context.Context { 19 | existingArgs, _ := ctx.Value(argsContextKey{}).(*[]any) 20 | if existingArgs == nil { 21 | args := append([]any{}, args...) 22 | return context.WithValue(ctx, argsContextKey{}, &args) 23 | } 24 | 25 | *existingArgs = append(*existingArgs, args...) 26 | return ctx 27 | } 28 | 29 | type base64Bytes []byte 30 | 31 | func (b base64Bytes) LogValue() slog.Value { 32 | return slog.StringValue(base64.StdEncoding.EncodeToString(b)) 33 | } 34 | 35 | func Bytes(b []byte) slog.Value { 36 | return slog.AnyValue(base64Bytes(b)) 37 | } 38 | -------------------------------------------------------------------------------- /internal/infra/storage/interfaces.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | 8 | "github.com/aws/aws-sdk-go-v2/service/s3" 9 | ) 10 | 11 | //go:generate go tool go.uber.org/mock/mockgen -destination=mocks_world_test.go -package=$GOPACKAGE -source=$GOFILE 12 | 13 | type S3Client interface { 14 | PutObject(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error) 15 | HeadObject(context.Context, *s3.HeadObjectInput, ...func(*s3.Options)) (*s3.HeadObjectOutput, error) 16 | GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error) 17 | } 18 | 19 | type ( 20 | DiskFile interface { 21 | io.ReadWriteSeeker 22 | io.Closer 23 | Stat() (os.FileInfo, error) 24 | } 25 | 26 | DiskRoot interface { 27 | OpenFile(name string, flag int, perm os.FileMode) (DiskFile, error) 28 | Rename(oldpath, newpath string) error 29 | Mkdir(name string, perm os.FileMode) error 30 | Remove(name string) error 31 | FullPath(name string) string 32 | } 33 | ) 34 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | permissions: 9 | contents: write 10 | packages: write 11 | 12 | jobs: 13 | release: 14 | name: Release 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Checkout code 19 | uses: actions/checkout@v4 20 | with: 21 | fetch-depth: 0 22 | 23 | - name: Set up Go 24 | uses: actions/setup-go@v5 25 | with: 26 | go-version: '1.25' 27 | cache: true 28 | 29 | - name: Set up Docker Buildx 30 | uses: docker/setup-buildx-action@v3 31 | 32 | - name: Log in to GitHub Container Registry 33 | uses: docker/login-action@v3 34 | with: 35 | registry: ghcr.io 36 | username: ${{ github.actor }} 37 | password: ${{ secrets.GITHUB_TOKEN }} 38 | 39 | - name: Run GoReleaser 40 | uses: goreleaser/goreleaser-action@v6 41 | with: 42 | distribution: goreleaser 43 | version: '~> v2.12' 44 | args: release --clean 45 | env: 46 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 47 | -------------------------------------------------------------------------------- /internal/infra/storage/disk_root.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | ) 8 | 9 | type SystemDiskRoot struct { 10 | root *os.Root 11 | } 12 | 13 | func NewSystemDiskRoot(dir string) (*SystemDiskRoot, error) { 14 | if dir == "" { 15 | return nil, fmt.Errorf("dir is empty") 16 | } 17 | 18 | root, err := os.OpenRoot(dir) 19 | if err != nil { 20 | return nil, err 21 | } 22 | return &SystemDiskRoot{root: root}, nil 23 | } 24 | 25 | func (s *SystemDiskRoot) OpenFile(name string, flag int, perm os.FileMode) (DiskFile, error) { 26 | return s.root.OpenFile(name, flag, perm) 27 | } 28 | 29 | func (s *SystemDiskRoot) Rename(oldpath, newpath string) error { 30 | return s.root.Rename(oldpath, newpath) 31 | } 32 | 33 | func (s *SystemDiskRoot) Mkdir(name string, perm os.FileMode) error { 34 | return s.root.Mkdir(name, perm) 35 | } 36 | 37 | func (s *SystemDiskRoot) Remove(name string) error { 38 | return s.root.Remove(name) 39 | } 40 | 41 | func (s *SystemDiskRoot) FullPath(name string) string { 42 | return filepath.Join(s.root.Name(), name) 43 | } 44 | 45 | func (s *SystemDiskRoot) Close() error { 46 | return s.root.Close() 47 | } 48 | -------------------------------------------------------------------------------- /functests/testscripts/disable-get-test.txtar: -------------------------------------------------------------------------------- 1 | env COMPILER_EXPRESSION='(compile|gccgo)( |\.exe).*main\.go' 2 | 3 | # control run 4 | go build -x -o test_bin$exe ./main.go 5 | stderr ${COMPILER_EXPRESSION} 6 | exec ./test_bin$exe 7 | cmp stdout expected_output.txt 8 | rm test_bin$exe 9 | 10 | # prepare environment 11 | env CACHEPROG_ROOT_DIRECTORY=$WORK/disk-cache 12 | mkdir -p ${CACHEPROG_ROOT_DIRECTORY} 13 | [!short] env CACHEPROG_REMOTE_STORAGE_TYPE=s3 CACHEPROG_S3_PREFIX=tests/disable-get-test 14 | # cleanup path for consistency 15 | [!short] env MC_FULLPATH=${MINIO_ALIAS}/${CACHEPROG_S3_BUCKET}/${CACHEPROG_S3_PREFIX} 16 | [!short] mc rm --recursive --force ${MC_FULLPATH} 17 | 18 | # run with cacheprog first time to populate cache 19 | set-cacheprog GOCACHEPROG 20 | go build -x -o test_bin$exe ./main.go 21 | stderr ${COMPILER_EXPRESSION} 22 | exec ./test_bin$exe 23 | cmp stdout expected_output.txt 24 | rm test_bin$exe 25 | 26 | # disable 'get', we must recompile project again 27 | env CACHEPROG_DISABLE_GET=true 28 | go build -x -o test_bin$exe ./main.go 29 | stderr ${COMPILER_EXPRESSION} 30 | exec ./test_bin$exe 31 | cmp stdout expected_output.txt 32 | rm test_bin$exe 33 | 34 | -- main.go -- 35 | package main 36 | 37 | import "fmt" 38 | 39 | func main() { 40 | fmt.Println("Hello world!") 41 | } 42 | 43 | -- expected_output.txt -- 44 | Hello world! 45 | -------------------------------------------------------------------------------- /internal/infra/logging/handler.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "cmp" 5 | "context" 6 | "io" 7 | "log/slog" 8 | "os" 9 | "time" 10 | ) 11 | 12 | type Config struct { 13 | Level slog.Level // logging level 14 | Output io.Writer // logging output, stderr if not provided 15 | } 16 | 17 | func CreateHandler(w io.Writer, level slog.Level) slog.Handler { 18 | return &argsHandler{ 19 | slog.NewTextHandler(w, 20 | &slog.HandlerOptions{ 21 | Level: level, 22 | AddSource: level == slog.LevelDebug, 23 | ReplaceAttr: func(_ []string, a slog.Attr) slog.Attr { 24 | if a.Key == "time" { 25 | return slog.Attr{ 26 | Key: "time", 27 | Value: slog.StringValue(a.Value.Time().Format(time.DateTime)), 28 | } 29 | } 30 | 31 | return a 32 | }, 33 | }, 34 | ), 35 | } 36 | } 37 | 38 | func ConfigureLogger(cfg Config) *slog.Logger { 39 | return slog.New(CreateHandler(cmp.Or(io.Writer(os.Stderr), cfg.Output), cfg.Level)) 40 | } 41 | 42 | type argsHandler struct { 43 | slog.Handler 44 | } 45 | 46 | func (h *argsHandler) Handle(ctx context.Context, record slog.Record) error { 47 | args, _ := ctx.Value(argsContextKey{}).([]any) 48 | record.Add(args...) 49 | return h.Handler.Handle(ctx, record) 50 | } 51 | 52 | func (h *argsHandler) WithAttrs(attrs []slog.Attr) slog.Handler { 53 | return &argsHandler{h.Handler.WithAttrs(attrs)} 54 | } 55 | 56 | func (h *argsHandler) WithGroup(name string) slog.Handler { 57 | return &argsHandler{h.Handler.WithGroup(name)} 58 | } 59 | -------------------------------------------------------------------------------- /functests/testscripts/metrics-test.txtar: -------------------------------------------------------------------------------- 1 | # prepare environment, we don't need s3 here 2 | recording-server start CACHEPROG_METRICS_PUSH_ENDPOINT 3 | env CACHEPROG_ROOT_DIRECTORY=$WORK/disk-cache CACHEPROG_METRICS_PUSH_METHOD=PUT CACHEPROG_METRICS_PUSH_EXTRA_LABELS=testlabel=testvalue 4 | set-cacheprog GOCACHEPROG 5 | 6 | # run with vm-style histograms 7 | env CACHEPROG_METRICS_PUSH_EXTRA_HEADERS=X-Test-Record-Key:vmhist,X-Extra-Header:some CACHEPROG_USE_VM_HISTOGRAMS=1 8 | go build -o test_bin$exe ./main.go 9 | rm test_bin$exe 10 | recording-server get vmhist method 11 | stdout PUT 12 | recording-server get vmhist headers 13 | stdout 'X-Extra-Header: some' 14 | recording-server get vmhist body 15 | stdout 'cacheprog_overall_run_time_ms' 16 | stdout 'cacheprog_object_get_size_bytes_bucket.*vmrange=(.+)\.\.(.+)' 17 | stdout 'testlabel="testvalue"' 18 | 19 | # run with prometheus-style histograms 20 | env CACHEPROG_METRICS_PUSH_EXTRA_HEADERS=X-Test-Record-Key:promhist CACHEPROG_USE_VM_HISTOGRAMS=0 21 | rm ${CACHEPROG_ROOT_DIRECTORY} 22 | go build -o test_bin$exe ./main.go 23 | rm test_bin$exe 24 | recording-server get promhist method 25 | stdout PUT 26 | recording-server get promhist headers 27 | ! stdout 'X-Extra-Header: some' 28 | recording-server get promhist body 29 | stdout 'cacheprog_overall_run_time_ms' 30 | stdout 'cacheprog_object_get_size_bytes_bucket.*le="\+Inf"' 31 | stdout 'testlabel="testvalue"' 32 | 33 | -- main.go -- 34 | package main 35 | 36 | import "fmt" 37 | 38 | func main() { 39 | fmt.Println("Hello world!") 40 | } 41 | -------------------------------------------------------------------------------- /functests/testscripts/plain-test.txtar: -------------------------------------------------------------------------------- 1 | env COMPILER_EXPRESSION='(compile|gccgo)( |\.exe).*main\.go' 2 | 3 | # control run 4 | go build -x -o test_bin$exe ./main.go 5 | stderr ${COMPILER_EXPRESSION} 6 | exec ./test_bin$exe 7 | cmp stdout expected_output.txt 8 | rm test_bin$exe 9 | # run with cache 10 | mkdir -p $WORK/disk-cache 11 | env CACHEPROG_ROOT_DIRECTORY=$WORK/disk-cache 12 | 13 | # if short testing enable, use only disk-backed cache 14 | [!short] env CACHEPROG_REMOTE_STORAGE_TYPE=s3 CACHEPROG_S3_PREFIX=tests/plain-test 15 | # cleanup path for consistency 16 | [!short] env MC_FULLPATH=${MINIO_ALIAS}/${CACHEPROG_S3_BUCKET}/${CACHEPROG_S3_PREFIX} 17 | [!short] mc rm --recursive --force ${MC_FULLPATH} 18 | 19 | # run with cacheprog first time 20 | set-cacheprog GOCACHEPROG 21 | go build -x -o test_bin$exe ./main.go 22 | stderr ${COMPILER_EXPRESSION} 23 | exec ./test_bin$exe 24 | cmp stdout expected_output.txt 25 | rm test_bin$exe 26 | 27 | # ensure that minio bucket is not empty 28 | [!short] mc du ${MC_FULLPATH} 29 | [!short] stdout ${CACHEPROG_S3_BUCKET}/${CACHEPROG_S3_PREFIX} 30 | [!short] ! stdout '^0B' 31 | [!short] ! stdout '(^|\W)0 object' 32 | 33 | # run with cache second time to ensure we're getting consistent result, clean disk cache 34 | rm ${CACHEPROG_ROOT_DIRECTORY} 35 | go build -x -o test_bin$exe ./main.go 36 | # if s3 used we should not run compiler, if s3 is not used we should 37 | [!short] ! stderr ${COMPILER_EXPRESSION} 38 | [short] stderr ${COMPILER_EXPRESSION} 39 | exec ./test_bin$exe 40 | cmp stdout expected_output.txt 41 | rm test_bin$exe 42 | 43 | -- main.go -- 44 | package main 45 | 46 | import "fmt" 47 | 48 | func main() { 49 | fmt.Println("Hello world!") 50 | } 51 | 52 | -- expected_output.txt -- 53 | Hello world! 54 | -------------------------------------------------------------------------------- /functests/env/recordingserver.go: -------------------------------------------------------------------------------- 1 | package env 2 | 3 | import ( 4 | "compress/gzip" 5 | "io" 6 | "net/http" 7 | "net/url" 8 | "sync" 9 | ) 10 | 11 | type recordingServer struct { 12 | mu sync.RWMutex 13 | rawRequests map[string]recordedRequest 14 | } 15 | 16 | type recordedRequest struct { 17 | Method string 18 | Path string 19 | Query url.Values 20 | Headers http.Header 21 | Body []byte 22 | } 23 | 24 | func newRecordingServer() *recordingServer { 25 | return &recordingServer{ 26 | rawRequests: make(map[string]recordedRequest), 27 | } 28 | } 29 | 30 | func (s *recordingServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { 31 | recordKey := r.Header.Get("X-Test-Record-Key") 32 | if recordKey == "" { 33 | http.Error(w, "X-Test-Record-Key header is required", http.StatusBadRequest) 34 | return 35 | } 36 | 37 | bodyReader := r.Body 38 | if r.Header.Get("Content-Encoding") == "gzip" { 39 | var err error 40 | bodyReader, err = gzip.NewReader(r.Body) 41 | if err != nil { 42 | http.Error(w, "failed to create gzip reader", http.StatusInternalServerError) 43 | return 44 | } 45 | } 46 | 47 | body, err := io.ReadAll(bodyReader) 48 | if err != nil { 49 | http.Error(w, "failed to read request body", http.StatusInternalServerError) 50 | return 51 | } 52 | 53 | s.mu.Lock() 54 | defer s.mu.Unlock() 55 | s.rawRequests[recordKey] = recordedRequest{ 56 | Method: r.Method, 57 | Path: r.URL.Path, 58 | Query: r.URL.Query(), 59 | Headers: r.Header, 60 | Body: body, 61 | } 62 | } 63 | 64 | func (s *recordingServer) GetRequest(recordKey string) (recordedRequest, bool) { 65 | s.mu.RLock() 66 | defer s.mu.RUnlock() 67 | request, ok := s.rawRequests[recordKey] 68 | return request, ok 69 | } 70 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | default: none 4 | enable: 5 | - bodyclose 6 | - copyloopvar 7 | - dogsled 8 | - dupl 9 | - errcheck 10 | - errchkjson 11 | - errname 12 | - funlen 13 | - goconst 14 | - gocritic 15 | - gocyclo 16 | - goprintffuncname 17 | - gosec 18 | - govet 19 | - ineffassign 20 | - mirror 21 | - misspell 22 | - nakedret 23 | - noctx 24 | - nolintlint 25 | - prealloc 26 | - revive 27 | - staticcheck 28 | - unconvert 29 | - unparam 30 | - unused 31 | - usestdlibvars 32 | - whitespace 33 | settings: 34 | errcheck: 35 | exclude-functions: 36 | - encoding/json.Marshal 37 | - encoding/json.MarshalIndent 38 | errchkjson: 39 | check-error-free-encoding: true 40 | funlen: 41 | lines: 100 42 | statements: 60 43 | gocyclo: 44 | min-complexity: 20 45 | nolintlint: 46 | require-explanation: true 47 | require-specific: true 48 | exclusions: 49 | generated: lax 50 | presets: 51 | - comments 52 | - common-false-positives 53 | - legacy 54 | - std-error-handling 55 | rules: 56 | - linters: 57 | - dupl 58 | - errcheck 59 | - funlen 60 | - gosec 61 | path: _test.go 62 | - linters: 63 | - goconst 64 | path: (.+)_test\.go 65 | paths: 66 | - third_party$ 67 | - builtin$ 68 | - examples$ 69 | formatters: 70 | enable: 71 | - gci 72 | - gofmt 73 | settings: 74 | gci: 75 | sections: 76 | - standard 77 | - default 78 | - prefix(github.com/platacard/cacheprog) 79 | exclusions: 80 | generated: lax 81 | paths: 82 | - third_party$ 83 | - builtin$ 84 | - examples$ 85 | -------------------------------------------------------------------------------- /deployments/compose/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | toxiproxy: 3 | image: ${TOXIPROXY_IMAGE:-ghcr.io/shopify/toxiproxy:2.12.0} 4 | restart: always 5 | ports: 6 | - "127.0.0.1:8474:8474" 7 | - "127.0.0.1:9000:9000" # for minio, simulate network latency 8 | volumes: 9 | - ./toxiproxy.json:/toxiproxy.json:ro 10 | command: -config /toxiproxy.json 11 | networks: 12 | - minio_network 13 | minio: 14 | image: ${MINIO_IMAGE:-minio/minio:RELEASE.2025-07-23T15-54-02Z} 15 | restart: always 16 | ports: 17 | - "127.0.0.1:9001:9001" 18 | environment: 19 | MINIO_ROOT_USER: "${MINIO_ROOT_USER:-minioadmin}" 20 | MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-minioadmin}" 21 | command: server /data --console-address ":9001" 22 | healthcheck: 23 | test: [ "CMD", "mc", "ready", "local" ] 24 | interval: 2s 25 | timeout: 5s 26 | retries: 5 27 | networks: 28 | - minio_network 29 | mc: 30 | image: ${MINIO_IMAGE:-minio/minio:RELEASE.2025-07-23T15-54-02Z} 31 | container_name: minio-client 32 | depends_on: 33 | minio: 34 | condition: service_healthy 35 | restart: on-failure 36 | entrypoint: 37 | - /bin/bash 38 | - -c 39 | - | 40 | set -eou pipefail; 41 | until mc alias set myminio http://minio:9000 ${MINIO_ROOT_USER:-minioadmin} ${MINIO_ROOT_PASSWORD:-minioadmin}; do 42 | echo 'Waiting for MinIO to be ready...' 43 | sleep 1 44 | done; 45 | mc mb --ignore-existing myminio/files-bucket; 46 | read -s; 47 | healthcheck: 48 | test: ["CMD", "mc", "stat", "myminio/files-bucket"] 49 | interval: 2s 50 | timeout: 5s 51 | retries: 5 52 | networks: 53 | - minio_network 54 | stdin_open: true 55 | tty: true 56 | 57 | networks: 58 | minio_network: 59 | -------------------------------------------------------------------------------- /internal/app/app_proxy.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "log/slog" 8 | "net/http" 9 | "net/url" 10 | "time" 11 | 12 | "github.com/platacard/cacheprog/internal/app/proxy" 13 | ) 14 | 15 | type ProxyAppArgs struct { 16 | MetricsProxyArgs 17 | RemoteStorageArgs 18 | 19 | ListenAddress string `arg:"--listen-address,env:PROXY_LISTEN_ADDRESS" placeholder:"ADDR" default:":8080" help:"Listen address"` 20 | } 21 | 22 | type MetricsProxyArgs struct { 23 | Endpoint *url.URL `arg:"--metrics-proxy-endpoint,env:METRICS_PROXY_ENDPOINT" placeholder:"URL" help:"Metrics endpoint, metrics push proxy endpoint will be enabled if provided"` 24 | ExtraLabels map[string]string `arg:"--metrics-proxy-extra-labels,env:METRICS_PROXY_EXTRA_LABELS" placeholder:"[key=value]" help:"Extra labels to be added to each metric, format: key=value"` 25 | ExtraHeaders []httpHeader `arg:"--metrics-proxy-extra-headers,env:METRICS_PROXY_EXTRA_HEADERS" placeholder:"[key:value]" help:"Extra headers to be added to each request."` 26 | } 27 | 28 | func (a *ProxyAppArgs) Run(ctx context.Context) error { 29 | remoteStorage, err := a.configureRemoteStorage() 30 | if err != nil { 31 | return fmt.Errorf("failed to configure remote storage: %w", err) 32 | } 33 | 34 | handler, err := proxy.NewHandler(remoteStorage, proxy.MetricsConfig{ 35 | Endpoint: urlOrEmpty(a.Endpoint), 36 | ExtraLabels: a.ExtraLabels, 37 | ExtraHeaders: headerValuesToHTTP(a.ExtraHeaders), 38 | }) 39 | if err != nil { 40 | return fmt.Errorf("failed to configure proxy handler: %w", err) 41 | } 42 | 43 | srv := &http.Server{ 44 | Addr: a.ListenAddress, 45 | Handler: handler, 46 | ReadHeaderTimeout: time.Minute, 47 | } 48 | defer context.AfterFunc(ctx, func() { 49 | _ = srv.Close() 50 | })() 51 | 52 | slog.Info("Listening", "address", a.ListenAddress) 53 | 54 | if err := srv.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { 55 | return fmt.Errorf("failed to serve: %w", err) 56 | } 57 | 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /functests/testscripts/cgo-test.txtar: -------------------------------------------------------------------------------- 1 | [!cgo] skip 'CGO is disabled' 2 | env COMPILER_EXPRESSION='(cgo).*main\.go' 3 | 4 | # control run 5 | go build -x -o test_bin$exe . 6 | stderr ${COMPILER_EXPRESSION} 7 | exec ./test_bin$exe 8 | cmp stdout expected_output.txt 9 | rm test_bin$exe 10 | 11 | # run with cache 12 | mkdir -p $WORK/disk-cache 13 | 14 | # if short testing enable, use only disk-backed cache 15 | [!short] env CACHEPROG_REMOTE_STORAGE_TYPE=s3 CACHEPROG_S3_PREFIX=tests/cgo-test 16 | # cleanup path for consistency 17 | [!short] env MC_FULLPATH=${MINIO_ALIAS}/${CACHEPROG_S3_BUCKET}/${CACHEPROG_S3_PREFIX} 18 | [!short] mc rm --recursive --force ${MC_FULLPATH} 19 | 20 | # run with cacheprog first time 21 | env CACHEPROG_ROOT_DIRECTORY=$WORK/disk-cache 22 | set-cacheprog GOCACHEPROG 23 | go build -x -o test_bin$exe . 24 | stderr ${COMPILER_EXPRESSION} 25 | exec ./test_bin$exe 26 | cmp stdout expected_output.txt 27 | rm test_bin$exe 28 | 29 | # ensure that minio bucket is not empty 30 | [!short] mc du ${MC_FULLPATH} 31 | [!short] stdout ${CACHEPROG_S3_BUCKET}/${CACHEPROG_S3_PREFIX} 32 | [!short] ! stdout '^0B' 33 | [!short] ! stdout '(^|\W)0 object' 34 | 35 | # run with cache second time to ensure we're getting consistent result, clean disk cache 36 | rm ${CACHEPROG_ROOT_DIRECTORY} 37 | go build -x -o test_bin$exe . 38 | # if s3 used we should not run compiler, if s3 is not used we should 39 | [!short] ! stderr ${COMPILER_EXPRESSION} 40 | [short] stderr ${COMPILER_EXPRESSION} 41 | exec ./test_bin$exe 42 | cmp stdout expected_output.txt 43 | rm test_bin$exe 44 | 45 | -- hello.h -- 46 | #ifndef HELLO_H 47 | #define HELLO_H 48 | const char * get_hello(); 49 | #endif 50 | 51 | -- hello.c -- 52 | #include "hello.h" 53 | 54 | const char * get_hello() { 55 | return "Hello from C in another file!"; 56 | } 57 | 58 | -- main.go -- 59 | package main 60 | 61 | // #include "hello.h" 62 | import "C" 63 | 64 | import "fmt" 65 | 66 | func main() { 67 | fmt.Println(C.GoString(C.get_hello())) 68 | } 69 | 70 | -- go.mod -- 71 | module cgotest 72 | 73 | go 1.25 74 | 75 | -- expected_output.txt -- 76 | Hello from C in another file! 77 | -------------------------------------------------------------------------------- /functests/testscripts/golangci-lint-v1-test.txtar: -------------------------------------------------------------------------------- 1 | # this is officially not recommended way of installing golangci-lint 2 | # but it's choosen because it's easiest and most portable way to get it for testing purposes 3 | install-go-binary github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.8 ./bin 4 | exec ./bin/golangci-lint$exe --version 5 | stdout 'v1\.64\.8(.*)h1:y5TdeVidMtBGG32zgSC7ZXTFNHrsJkDnpO4ItB3Am\+I=' 6 | 7 | # control run 8 | env GOLANGCI_LINT_CACHE=$WORK/.golangci-lint-cache 9 | exec ./bin/golangci-lint$exe run ./... 10 | ! stderr '(error|fail)' 11 | rm ${GOLANGCI_LINT_CACHE} 12 | 13 | # run with cache 14 | mkdir -p $WORK/disk-cache 15 | env CACHEPROG_ROOT_DIRECTORY=$WORK/disk-cache 16 | 17 | # if short testing enable, use only disk-backed cache 18 | [!short] env CACHEPROG_REMOTE_STORAGE_TYPE=s3 CACHEPROG_S3_PREFIX=tests/golangci-lint-v1-test 19 | # cleanup path for consistency 20 | [!short] env MC_FULLPATH=${MINIO_ALIAS}/${CACHEPROG_S3_BUCKET}/${CACHEPROG_S3_PREFIX} 21 | [!short] mc rm --recursive --force ${MC_FULLPATH} 22 | 23 | # run with cacheprog first time 24 | set-cacheprog GOCACHEPROG 25 | set-cacheprog GOLANGCI_LINT_CACHEPROG 26 | exec ./bin/golangci-lint$exe run ./... 27 | # we have no way to actually determine if cache entries has been used or not, so just rely on cacheprog log output 28 | stderr 'Starting cacheprog' 29 | ! stderr '(error|fail)' 30 | 31 | # ensure that minio bucket is not empty 32 | [!short] mc du ${MC_FULLPATH} 33 | [!short] stdout ${CACHEPROG_S3_BUCKET}/${CACHEPROG_S3_PREFIX} 34 | [!short] ! stdout '^0B' 35 | [!short] ! stdout '(^|\W)0 object' 36 | 37 | # run with cacheprog second time, clean disk storage 38 | rm ${CACHEPROG_ROOT_DIRECTORY} 39 | exec ./bin/golangci-lint$exe run ./... 40 | stderr 'Starting cacheprog' 41 | ! stderr '(error|fail)' 42 | 43 | -- main.go -- 44 | package main 45 | 46 | import "fmt" 47 | 48 | func main() { 49 | fmt.Println("Hello world!") 50 | } 51 | 52 | -- go.mod -- 53 | module linttest 54 | 55 | go 1.25 56 | 57 | -- golangci-lint.yml -- 58 | linters: 59 | enable-all: true 60 | disable: 61 | - forbidigo 62 | - gofmt 63 | - goimports 64 | - gofumpt 65 | -------------------------------------------------------------------------------- /internal/infra/storage/http.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "cmp" 5 | "context" 6 | "fmt" 7 | "io" 8 | "log/slog" 9 | "net/http" 10 | 11 | "github.com/platacard/cacheprog/internal/app/cacheprog" 12 | ) 13 | 14 | type HTTP struct { 15 | client *http.Client 16 | endpoint string 17 | extraHeaders http.Header 18 | } 19 | 20 | func NewHTTP(client *http.Client, endpoint string, extraHeaders http.Header) *HTTP { 21 | return &HTTP{ 22 | client: cmp.Or(client, http.DefaultClient), 23 | endpoint: endpoint, 24 | extraHeaders: extraHeaders, 25 | } 26 | } 27 | 28 | func ConfigureHTTP(baseURL string, extraHeaders http.Header) (*HTTP, error) { 29 | if baseURL == "" { 30 | return nil, fmt.Errorf("base URL is required") 31 | } 32 | 33 | slog.Info("Configuring HTTP storage with base URL", slog.String("baseURL", baseURL)) 34 | 35 | return NewHTTP(http.DefaultClient, baseURL, extraHeaders), nil 36 | } 37 | 38 | func (h *HTTP) Get(ctx context.Context, request *cacheprog.GetRequest) (*cacheprog.GetResponse, error) { 39 | req, err := NewGetRequest(ctx, h.endpoint, request) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | for k, vs := range h.extraHeaders { 45 | for _, v := range vs { 46 | req.Header.Add(k, v) 47 | } 48 | } 49 | 50 | resp, err := h.client.Do(req) //nolint:bodyclose // closed in upper layer 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | return ParseGetResponse(resp) 56 | } 57 | 58 | func (h *HTTP) Put(ctx context.Context, request *cacheprog.PutRequest) (*cacheprog.PutResponse, error) { 59 | req, err := NewPutRequest(ctx, h.endpoint, request) 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | for k, vs := range h.extraHeaders { 65 | for _, v := range vs { 66 | req.Header.Add(k, v) 67 | } 68 | } 69 | 70 | resp, err := h.client.Do(req) 71 | if resp != nil { 72 | defer resp.Body.Close() 73 | } 74 | if err != nil { 75 | return nil, err 76 | } 77 | 78 | if resp.StatusCode != http.StatusOK { 79 | body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) 80 | return nil, fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(body)) 81 | } 82 | 83 | return &cacheprog.PutResponse{}, nil 84 | } 85 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | before: 4 | hooks: 5 | - go mod tidy 6 | - go mod download 7 | 8 | builds: 9 | - id: cacheprog 10 | main: ./cmd/cacheprog 11 | binary: cacheprog 12 | env: 13 | - CGO_ENABLED=0 14 | goos: 15 | - linux 16 | - darwin 17 | - windows 18 | goarch: 19 | - amd64 20 | - arm64 21 | - arm 22 | ignore: 23 | - goos: windows 24 | goarch: arm 25 | ldflags: 26 | - -s -w 27 | - -X 'main.Version=v{{ .Version }}' 28 | flags: 29 | - -trimpath 30 | 31 | archives: 32 | - id: cacheprog 33 | formats: [ 'tar.gz' ] 34 | name_template: >- 35 | {{ .ProjectName }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }} 36 | format_overrides: 37 | - goos: windows 38 | formats: [ 'zip' ] 39 | files: 40 | - LICENSE 41 | - README.md 42 | 43 | dockers_v2: 44 | - images: 45 | - "ghcr.io/{{ .Env.GITHUB_REPOSITORY_OWNER }}/cacheprog" 46 | tags: 47 | - "v{{ .Version }}" 48 | - "v{{ .Major }}.{{ .Minor }}" 49 | - "latest" 50 | dockerfile: build/cacheprog/goreleaser.Dockerfile 51 | labels: 52 | "org.opencontainers.image.created": "{{ .Date }}" 53 | "org.opencontainers.image.title": "{{ .ProjectName }}" 54 | "org.opencontainers.image.revision": "{{ .FullCommit }}" 55 | "org.opencontainers.image.version": "v{{ .Version }}" 56 | "org.opencontainers.image.source": "{{ .GitURL }}" 57 | 58 | checksum: 59 | name_template: 'checksums.txt' 60 | 61 | changelog: 62 | sort: asc 63 | use: github 64 | filters: 65 | exclude: 66 | - '^docs:' 67 | - '^test:' 68 | - '^ci:' 69 | - '^chore:' 70 | - Merge pull request 71 | - Merge remote-tracking branch 72 | - Merge branch 73 | groups: 74 | - title: 'Features' 75 | regexp: '^.*?feat(\([[:word:]]+\))??!?:.+$' 76 | order: 0 77 | - title: 'Bug fixes' 78 | regexp: '^.*?fix(\([[:word:]]+\))??!?:.+$' 79 | order: 1 80 | - title: 'Performance improvements' 81 | regexp: '^.*?perf(\([[:word:]]+\))??!?:.+$' 82 | order: 2 83 | - title: 'Refactors' 84 | regexp: '^.*?refactor(\([[:word:]]+\))??!?:.+$' 85 | order: 3 86 | - title: 'Other' 87 | order: 999 88 | 89 | -------------------------------------------------------------------------------- /functests/testscripts/golangci-lint-v2-test.txtar: -------------------------------------------------------------------------------- 1 | # this is officially not recommended way of installing golangci-lint 2 | # but it's choosen because it's easiest and most portable way to get it for testing purposes 3 | install-go-binary github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.4.0 ./bin 4 | exec ./bin/golangci-lint$exe --version 5 | stdout 'version 2\.4\.0(.*)h1:qz6O6vr7kVzXJqyvHjHSz5fA3D\+PM8v96QU5gxZCNWM=' 6 | 7 | # this needed to eliminate error "failed to create modcache index dir" from logs 8 | # https://github.com/golangci/golangci-lint/issues/6037 9 | env HOME=$WORK/.testhome 10 | mkdir -p $HOME 11 | 12 | # control run 13 | env GOLANGCI_LINT_CACHE=$WORK/.golangci-lint-cache 14 | exec ./bin/golangci-lint$exe run ./... 15 | ! stderr '(error|fail)' 16 | rm ${GOLANGCI_LINT_CACHE} 17 | 18 | # run with cache 19 | mkdir -p $WORK/disk-cache 20 | env CACHEPROG_ROOT_DIRECTORY=$WORK/disk-cache 21 | 22 | # if short testing enable, use only disk-backed cache 23 | [!short] env CACHEPROG_REMOTE_STORAGE_TYPE=s3 CACHEPROG_S3_PREFIX=tests/golangci-lint-v2-test 24 | # cleanup path for consistency 25 | [!short] env MC_FULLPATH=${MINIO_ALIAS}/${CACHEPROG_S3_BUCKET}/${CACHEPROG_S3_PREFIX} 26 | [!short] mc rm --recursive --force ${MC_FULLPATH} 27 | 28 | # run with cacheprog first time 29 | set-cacheprog GOCACHEPROG 30 | set-cacheprog GOLANGCI_LINT_CACHEPROG 31 | exec ./bin/golangci-lint$exe run ./... 32 | # we have no way to actually determine if cache entries has been used or not, so just rely on cacheprog log output 33 | stderr 'Starting cacheprog' 34 | ! stderr '(error|fail)' 35 | 36 | # ensure that minio bucket is not empty 37 | [!short] mc du ${MC_FULLPATH} 38 | [!short] stdout ${CACHEPROG_S3_BUCKET}/${CACHEPROG_S3_PREFIX} 39 | [!short] ! stdout '^0B' 40 | [!short] ! stdout '(^|\W)0 object' 41 | 42 | # run with cacheprog second time, clean disk storage 43 | rm ${CACHEPROG_ROOT_DIRECTORY} 44 | exec ./bin/golangci-lint$exe run ./... 45 | stderr 'Starting cacheprog' 46 | ! stderr '(error|fail)' 47 | 48 | -- main.go -- 49 | package main 50 | 51 | import "fmt" 52 | 53 | func main() { 54 | fmt.Println("Hello world!") 55 | } 56 | 57 | -- go.mod -- 58 | module linttest 59 | 60 | go 1.25 61 | 62 | -- golangci-lint.yml -- 63 | version: "2" 64 | linters: 65 | enable-all: true 66 | disable: 67 | - forbidigo 68 | - gofmt 69 | - goimports 70 | - gofumpt 71 | -------------------------------------------------------------------------------- /pkg/httpstorage/mocks_world_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: server.go 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -destination=mocks_world_test.go -package=httpstorage -source=server.go 7 | // 8 | 9 | // Package httpstorage is a generated GoMock package. 10 | package httpstorage 11 | 12 | import ( 13 | context "context" 14 | reflect "reflect" 15 | 16 | gomock "go.uber.org/mock/gomock" 17 | ) 18 | 19 | // MockRemoteStorage is a mock of RemoteStorage interface. 20 | type MockRemoteStorage struct { 21 | ctrl *gomock.Controller 22 | recorder *MockRemoteStorageMockRecorder 23 | isgomock struct{} 24 | } 25 | 26 | // MockRemoteStorageMockRecorder is the mock recorder for MockRemoteStorage. 27 | type MockRemoteStorageMockRecorder struct { 28 | mock *MockRemoteStorage 29 | } 30 | 31 | // NewMockRemoteStorage creates a new mock instance. 32 | func NewMockRemoteStorage(ctrl *gomock.Controller) *MockRemoteStorage { 33 | mock := &MockRemoteStorage{ctrl: ctrl} 34 | mock.recorder = &MockRemoteStorageMockRecorder{mock} 35 | return mock 36 | } 37 | 38 | // EXPECT returns an object that allows the caller to indicate expected use. 39 | func (m *MockRemoteStorage) EXPECT() *MockRemoteStorageMockRecorder { 40 | return m.recorder 41 | } 42 | 43 | // Get mocks base method. 44 | func (m *MockRemoteStorage) Get(ctx context.Context, request *GetRequest) (*GetResponse, error) { 45 | m.ctrl.T.Helper() 46 | ret := m.ctrl.Call(m, "Get", ctx, request) 47 | ret0, _ := ret[0].(*GetResponse) 48 | ret1, _ := ret[1].(error) 49 | return ret0, ret1 50 | } 51 | 52 | // Get indicates an expected call of Get. 53 | func (mr *MockRemoteStorageMockRecorder) Get(ctx, request any) *gomock.Call { 54 | mr.mock.ctrl.T.Helper() 55 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockRemoteStorage)(nil).Get), ctx, request) 56 | } 57 | 58 | // Put mocks base method. 59 | func (m *MockRemoteStorage) Put(ctx context.Context, request *PutRequest) (*PutResponse, error) { 60 | m.ctrl.T.Helper() 61 | ret := m.ctrl.Call(m, "Put", ctx, request) 62 | ret0, _ := ret[0].(*PutResponse) 63 | ret1, _ := ret[1].(error) 64 | return ret0, ret1 65 | } 66 | 67 | // Put indicates an expected call of Put. 68 | func (mr *MockRemoteStorageMockRecorder) Put(ctx, request any) *gomock.Call { 69 | mr.mock.ctrl.T.Helper() 70 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockRemoteStorage)(nil).Put), ctx, request) 71 | } 72 | -------------------------------------------------------------------------------- /internal/app/proxy/handler_test.go: -------------------------------------------------------------------------------- 1 | package proxy 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "net/http/httptest" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/VictoriaMetrics/metrics" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | func TestHandler_Metrics(t *testing.T) { 18 | const ( 19 | emptyValueLabel = "empty_value" 20 | 21 | plainValueLabel = "plain_value" 22 | plainValue = "value" 23 | 24 | encodedValueLabel = "encoded_value" 25 | encodedValue = "encoded/value" // not path-safe 26 | 27 | extraHeader = "X-Extra-Header" 28 | extraValue = "extra_value" 29 | ) 30 | 31 | targetServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 32 | if r.Method != http.MethodPost { 33 | http.Error(w, "expected POST request", http.StatusMethodNotAllowed) 34 | return 35 | } 36 | 37 | if r.Header.Get(extraHeader) != extraValue { 38 | http.Error(w, "expected extra header", http.StatusBadRequest) 39 | return 40 | } 41 | 42 | if !strings.HasPrefix(r.URL.Path, "/metrics/job") { 43 | http.Error(w, "expected metrics path", http.StatusBadRequest) 44 | return 45 | } 46 | 47 | if !strings.Contains(r.URL.Path, fmt.Sprintf("/%s@base64/=", emptyValueLabel)) { 48 | http.Error(w, "expected empty value label", http.StatusBadRequest) 49 | return 50 | } 51 | 52 | if !strings.Contains(r.URL.Path, fmt.Sprintf("/%s/%s", plainValueLabel, plainValue)) { 53 | http.Error(w, "expected plain value label", http.StatusBadRequest) 54 | return 55 | } 56 | 57 | if !strings.Contains(r.URL.Path, fmt.Sprintf("/%s@base64/%s", encodedValueLabel, base64.RawURLEncoding.EncodeToString([]byte(encodedValue)))) { 58 | http.Error(w, "expected encoded value label", http.StatusBadRequest) 59 | return 60 | } 61 | 62 | _, err := io.ReadAll(r.Body) 63 | require.NoError(t, err) 64 | w.WriteHeader(http.StatusNoContent) 65 | })) 66 | t.Cleanup(targetServer.Close) 67 | 68 | handler, err := NewHandler(nil, MetricsConfig{ 69 | Endpoint: targetServer.URL, 70 | ExtraLabels: map[string]string{ 71 | emptyValueLabel: "", 72 | plainValueLabel: plainValue, 73 | encodedValueLabel: encodedValue, 74 | }, 75 | ExtraHeaders: http.Header{ 76 | extraHeader: {extraValue}, 77 | }, 78 | }) 79 | require.NoError(t, err) 80 | 81 | metricServer := httptest.NewServer(handler) 82 | t.Cleanup(metricServer.Close) 83 | 84 | require.NoError(t, metrics.PushMetrics(context.Background(), metricServer.URL+"/metricsproxy", true, &metrics.PushOptions{ 85 | Method: "POST", 86 | })) 87 | } 88 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/platacard/cacheprog 2 | 3 | go 1.25 4 | 5 | require ( 6 | github.com/KimMachineGun/automemlimit v0.7.5 7 | github.com/VictoriaMetrics/metrics v1.40.2 8 | github.com/alexflint/go-arg v1.6.0 9 | github.com/aws/aws-sdk-go-v2 v1.40.0 10 | github.com/aws/aws-sdk-go-v2/config v1.32.1 11 | github.com/aws/aws-sdk-go-v2/credentials v1.19.1 12 | github.com/aws/aws-sdk-go-v2/service/s3 v1.92.0 13 | github.com/aws/smithy-go v1.23.2 14 | github.com/felixge/fgprof v0.9.5 15 | github.com/klauspost/compress v1.18.1 16 | github.com/stretchr/testify v1.11.1 17 | github.com/valyala/bytebufferpool v1.0.0 18 | github.com/valyala/fasttemplate v1.2.2 19 | go.uber.org/mock v0.6.0 20 | ) 21 | 22 | require ( 23 | github.com/alexflint/go-scalar v1.2.0 // indirect 24 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect 25 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 // indirect 26 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 // indirect 27 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 // indirect 28 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect 29 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 // indirect 30 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect 31 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 // indirect 32 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 // indirect 33 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 // indirect 34 | github.com/aws/aws-sdk-go-v2/service/signin v1.0.1 // indirect 35 | github.com/aws/aws-sdk-go-v2/service/sso v1.30.4 // indirect 36 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.9 // indirect 37 | github.com/aws/aws-sdk-go-v2/service/sts v1.41.1 // indirect 38 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 39 | github.com/google/go-cmp v0.7.0 // indirect 40 | github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect 41 | github.com/kr/pretty v0.3.1 // indirect 42 | github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect 43 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 44 | github.com/rogpeppe/go-internal v1.14.1 // indirect 45 | github.com/valyala/fastrand v1.1.0 // indirect 46 | github.com/valyala/histogram v1.2.0 // indirect 47 | golang.org/x/mod v0.27.0 // indirect 48 | golang.org/x/sync v0.16.0 // indirect 49 | golang.org/x/sys v0.35.0 // indirect 50 | golang.org/x/tools v0.36.0 // indirect 51 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 52 | gopkg.in/yaml.v3 v3.0.1 // indirect 53 | ) 54 | 55 | tool go.uber.org/mock/mockgen 56 | -------------------------------------------------------------------------------- /functests/testscripts/golangci-lint-v2-cgo-test.txtar: -------------------------------------------------------------------------------- 1 | [!cgo] skip 'CGO is disabled' 2 | 3 | # this is officially not recommended way of installing golangci-lint 4 | # but it's choosen because it's easiest and most portable way to get it for testing purposes 5 | install-go-binary github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.4.0 ./bin 6 | exec ./bin/golangci-lint$exe --version 7 | stdout 'version 2\.4\.0(.*)h1:qz6O6vr7kVzXJqyvHjHSz5fA3D\+PM8v96QU5gxZCNWM=' 8 | 9 | # this needed to eliminate error "failed to create modcache index dir" from logs 10 | # https://github.com/golangci/golangci-lint/issues/6037 11 | env HOME=$WORK/.testhome 12 | mkdir -p $HOME 13 | 14 | # control run 15 | env GOLANGCI_LINT_CACHE=$WORK/.golangci-lint-cache 16 | exec ./bin/golangci-lint$exe run ./... 17 | ! stderr '(error|fail)' 18 | rm ${GOLANGCI_LINT_CACHE} 19 | 20 | # run with cache 21 | mkdir -p $WORK/disk-cache 22 | env CACHEPROG_ROOT_DIRECTORY=$WORK/disk-cache 23 | 24 | # if short testing enable, use only disk-backed cache 25 | [!short] env CACHEPROG_REMOTE_STORAGE_TYPE=s3 CACHEPROG_S3_PREFIX=tests/golangci-lint-v2-cgo-test 26 | # cleanup path for consistency 27 | [!short] env MC_FULLPATH=${MINIO_ALIAS}/${CACHEPROG_S3_BUCKET}/${CACHEPROG_S3_PREFIX} 28 | [!short] mc rm --recursive --force ${MC_FULLPATH} 29 | 30 | # run with cacheprog first time 31 | set-cacheprog GOCACHEPROG 32 | set-cacheprog GOLANGCI_LINT_CACHEPROG 33 | exec ./bin/golangci-lint$exe run ./... 34 | # we have no way to actually determine if cache entries has been used or not, so just rely on cacheprog log output 35 | stderr 'Starting cacheprog' 36 | ! stderr '(error|fail)' 37 | 38 | # ensure that minio bucket is not empty 39 | [!short] mc du ${MC_FULLPATH} 40 | [!short] stdout ${CACHEPROG_S3_BUCKET}/${CACHEPROG_S3_PREFIX} 41 | [!short] ! stdout '^0B' 42 | [!short] ! stdout '(^|\W)0 object' 43 | 44 | # run with cacheprog second time, clean disk storage 45 | rm ${CACHEPROG_ROOT_DIRECTORY} 46 | exec ./bin/golangci-lint$exe run ./... 47 | stderr 'Starting cacheprog' 48 | ! stderr '(error|fail)' 49 | 50 | -- hello.h -- 51 | #ifndef HELLO_H 52 | #define HELLO_H 53 | const char * get_hello(); 54 | #endif 55 | 56 | -- hello.c -- 57 | #include "hello.h" 58 | 59 | const char * get_hello() { 60 | return "Hello from C in another file!"; 61 | } 62 | 63 | -- main.go -- 64 | package main 65 | 66 | // #include "hello.h" 67 | import "C" 68 | 69 | import "fmt" 70 | 71 | func main() { 72 | fmt.Println(C.GoString(C.get_hello())) 73 | } 74 | 75 | -- go.mod -- 76 | module lintcgotest 77 | 78 | go 1.25 79 | 80 | -- golangci-lint.yml -- 81 | version: "2" 82 | linters: 83 | enable-all: true 84 | disable: 85 | - forbidigo 86 | - gofmt 87 | - goimports 88 | - gofumpt 89 | -------------------------------------------------------------------------------- /internal/infra/cacheproto/reader.go: -------------------------------------------------------------------------------- 1 | package cacheproto 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/base64" 7 | "encoding/json" 8 | "errors" 9 | "fmt" 10 | "io" 11 | ) 12 | 13 | type Reader struct { 14 | br *bufio.Reader 15 | } 16 | 17 | func NewReader(r io.Reader) *Reader { 18 | return &Reader{ 19 | br: bufio.NewReader(r), 20 | } 21 | } 22 | 23 | func (r *Reader) ReadRequest() (*Request, error) { 24 | // first line - json request object 25 | 26 | line, err := r.br.ReadBytes('\n') 27 | if err != nil { 28 | return nil, fmt.Errorf("read line: %w", err) 29 | } 30 | 31 | var request Request 32 | if err = json.Unmarshal(line, &request); err != nil { 33 | return nil, fmt.Errorf("unmarshal request: %w", err) 34 | } 35 | 36 | // read empty line 37 | if _, err = r.br.ReadSlice('\n'); err != nil { 38 | return nil, fmt.Errorf("read empty line: %w", err) 39 | } 40 | 41 | if request.Command != CmdPut { 42 | return &request, nil 43 | } 44 | 45 | // for "put" command - read body, base64 encoded json string 46 | // pick next character - must be '"' if body is present 47 | if request.BodySize == 0 { 48 | request.Body = bytes.NewReader(nil) 49 | return &request, nil 50 | } 51 | 52 | bodyBegin, err := r.br.ReadByte() 53 | if err != nil { 54 | return nil, fmt.Errorf("read body begin: %w", err) 55 | } 56 | 57 | if bodyBegin != '"' { 58 | return nil, fmt.Errorf("unexpected body begin: %q", bodyBegin) 59 | } 60 | 61 | request.Body = newBodyReader(r.br, request.BodySize) 62 | 63 | return &request, nil 64 | } 65 | 66 | type bodyReader struct { 67 | br *bufio.Reader 68 | decoder io.Reader 69 | 70 | atEOF bool 71 | } 72 | 73 | func newBodyReader(br *bufio.Reader, size int64) *bodyReader { 74 | encoding := base64.StdEncoding 75 | return &bodyReader{ 76 | br: br, 77 | decoder: base64.NewDecoder( 78 | encoding, 79 | &io.LimitedReader{ 80 | R: br, 81 | N: int64(encoding.EncodedLen(int(size))), // prevent buffering in decoder since we know the size 82 | }, 83 | ), 84 | } 85 | } 86 | 87 | var bodyEnd = [...]byte{'"', '\n'} 88 | 89 | func (b *bodyReader) Read(p []byte) (n int, err error) { 90 | if b.atEOF { 91 | return 0, io.EOF 92 | } 93 | 94 | n, err = b.decoder.Read(p) 95 | if !errors.Is(err, io.EOF) { 96 | return n, err 97 | } 98 | 99 | b.atEOF = true 100 | 101 | // validate that the body ends with '"' and '\n' 102 | // read the rest of the body 103 | 104 | var endData [len(bodyEnd)]byte 105 | if _, err = io.ReadFull(b.br, endData[:]); err != nil { 106 | return n, fmt.Errorf("read body end: %w", err) 107 | } 108 | 109 | if endData != bodyEnd { 110 | return n, fmt.Errorf("unexpected body end: %q", endData) 111 | } 112 | 113 | return n, nil 114 | } 115 | -------------------------------------------------------------------------------- /pkg/httpstorage/server.go: -------------------------------------------------------------------------------- 1 | package httpstorage 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net/http" 7 | 8 | "github.com/platacard/cacheprog/internal/app/cacheprog" 9 | "github.com/platacard/cacheprog/internal/infra/storage" 10 | ) 11 | 12 | //go:generate go tool go.uber.org/mock/mockgen -destination=mocks_world_test.go -package=$GOPACKAGE -source=$GOFILE 13 | 14 | type ( 15 | GetRequest = cacheprog.GetRequest 16 | GetResponse = cacheprog.GetResponse 17 | PutRequest = cacheprog.PutRequest 18 | PutResponse = cacheprog.PutResponse 19 | 20 | RemoteStorage interface { 21 | // Get fetches object from remote storage. Returns [ErrNotFound] if object was not found. 22 | Get(ctx context.Context, request *GetRequest) (*GetResponse, error) 23 | 24 | // Put pushes object to remote storage. 25 | Put(ctx context.Context, request *PutRequest) (*PutResponse, error) 26 | } 27 | ) 28 | 29 | var ( 30 | // ErrNotFound is returned when object is not found in remote storage. 31 | ErrNotFound = cacheprog.ErrNotFound 32 | ) 33 | 34 | // NewServer creates a http handler meant to be used as a 'http' remote storage for cacheprog. 35 | // It has 2 endpoints: 36 | // - PUT /cache/{actionID} - pushes object to remote storage, returns 200 on success, 400 on bad request, 500 on error 37 | // - GET /cache/{actionID} - fetches object from remote storage, returns 200 on success, 400 on bad request, 404 on not found object, 500 on error 38 | func NewServer(remoteStorage RemoteStorage) http.Handler { 39 | mux := http.NewServeMux() 40 | mux.HandleFunc("PUT /cache/{actionID}", func(w http.ResponseWriter, r *http.Request) { 41 | ctx := r.Context() 42 | 43 | putRequest, err := storage.ParsePutRequest(r) 44 | if err != nil { 45 | http.Error(w, err.Error(), http.StatusBadRequest) 46 | return 47 | } 48 | 49 | if remoteStorage == nil { 50 | w.WriteHeader(http.StatusOK) 51 | return 52 | } 53 | 54 | _, err = remoteStorage.Put(ctx, putRequest) 55 | if err != nil { 56 | http.Error(w, err.Error(), http.StatusInternalServerError) 57 | return 58 | } 59 | 60 | w.WriteHeader(http.StatusOK) 61 | }) 62 | mux.HandleFunc("GET /cache/{actionID}", func(w http.ResponseWriter, r *http.Request) { 63 | ctx := r.Context() 64 | 65 | getRequest, err := storage.ParseGetRequest(r) 66 | if err != nil { 67 | http.Error(w, err.Error(), http.StatusBadRequest) 68 | return 69 | } 70 | 71 | if remoteStorage == nil { 72 | w.WriteHeader(http.StatusNotFound) 73 | return 74 | } 75 | 76 | getResponse, err := remoteStorage.Get(ctx, getRequest) 77 | switch { 78 | case errors.Is(err, nil): 79 | case errors.Is(err, ErrNotFound): 80 | w.WriteHeader(http.StatusNotFound) 81 | return 82 | default: 83 | http.Error(w, err.Error(), http.StatusInternalServerError) 84 | return 85 | } 86 | 87 | if err = storage.WriteGetResponse(w, getResponse); err != nil { 88 | http.Error(w, err.Error(), http.StatusInternalServerError) 89 | return 90 | } 91 | }) 92 | 93 | return mux 94 | } 95 | -------------------------------------------------------------------------------- /functests/testscripts/proxy-mode-test.txtar: -------------------------------------------------------------------------------- 1 | # run server 2 | set-cacheprog CACHEPROG_PATH 3 | [!short] env CACHEPROG_REMOTE_STORAGE_TYPE=s3 CACHEPROG_S3_PREFIX=tests/server-mode-test 4 | [!short] env MC_FULLPATH=${MINIO_ALIAS}/${CACHEPROG_S3_BUCKET}/${CACHEPROG_S3_PREFIX} 5 | [!short] mc rm --recursive --force ${MC_FULLPATH} 6 | allocate-port CACHEPROG_SERVER_PORT 7 | env CACHEPROG_PROXY_LISTEN_ADDRESS=localhost:${CACHEPROG_SERVER_PORT} 8 | recording-server start CACHEPROG_METRICS_PROXY_ENDPOINT 9 | env CACHEPROG_METRICS_PROXY_EXTRA_HEADERS=X-Extra-Header:some CACHEPROG_METRICS_PROXY_EXTRA_LABELS=job=proxytest,labeltoproxy=valuetoproxy 10 | exec ${CACHEPROG_PATH} proxy &proxy& 11 | 12 | # prepare client cacheprog environment 13 | mkdir -p $WORK/disk-cache 14 | env CACHEPROG_ROOT_DIRECTORY=$WORK/disk-cache 15 | [!short] env CACHEPROG_REMOTE_STORAGE_TYPE=disabled 16 | env CACHEPROG_METRICS_PUSH_EXTRA_LABELS= CACHEPROG_METRICS_PUSH_METHOD=PUT CACHEPROG_USE_VM_HISTOGRAMS=1 17 | env CACHEPROG_REMOTE_STORAGE_TYPE=http CACHEPROG_HTTP_STORAGE_BASE_URL=http://localhost:${CACHEPROG_SERVER_PORT}/ 18 | env CACHEPROG_METRICS_PUSH_ENDPOINT=http://localhost:${CACHEPROG_SERVER_PORT}/metricsproxy 19 | set-cacheprog GOCACHEPROG 20 | 21 | # run first time 22 | env COMPILER_EXPRESSION='(compile|gccgo)( |\.exe).*main\.go' 23 | env CACHEPROG_METRICS_PUSH_EXTRA_HEADERS=X-Test-Record-Key:metricproxy_1,X-Some-Header:hdr 24 | go build -x -o test_bin$exe ./main.go 25 | stderr ${COMPILER_EXPRESSION} 26 | ! stderr 'Configuring S3 storage' 27 | exec ./test_bin$exe 28 | cmp stdout expected_output.txt 29 | rm test_bin$exe 30 | 31 | # ensure that minio bucket is not empty 32 | [!short] mc du ${MC_FULLPATH} 33 | [!short] stdout ${CACHEPROG_S3_BUCKET}/${CACHEPROG_S3_PREFIX} 34 | [!short] ! stdout '^0B' 35 | [!short] ! stdout '(^|\W)0 object' 36 | 37 | # run with cache second time to ensure we're getting consistent result, clean disk cache 38 | rm ${CACHEPROG_ROOT_DIRECTORY} 39 | env CACHEPROG_METRICS_PUSH_EXTRA_HEADERS=X-Test-Record-Key:metricproxy_2,X-Some-Header:hdr2 40 | go build -x -o test_bin$exe ./main.go 41 | # if s3 used we should not run compiler, if s3 is not used we should 42 | [!short] ! stderr ${COMPILER_EXPRESSION} 43 | [short] stderr ${COMPILER_EXPRESSION} 44 | ! stderr 'Configuring S3 storage' 45 | exec ./test_bin$exe 46 | cmp stdout expected_output.txt 47 | rm test_bin$exe 48 | 49 | # terminate proxy and check if everything has been propagated 50 | kill -INT proxy 51 | wait proxy 52 | [!short] stderr 'Configuring S3 storage' 53 | 54 | # check if metrics has been propagated 55 | recording-server get metricproxy_1 path 56 | # this is prometheus-format path, accepted both by Prometheus and VictoriaMetrics 57 | stdout '/metrics/job/proxytest/labeltoproxy/valuetoproxy' 58 | recording-server get metricproxy_1 headers 59 | stdout 'X-Extra-Header: some' 60 | stdout 'X-Some-Header: hdr' 61 | recording-server get metricproxy_1 body 62 | stdout 'cacheprog_overall_run_time_ms' 63 | 64 | recording-server get metricproxy_2 path 65 | stdout '/metrics/job/proxytest/labeltoproxy/valuetoproxy' 66 | recording-server get metricproxy_2 headers 67 | stdout 'X-Extra-Header: some' 68 | stdout 'X-Some-Header: hdr2' 69 | recording-server get metricproxy_2 body 70 | stdout 'cacheprog_overall_run_time_ms' 71 | 72 | -- main.go -- 73 | package main 74 | 75 | import "fmt" 76 | 77 | func main() { 78 | fmt.Println("Hello world!") 79 | } 80 | 81 | -- expected_output.txt -- 82 | Hello world! 83 | -------------------------------------------------------------------------------- /internal/infra/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "strings" 8 | "sync/atomic" 9 | "time" 10 | 11 | "github.com/VictoriaMetrics/metrics" 12 | ) 13 | 14 | const metricPrefix = "cacheprog_" 15 | 16 | var ( 17 | enableVMHistograms atomic.Bool 18 | ) 19 | 20 | type PushConfig struct { 21 | Endpoint string // metrics endpoint, metrics will be pushed if provided 22 | ExtraLabels map[string]string // extra labels to be added to each metric 23 | ExtraHeaders http.Header // extra headers to be added to each request 24 | Method string // HTTP method to use for sending metrics 25 | } 26 | 27 | // EnableVMHistograms enables victoriametrics-style histograms instead of prometheus-style if 'true' is passed. 28 | func EnableVMHistograms(enable bool) { 29 | enableVMHistograms.Store(enable) 30 | } 31 | 32 | type histogram interface { 33 | Update(float64) 34 | UpdateDuration(time.Time) 35 | } 36 | 37 | func getOrCreateHistogram(name string) histogram { 38 | if enableVMHistograms.Load() { 39 | return metrics.GetOrCreateHistogram(name) 40 | } 41 | return metrics.GetOrCreatePrometheusHistogram(name) 42 | } 43 | 44 | func ObserveOverallRunTime() func() { 45 | start := time.Now() 46 | return func() { 47 | metrics.GetOrCreateCounter(metricPrefix + "overall_run_time_ms"). 48 | AddInt64(time.Since(start).Milliseconds()) 49 | } 50 | } 51 | 52 | func ObserveObject(storageType, operation string, size int64) { 53 | metrics.GetOrCreateCounter(fmt.Sprintf("%sobject_get_size_total{storage_type=%q,operation=%q}", 54 | metricPrefix, storageType, operation)).Add(1) 55 | metrics.GetOrCreateCounter(fmt.Sprintf("%sobject_get_size_bytes_total{storage_type=%q,operation=%q}", 56 | metricPrefix, storageType, operation)).AddInt64(size) 57 | getOrCreateHistogram(fmt.Sprintf("%sobject_get_size_bytes{storage_type=%q,operation=%q}", 58 | metricPrefix, storageType, operation)).Update(float64(size)) 59 | } 60 | 61 | func ObserveObjectMiss(storageType string) { 62 | metrics.GetOrCreateCounter(fmt.Sprintf("%sobject_miss_total{storage_type=%q}", metricPrefix, storageType)). 63 | Add(1) 64 | } 65 | 66 | func ObserveObjectDuration(storageType, operation string) func() { 67 | start := time.Now() 68 | return func() { 69 | getOrCreateHistogram(fmt.Sprintf("%sobject_get_duration_seconds{storage_type=%q,operation=%q}", 70 | metricPrefix, storageType, operation)).UpdateDuration(start) 71 | } 72 | } 73 | 74 | func ObserveStorageError(storageType, operation string) { 75 | metrics.GetOrCreateCounter(fmt.Sprintf("%sstorage_error_total{storage_type=%q,operation=%q}", 76 | metricPrefix, storageType, operation)).Add(1) 77 | } 78 | 79 | func PushMetrics(ctx context.Context, cfg PushConfig) error { 80 | if cfg.Endpoint == "" { 81 | return nil 82 | } 83 | 84 | extraLabels := make([]string, 0, len(cfg.ExtraLabels)) 85 | for k, v := range cfg.ExtraLabels { 86 | extraLabels = append(extraLabels, fmt.Sprintf("%s=%q", k, v)) 87 | } 88 | 89 | extraHeaders := make([]string, 0, len(cfg.ExtraHeaders)) 90 | for k, vs := range cfg.ExtraHeaders { 91 | for _, v := range vs { 92 | extraHeaders = append(extraHeaders, fmt.Sprintf("%s: %s", k, v)) 93 | } 94 | } 95 | 96 | return metrics.PushMetrics(ctx, cfg.Endpoint, true, &metrics.PushOptions{ 97 | ExtraLabels: strings.Join(extraLabels, ","), 98 | Headers: extraHeaders, 99 | Method: cfg.Method, 100 | }) 101 | } 102 | -------------------------------------------------------------------------------- /internal/infra/cacheproto/models.go: -------------------------------------------------------------------------------- 1 | package cacheproto 2 | 3 | import ( 4 | "io" 5 | "time" 6 | ) 7 | 8 | // Cmd is a command that can be issued to a child process. 9 | // 10 | // If the interface needs to grow, we can add new commands or new versioned 11 | // commands like "get2". 12 | type Cmd string 13 | 14 | const ( 15 | CmdGet = Cmd("get") 16 | CmdPut = Cmd("put") 17 | CmdClose = Cmd("close") // handled by server, not passed to handler 18 | ) 19 | 20 | // Request is the JSON-encoded message that's sent from cmd/go to 21 | // the GOCACHEPROG child process over stdin. Each JSON object is on its 22 | // own line. A Request of Type "put" with BodySize > 0 will be followed 23 | // by a line containing a base64-encoded JSON string literal of the body. 24 | type Request struct { 25 | // ID is a unique number per process across all requests. 26 | // It must be echoed in the ProgResponse from the child. 27 | ID int64 28 | // Command is the type of request. 29 | // The cmd/go tool will only send commands that were declared 30 | // as supported by the child. 31 | Command Cmd 32 | // ActionID is non-nil for get and puts. 33 | ActionID []byte `json:",omitempty"` // or nil if not used 34 | // OutputID is set for Type "put". 35 | OutputID []byte `json:",omitempty"` // or nil if not used 36 | // ObjectID is legacy field for "put" command, some older implementations used it instead of OutputID (like old versions of golangci-lint). 37 | ObjectID []byte `json:",omitempty"` // or nil if not used 38 | // Body is the body for "put" requests. It's sent after the JSON object 39 | // as a base64-encoded JSON string when BodySize is non-zero. 40 | // It's sent as a separate JSON value instead of being a struct field 41 | // send in this JSON object so large values can be streamed in both directions. 42 | // The base64 string body of a ProgRequest will always be written 43 | // immediately after the JSON object and a newline. 44 | Body io.Reader `json:"-"` 45 | // BodySize is the number of bytes of Body. If zero, the body isn't written. 46 | BodySize int64 `json:",omitempty"` 47 | } 48 | 49 | // Response is the JSON response from the child process to cmd/go. 50 | // 51 | // Except the first protocol message that the child writes to its 52 | // stdout with ID==0 and KnownCommands populated, these are only sent in 53 | // response to a Request from cmd/go. 54 | // 55 | // Responses can be sent in any order. The ID must match the request they're 56 | // replying to. 57 | type Response struct { 58 | ID int64 // that corresponds to ProgRequest; they can be answered out of order 59 | Err string `json:",omitempty"` // if non-empty, the error 60 | // KnownCommands is included in the first message that cache helper program 61 | // writes to stdout on startup (with ID==0). It includes the 62 | // ProgRequest.Command types that are supported by the program. 63 | // 64 | // This lets us extend the protocol gracefully over time (adding "get2", 65 | // etc), or fail gracefully when needed. It also lets us verify the program 66 | // wants to be a cache helper. 67 | KnownCommands []Cmd `json:",omitempty"` 68 | // For Get requests. 69 | Miss bool `json:",omitempty"` // cache miss 70 | OutputID []byte `json:",omitempty"` 71 | Size int64 `json:",omitempty"` // in bytes 72 | Time *time.Time `json:",omitempty"` // an Entry.Time; when the object was added to the docs 73 | // DiskPath is the absolute path on disk of the OutputID corresponding 74 | // a "get" request's ActionID (on cache hit) or a "put" request's 75 | // provided OutputID. 76 | DiskPath string `json:",omitempty"` 77 | } 78 | -------------------------------------------------------------------------------- /internal/infra/cacheproto/mocks_world_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: server.go 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -destination=mocks_world_test.go -package=cacheproto -source=server.go 7 | // 8 | 9 | // Package cacheproto is a generated GoMock package. 10 | package cacheproto 11 | 12 | import ( 13 | context "context" 14 | reflect "reflect" 15 | 16 | gomock "go.uber.org/mock/gomock" 17 | ) 18 | 19 | // MockResponseWriter is a mock of ResponseWriter interface. 20 | type MockResponseWriter struct { 21 | ctrl *gomock.Controller 22 | recorder *MockResponseWriterMockRecorder 23 | isgomock struct{} 24 | } 25 | 26 | // MockResponseWriterMockRecorder is the mock recorder for MockResponseWriter. 27 | type MockResponseWriterMockRecorder struct { 28 | mock *MockResponseWriter 29 | } 30 | 31 | // NewMockResponseWriter creates a new mock instance. 32 | func NewMockResponseWriter(ctrl *gomock.Controller) *MockResponseWriter { 33 | mock := &MockResponseWriter{ctrl: ctrl} 34 | mock.recorder = &MockResponseWriterMockRecorder{mock} 35 | return mock 36 | } 37 | 38 | // EXPECT returns an object that allows the caller to indicate expected use. 39 | func (m *MockResponseWriter) EXPECT() *MockResponseWriterMockRecorder { 40 | return m.recorder 41 | } 42 | 43 | // WriteResponse mocks base method. 44 | func (m *MockResponseWriter) WriteResponse(arg0 *Response) error { 45 | m.ctrl.T.Helper() 46 | ret := m.ctrl.Call(m, "WriteResponse", arg0) 47 | ret0, _ := ret[0].(error) 48 | return ret0 49 | } 50 | 51 | // WriteResponse indicates an expected call of WriteResponse. 52 | func (mr *MockResponseWriterMockRecorder) WriteResponse(arg0 any) *gomock.Call { 53 | mr.mock.ctrl.T.Helper() 54 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteResponse", reflect.TypeOf((*MockResponseWriter)(nil).WriteResponse), arg0) 55 | } 56 | 57 | // MockHandler is a mock of Handler interface. 58 | type MockHandler struct { 59 | ctrl *gomock.Controller 60 | recorder *MockHandlerMockRecorder 61 | isgomock struct{} 62 | } 63 | 64 | // MockHandlerMockRecorder is the mock recorder for MockHandler. 65 | type MockHandlerMockRecorder struct { 66 | mock *MockHandler 67 | } 68 | 69 | // NewMockHandler creates a new mock instance. 70 | func NewMockHandler(ctrl *gomock.Controller) *MockHandler { 71 | mock := &MockHandler{ctrl: ctrl} 72 | mock.recorder = &MockHandlerMockRecorder{mock} 73 | return mock 74 | } 75 | 76 | // EXPECT returns an object that allows the caller to indicate expected use. 77 | func (m *MockHandler) EXPECT() *MockHandlerMockRecorder { 78 | return m.recorder 79 | } 80 | 81 | // Handle mocks base method. 82 | func (m *MockHandler) Handle(ctx context.Context, writer ResponseWriter, req *Request) { 83 | m.ctrl.T.Helper() 84 | m.ctrl.Call(m, "Handle", ctx, writer, req) 85 | } 86 | 87 | // Handle indicates an expected call of Handle. 88 | func (mr *MockHandlerMockRecorder) Handle(ctx, writer, req any) *gomock.Call { 89 | mr.mock.ctrl.T.Helper() 90 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Handle", reflect.TypeOf((*MockHandler)(nil).Handle), ctx, writer, req) 91 | } 92 | 93 | // Supports mocks base method. 94 | func (m *MockHandler) Supports(cmd Cmd) bool { 95 | m.ctrl.T.Helper() 96 | ret := m.ctrl.Call(m, "Supports", cmd) 97 | ret0, _ := ret[0].(bool) 98 | return ret0 99 | } 100 | 101 | // Supports indicates an expected call of Supports. 102 | func (mr *MockHandlerMockRecorder) Supports(cmd any) *gomock.Call { 103 | mr.mock.ctrl.T.Helper() 104 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Supports", reflect.TypeOf((*MockHandler)(nil).Supports), cmd) 105 | } 106 | -------------------------------------------------------------------------------- /internal/infra/cacheproto/server_test.go: -------------------------------------------------------------------------------- 1 | package cacheproto 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/base64" 7 | "encoding/json" 8 | "fmt" 9 | "io" 10 | "sync" 11 | "testing" 12 | "time" 13 | 14 | "github.com/stretchr/testify/require" 15 | "go.uber.org/mock/gomock" 16 | ) 17 | 18 | func TestServer_ParallelRequestsBlocking(t *testing.T) { 19 | ctrl := gomock.NewController(t) 20 | 21 | handler := NewMockHandler(ctrl) 22 | var inBuf, outBuf bytes.Buffer 23 | 24 | // Create a server 25 | s := NewServer(ServerOptions{ 26 | Reader: &inBuf, 27 | Writer: &outBuf, 28 | Handler: handler, 29 | }) 30 | 31 | // Setup handler expectations 32 | handler.EXPECT().Supports(gomock.Any()).Return(true).AnyTimes() 33 | 34 | var handlerWg sync.WaitGroup 35 | handlerWg.Add(3) 36 | 37 | handler.EXPECT().Handle(gomock.Any(), gomock.Any(), gomock.Any()). 38 | Times(3). 39 | Do(func(_ context.Context, _ ResponseWriter, req *Request) { 40 | // Simulate slow processing of request body 41 | if req.Body != nil { 42 | _, err := io.Copy(io.Discard, req.Body) 43 | require.NoError(t, err) 44 | } 45 | handlerWg.Done() 46 | }) 47 | 48 | // Write two requests to input buffer 49 | // First request with a body that will be read slowly 50 | req1Body := "hello world" 51 | req1BodyBase64 := base64.StdEncoding.EncodeToString([]byte(req1Body)) 52 | writeRequest(&inBuf, &Request{ 53 | ID: 1, 54 | Command: CmdPut, 55 | BodySize: int64(len(req1Body)), 56 | }) 57 | fmt.Fprintf(&inBuf, "\"%s\"\n", req1BodyBase64) 58 | 59 | // Second request without body 60 | writeRequest(&inBuf, &Request{ 61 | ID: 2, 62 | Command: CmdGet, 63 | }) 64 | 65 | // Start server in background 66 | errCh := make(chan error, 1) 67 | go func() { 68 | errCh <- s.Run() 69 | }() 70 | 71 | // Wait for both handlers to complete 72 | handlerWg.Wait() 73 | 74 | // Stop server 75 | s.Stop() 76 | 77 | // Check for server errors 78 | require.NoError(t, <-errCh) 79 | } 80 | 81 | func writeRequest(w io.Writer, req *Request) { 82 | data, _ := json.Marshal(req) 83 | w.Write(data) 84 | w.Write([]byte("\n\n")) 85 | } 86 | 87 | func TestServer_Stop(t *testing.T) { 88 | ctrl := gomock.NewController(t) 89 | 90 | handler := NewMockHandler(ctrl) 91 | var inBuf, outBuf bytes.Buffer 92 | 93 | s := NewServer(ServerOptions{ 94 | Reader: &inBuf, 95 | Writer: &outBuf, 96 | Handler: handler, 97 | }) 98 | 99 | handler.EXPECT().Supports(gomock.Any()).Return(true).AnyTimes() 100 | 101 | // Start a long-running handler 102 | var onceHandlerStarted sync.Once 103 | handlerStarted := make(chan struct{}) 104 | 105 | handler.EXPECT().Handle(gomock.Any(), gomock.Any(), gomock.Any()). 106 | Do(func(context.Context, ResponseWriter, *Request) { 107 | onceHandlerStarted.Do(func() { 108 | close(handlerStarted) 109 | }) 110 | time.Sleep(100 * time.Millisecond) 111 | }).Times(2) 112 | 113 | // Write a request 114 | writeRequest(&inBuf, &Request{ 115 | ID: 1, 116 | Command: CmdGet, 117 | }) 118 | 119 | // Run server in background 120 | errCh := make(chan error, 1) 121 | go func() { 122 | errCh <- s.Run() 123 | }() 124 | 125 | // Wait for handler to start 126 | <-handlerStarted 127 | 128 | // Stop server - this should wait for handler to complete 129 | s.Stop() 130 | 131 | // Verify server stopped without errors 132 | require.NoError(t, <-errCh) 133 | } 134 | 135 | func TestServer_CloseOnEOF(t *testing.T) { 136 | ctrl := gomock.NewController(t) 137 | 138 | handler := NewMockHandler(ctrl) 139 | var inBuf, outBuf bytes.Buffer 140 | 141 | s := NewServer(ServerOptions{ 142 | Reader: &inBuf, 143 | Writer: &outBuf, 144 | Handler: handler, 145 | }) 146 | 147 | handler.EXPECT().Supports(gomock.Any()).Return(true).AnyTimes() 148 | 149 | // Expect Close command to be handled 150 | handler.EXPECT().Handle(gomock.Any(), gomock.Any(), gomock.Any()). 151 | Do(func(_ context.Context, _ ResponseWriter, req *Request) { 152 | require.Equal(t, CmdClose, req.Command) 153 | }) 154 | 155 | // Write nothing to input buffer - this will cause EOF on read 156 | 157 | // Run server 158 | err := s.Run() 159 | require.NoError(t, err) 160 | } 161 | -------------------------------------------------------------------------------- /internal/app/proxy/handler.go: -------------------------------------------------------------------------------- 1 | package proxy 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "log/slog" 7 | "maps" 8 | "net/http" 9 | "net/http/httputil" 10 | "net/url" 11 | "slices" 12 | "strings" 13 | 14 | "github.com/platacard/cacheprog/internal/app/cacheprog" 15 | "github.com/platacard/cacheprog/internal/infra/logging" 16 | "github.com/platacard/cacheprog/pkg/httpstorage" 17 | ) 18 | 19 | type MetricsConfig struct { 20 | Endpoint string // metrics endpoint, metrics push proxy will be enabled if provided 21 | ExtraLabels map[string]string // extra labels to be added to each metric 22 | ExtraHeaders http.Header // extra headers to be added to each request 23 | Method string // HTTP method to use for sending metrics 24 | } 25 | 26 | // NewHandler creates a http handler meant to be used as a proxy for cacheprog. 27 | // Among with remote storage it has 2 endpoints: 28 | // - /health - returns 200 29 | // - /metricsproxy - if [MetricsConfig.Endpoint] is provided - proxies metrics to the metrics endpoint, returns 200 on success, 500 on error 30 | func NewHandler(remoteStorage cacheprog.RemoteStorage, metricsConfig MetricsConfig) (http.Handler, error) { 31 | mux := http.NewServeMux() 32 | mux.HandleFunc("/health", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { 33 | w.WriteHeader(http.StatusOK) 34 | })) 35 | mux.Handle("/cache/", httpstorage.NewServer(remoteStorage)) 36 | 37 | if metricsConfig.Endpoint != "" { 38 | metricsURL, err := makeMetricsPushURL(metricsConfig) 39 | if err != nil { 40 | return nil, fmt.Errorf("failed to make metrics push URL: %w", err) 41 | } 42 | 43 | mux.Handle("/metricsproxy", http.StripPrefix("/metricsproxy", &httputil.ReverseProxy{ 44 | Rewrite: func(r *httputil.ProxyRequest) { 45 | r.SetURL(metricsURL) 46 | for k := range metricsConfig.ExtraHeaders { 47 | for _, v := range metricsConfig.ExtraHeaders[k] { 48 | r.Out.Header.Add(k, v) 49 | } 50 | } 51 | }, 52 | ErrorLog: slog.NewLogLogger(slog.Default().Handler(), slog.LevelError), 53 | ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { 54 | slog.ErrorContext(r.Context(), "failed to proxy metrics", "error", err) 55 | http.Error(w, err.Error(), http.StatusBadGateway) 56 | }, 57 | })) 58 | } 59 | 60 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 61 | mux.ServeHTTP(w, r.WithContext(logging.AttachArgs(r.Context(), 62 | slog.String("method", r.Method), 63 | slog.String("path", r.URL.Path), 64 | ))) 65 | }), nil 66 | } 67 | 68 | func makeMetricsPushURL(metricsConfig MetricsConfig) (*url.URL, error) { 69 | u, err := url.Parse(metricsConfig.Endpoint) 70 | if err != nil { 71 | return nil, fmt.Errorf("failed to parse metrics endpoint: %w", err) 72 | } 73 | 74 | if u.Path == "" { 75 | u.Path = "/" // make url absolute 76 | } 77 | 78 | if len(metricsConfig.ExtraLabels) == 0 { 79 | return u, nil 80 | } 81 | 82 | var pathElements []string 83 | pathElements = append(pathElements, "metrics") 84 | 85 | // metrics paths must start with job label 86 | var jobName string 87 | if jn, ok := metricsConfig.ExtraLabels["job"]; ok { 88 | jobName = jn 89 | delete(metricsConfig.ExtraLabels, "job") 90 | } 91 | if encodedJobName, isBase64 := encodeComponent(jobName); isBase64 { 92 | pathElements = append(pathElements, "job@base64", encodedJobName) 93 | } else { 94 | pathElements = append(pathElements, "job", jobName) 95 | } 96 | 97 | for _, k := range slices.Sorted(maps.Keys(metricsConfig.ExtraLabels)) { 98 | v := metricsConfig.ExtraLabels[k] 99 | if encoded, isBase64 := encodeComponent(v); isBase64 { 100 | pathElements = append(pathElements, k+"@base64", encoded) 101 | } else { 102 | pathElements = append(pathElements, k, encoded) 103 | } 104 | } 105 | 106 | return u.JoinPath(pathElements...), nil 107 | } 108 | 109 | // encodeComponent encodes the provided string with base64.RawURLEncoding in 110 | // case it contains '/' and as "=" in case it is empty. If neither is the case, 111 | // it uses url.QueryEscape instead. It returns true in the former two cases. 112 | // This function was copied from prometheus client. 113 | func encodeComponent(s string) (string, bool) { 114 | if s == "" { 115 | return "=", true 116 | } 117 | if strings.Contains(s, "/") { 118 | return base64.RawURLEncoding.EncodeToString([]byte(s)), true 119 | } 120 | return url.QueryEscape(s), false 121 | } 122 | -------------------------------------------------------------------------------- /internal/app/cacheprog/observers.go: -------------------------------------------------------------------------------- 1 | package cacheprog 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "log/slog" 7 | 8 | "github.com/platacard/cacheprog/internal/infra/logging" 9 | "github.com/platacard/cacheprog/internal/infra/metrics" 10 | ) 11 | 12 | type ObservingLocalStorage struct { 13 | LocalStorage 14 | } 15 | 16 | func (o ObservingLocalStorage) GetLocal(ctx context.Context, request *LocalGetRequest) (*LocalGetResponse, error) { 17 | slog.DebugContext(ctx, "Get object from local storage") 18 | defer metrics.ObserveObjectDuration("local", "get")() 19 | 20 | localObj, err := o.LocalStorage.GetLocal(ctx, request) 21 | switch { 22 | case errors.Is(err, nil): 23 | slog.DebugContext(ctx, "Local object found", "disk_path", localObj.DiskPath) 24 | metrics.ObserveObject("local", "get", localObj.Size) 25 | return localObj, nil 26 | case errors.Is(err, ErrNotFound): 27 | slog.DebugContext(ctx, "Local object not found") 28 | metrics.ObserveObjectMiss("local") 29 | return nil, err 30 | default: 31 | slog.ErrorContext(ctx, "Failed to get from local storage", logging.Error(err)) 32 | metrics.ObserveStorageError("local", "get") 33 | return nil, err 34 | } 35 | } 36 | 37 | func (o ObservingLocalStorage) GetLocalObject(ctx context.Context, request *LocalObjectGetRequest) (*LocalObjectGetResponse, error) { 38 | slog.DebugContext(ctx, "Get object stream from local storage") 39 | defer metrics.ObserveObjectDuration("local", "get")() 40 | 41 | localObj, err := o.LocalStorage.GetLocalObject(ctx, request) 42 | switch { 43 | case errors.Is(err, nil): 44 | slog.DebugContext(ctx, "Local object stream found") 45 | metrics.ObserveObject("local", "get", localObj.Size) 46 | return localObj, nil 47 | case errors.Is(err, ErrNotFound): 48 | slog.DebugContext(ctx, "Local object stream not found") 49 | metrics.ObserveObjectMiss("local") 50 | return nil, err 51 | default: 52 | slog.ErrorContext(ctx, "Failed to get from local storage", logging.Error(err)) 53 | metrics.ObserveStorageError("local", "get") 54 | return nil, err 55 | } 56 | } 57 | 58 | func (o ObservingLocalStorage) PutLocal(ctx context.Context, request *LocalPutRequest) (*LocalPutResponse, error) { 59 | ctx = logging.AttachArgs(ctx, "output_id", logging.Bytes(request.OutputID), "size", request.Size) 60 | slog.DebugContext(ctx, "Put object to local storage") 61 | defer metrics.ObserveObjectDuration("local", "put")() 62 | 63 | localObj, err := o.LocalStorage.PutLocal(ctx, request) 64 | if err != nil { 65 | slog.ErrorContext(ctx, "Failed to put to local storage", logging.Error(err)) 66 | metrics.ObserveStorageError("local", "put") 67 | return nil, err 68 | } 69 | 70 | slog.DebugContext(ctx, "Object stored in local storage", "disk_path", localObj.DiskPath) 71 | metrics.ObserveObject("local", "put", request.Size) 72 | return localObj, nil 73 | } 74 | 75 | type ObservingRemoteStorage struct { 76 | RemoteStorage 77 | } 78 | 79 | func (o ObservingRemoteStorage) Get(ctx context.Context, request *GetRequest) (*GetResponse, error) { 80 | slog.DebugContext(ctx, "Get object from remote storage") 81 | defer metrics.ObserveObjectDuration("remote", "get")() 82 | 83 | remoteObj, err := o.RemoteStorage.Get(ctx, request) 84 | switch { 85 | case errors.Is(err, nil): 86 | slog.DebugContext(ctx, "Remote object found", 87 | "output_id", logging.Bytes(remoteObj.OutputID), 88 | "mod_time", remoteObj.ModTime, 89 | "size", remoteObj.Size, 90 | ) 91 | metrics.ObserveObject("remote", "get", remoteObj.Size) 92 | return remoteObj, nil 93 | case errors.Is(err, ErrNotFound): 94 | slog.DebugContext(ctx, "Remote object not found") 95 | metrics.ObserveObjectMiss("remote") 96 | return nil, err 97 | default: 98 | slog.ErrorContext(ctx, "Failed to get from remote storage", logging.Error(err)) 99 | metrics.ObserveStorageError("remote", "get") 100 | return nil, err 101 | } 102 | } 103 | 104 | func (o ObservingRemoteStorage) Put(ctx context.Context, request *PutRequest) (*PutResponse, error) { 105 | ctx = logging.AttachArgs(ctx, "output_id", logging.Bytes(request.OutputID), "size", request.Size) 106 | slog.DebugContext(ctx, "Put object to remote storage") 107 | defer metrics.ObserveObjectDuration("remote", "put")() 108 | 109 | if _, err := o.RemoteStorage.Put(ctx, request); err != nil { 110 | slog.ErrorContext(ctx, "Failed to put to remote storage", logging.Error(err)) 111 | metrics.ObserveStorageError("remote", "put") 112 | return nil, err 113 | } 114 | 115 | slog.DebugContext(ctx, "Object stored in remote storage") 116 | metrics.ObserveObject("remote", "put", request.Size) 117 | return &PutResponse{}, nil 118 | } 119 | -------------------------------------------------------------------------------- /internal/infra/cacheproto/server.go: -------------------------------------------------------------------------------- 1 | package cacheproto 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "log/slog" 9 | "sync" 10 | "sync/atomic" 11 | 12 | "github.com/platacard/cacheprog/internal/infra/logging" 13 | ) 14 | 15 | //go:generate go tool go.uber.org/mock/mockgen -destination=mocks_world_test.go -package=$GOPACKAGE -source=$GOFILE 16 | 17 | type ResponseWriter interface { 18 | WriteResponse(*Response) error 19 | } 20 | 21 | type Handler interface { 22 | Handle(ctx context.Context, writer ResponseWriter, req *Request) 23 | 24 | // Supports used to indicate supported features by the handler. Needed to negotiate supported commands. 25 | Supports(cmd Cmd) bool 26 | } 27 | 28 | type ServerOptions struct { 29 | Reader io.Reader 30 | Writer io.Writer 31 | Handler Handler 32 | } 33 | 34 | type Server struct { 35 | reader *Reader 36 | writer *synchronizedWriter 37 | handler Handler 38 | 39 | stop atomic.Bool 40 | canStopWait chan struct{} // closing this guarantees that we can safely call stopWait.Wait 41 | stopWait sync.WaitGroup 42 | } 43 | 44 | func NewServer(opts ServerOptions) *Server { 45 | return &Server{ 46 | reader: NewReader(opts.Reader), 47 | writer: &synchronizedWriter{ 48 | writer: NewWriter(opts.Writer), 49 | }, 50 | handler: opts.Handler, 51 | canStopWait: make(chan struct{}), 52 | } 53 | } 54 | 55 | func (s *Server) Run() error { 56 | ctx := context.Background() 57 | 58 | if err := s.handshake(ctx); err != nil { 59 | return fmt.Errorf("handshake failure: %w", err) 60 | } 61 | 62 | sem := make(chan struct{}, 1) 63 | var seenClose bool 64 | for !s.stop.Load() { 65 | sem <- struct{}{} 66 | slog.DebugContext(ctx, "Waiting for request") 67 | request, err := s.reader.ReadRequest() 68 | if errors.Is(err, io.EOF) { 69 | slog.InfoContext(ctx, "Stopping server") 70 | s.stop.Store(true) 71 | close(s.canStopWait) 72 | break 73 | } 74 | if err != nil { 75 | return fmt.Errorf("read request: %w", err) 76 | } 77 | 78 | ctx := logging.AttachArgs(ctx, 79 | "command", request.Command, 80 | "id", request.ID, 81 | "action_id", logging.Bytes(request.ActionID), 82 | ) 83 | slog.DebugContext(ctx, "Request received") 84 | 85 | s.stopWait.Add(1) 86 | if s.stop.Load() { 87 | close(s.canStopWait) 88 | } 89 | 90 | if !seenClose && request.Command == CmdClose { 91 | seenClose = true 92 | } 93 | 94 | // this needed to ensure that entire body is read before we start reading next request 95 | reqBody := request.Body 96 | if reqBody != nil { 97 | request.Body = &unblockingReader{r: reqBody, sem: sem} 98 | } else { 99 | <-sem // release semaphore, we can read next request 100 | } 101 | 102 | go func() { 103 | defer s.stopWait.Done() 104 | s.handler.Handle(ctx, s.writer, request) 105 | }() 106 | } 107 | 108 | <-s.canStopWait 109 | s.stopWait.Wait() 110 | // sometimes client doesn't send close command, so we need to send it ourselves 111 | if !seenClose && s.handler.Supports(CmdClose) { 112 | slog.InfoContext(ctx, "Client didn't send close command, simulating it") 113 | s.handler.Handle(ctx, &noopWriter{ctx: ctx}, &Request{ 114 | Command: CmdClose, 115 | }) 116 | } 117 | slog.InfoContext(ctx, "Server stopped") 118 | return nil 119 | } 120 | 121 | func (s *Server) Stop() { 122 | s.stop.Store(true) 123 | <-s.canStopWait 124 | s.stopWait.Wait() 125 | } 126 | 127 | func (s *Server) handshake(ctx context.Context) error { 128 | var supportedCmds []Cmd 129 | 130 | if s.handler.Supports(CmdClose) { 131 | supportedCmds = append(supportedCmds, CmdClose) 132 | } 133 | 134 | if s.handler.Supports(CmdGet) { 135 | supportedCmds = append(supportedCmds, CmdGet) 136 | } 137 | 138 | if s.handler.Supports(CmdPut) { 139 | supportedCmds = append(supportedCmds, CmdPut) 140 | } 141 | 142 | err := s.writer.WriteResponse(&Response{ 143 | ID: 0, 144 | KnownCommands: supportedCmds, 145 | }) 146 | if err != nil { 147 | return fmt.Errorf("write hello: %w", err) 148 | } 149 | 150 | slog.DebugContext(ctx, "Handshake done", "supported_commands", supportedCmds) 151 | 152 | return nil 153 | } 154 | 155 | type synchronizedWriter struct { 156 | mu sync.Mutex 157 | writer ResponseWriter 158 | } 159 | 160 | func (w *synchronizedWriter) WriteResponse(response *Response) error { 161 | w.mu.Lock() 162 | defer w.mu.Unlock() 163 | 164 | return w.writer.WriteResponse(response) 165 | } 166 | 167 | type unblockingReader struct { 168 | r io.Reader 169 | sem <-chan struct{} 170 | } 171 | 172 | func (u *unblockingReader) Read(p []byte) (n int, err error) { 173 | n, err = u.r.Read(p) 174 | if err != nil { 175 | <-u.sem 176 | } 177 | return n, err 178 | } 179 | -------------------------------------------------------------------------------- /internal/infra/compression/codec.go: -------------------------------------------------------------------------------- 1 | package compression 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "sync" 8 | 9 | "github.com/klauspost/compress/zstd" 10 | "github.com/valyala/bytebufferpool" 11 | 12 | "github.com/platacard/cacheprog/internal/app/cacheprog" 13 | ) 14 | 15 | // zstd compression chosen during tests because it shows nice compression ratio and best decompression speed 16 | 17 | const ( 18 | compressionZstd = "zstd" 19 | compressionNone = "" 20 | ) 21 | 22 | type Codec struct { 23 | writerPool *sync.Pool 24 | readerPool *sync.Pool 25 | memPool *bytebufferpool.Pool 26 | } 27 | 28 | func NewCodec() *Codec { 29 | return &Codec{ 30 | writerPool: &sync.Pool{ 31 | New: func() any { 32 | // concurrency handled by upper layers 33 | writer, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1)) 34 | if err != nil { 35 | panic(err) 36 | } 37 | return writer 38 | }, 39 | }, 40 | readerPool: &sync.Pool{ 41 | New: func() any { 42 | reader, err := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1)) 43 | if err != nil { 44 | panic(err) 45 | } 46 | return reader 47 | }, 48 | }, 49 | memPool: new(bytebufferpool.Pool), 50 | } 51 | } 52 | 53 | func (c *Codec) Compress(request *cacheprog.CompressRequest) (*cacheprog.CompressResponse, error) { 54 | if request.Size == 0 { 55 | // do not even try to compress empty files 56 | return &cacheprog.CompressResponse{ 57 | Size: request.Size, 58 | Body: newHookableCloseSeeker(request.Body, nil), 59 | Algorithm: compressionNone, 60 | }, nil 61 | } 62 | 63 | // cachable objects are reasonably small to be compressed in memory 64 | 65 | buf := c.memPool.Get() 66 | buf.Reset() 67 | 68 | writer := c.writerPool.Get().(*zstd.Encoder) 69 | writer.Reset(buf) 70 | defer c.writerPool.Put(writer) 71 | 72 | _, err := writer.ReadFrom(request.Body) 73 | if err != nil { 74 | return nil, fmt.Errorf("copy body: %w", err) 75 | } 76 | 77 | if err = writer.Close(); err != nil { 78 | return nil, fmt.Errorf("finalize writer: %w", err) 79 | } 80 | 81 | size := int64(buf.Len()) 82 | 83 | if size >= request.Size { 84 | _, err = request.Body.Seek(0, io.SeekStart) 85 | if err != nil { 86 | return nil, fmt.Errorf("seek request body to start: %w", err) 87 | } 88 | 89 | // return original file 90 | return &cacheprog.CompressResponse{ 91 | Size: request.Size, 92 | Body: newHookableCloseSeeker(request.Body, nil), 93 | Algorithm: compressionNone, 94 | }, nil 95 | } 96 | 97 | return &cacheprog.CompressResponse{ 98 | Size: size, 99 | // get seekable reader using bytes.NewReader 100 | Body: newHookableCloseSeeker(bytes.NewReader(buf.B), func() error { 101 | c.memPool.Put(buf) 102 | return nil 103 | }), 104 | Algorithm: compressionZstd, 105 | }, nil 106 | } 107 | 108 | func (c *Codec) Decompress(request *cacheprog.DecompressRequest) (*cacheprog.DecompressResponse, error) { 109 | switch request.Algorithm { 110 | case compressionZstd: 111 | // pass 112 | case compressionNone: 113 | return &cacheprog.DecompressResponse{ 114 | Body: request.Body, 115 | }, nil 116 | default: 117 | return nil, fmt.Errorf("unsupported algorithm %q: %w", request.Algorithm, cacheprog.ErrNotFound) 118 | } 119 | 120 | // concurrency handled by upper layers 121 | reader := c.readerPool.Get().(*zstd.Decoder) 122 | if err := reader.Reset(request.Body); err != nil { 123 | return nil, fmt.Errorf("reset reader: %w", err) 124 | } 125 | 126 | return &cacheprog.DecompressResponse{ 127 | Body: &hookableCloseDecoder{d: reader, closeHook: func() error { 128 | err := request.Body.Close() 129 | c.readerPool.Put(reader) 130 | return err 131 | }}, 132 | }, nil 133 | } 134 | 135 | type hookableCloseSeeker struct { 136 | io.ReadSeeker 137 | closeHook func() error 138 | } 139 | 140 | func (h *hookableCloseSeeker) Close() error { 141 | if h.closeHook != nil { 142 | return h.closeHook() 143 | } 144 | return nil 145 | } 146 | 147 | type hookableCloseSeekerWriterTo struct { 148 | io.ReadSeeker 149 | closeHook func() error 150 | } 151 | 152 | func (h *hookableCloseSeekerWriterTo) Close() error { 153 | if h.closeHook != nil { 154 | return h.closeHook() 155 | } 156 | return nil 157 | } 158 | 159 | func (h *hookableCloseSeekerWriterTo) WriteTo(w io.Writer) (int64, error) { 160 | return h.ReadSeeker.(io.WriterTo).WriteTo(w) 161 | } 162 | 163 | func newHookableCloseSeeker(r io.ReadSeeker, closeHook func() error) io.ReadSeekCloser { 164 | if _, ok := r.(io.WriterTo); ok { 165 | return &hookableCloseSeekerWriterTo{r, closeHook} 166 | } 167 | return &hookableCloseSeeker{r, closeHook} 168 | } 169 | 170 | type hookableCloseDecoder struct { 171 | d *zstd.Decoder 172 | closeHook func() error 173 | } 174 | 175 | func (r *hookableCloseDecoder) Read(p []byte) (int, error) { 176 | return r.d.Read(p) 177 | } 178 | 179 | func (r *hookableCloseDecoder) WriteTo(w io.Writer) (int64, error) { 180 | return r.d.WriteTo(w) 181 | } 182 | 183 | func (r *hookableCloseDecoder) Close() error { 184 | if r.closeHook != nil { 185 | return r.closeHook() 186 | } 187 | return nil 188 | } 189 | -------------------------------------------------------------------------------- /internal/infra/storage/http_test.go: -------------------------------------------------------------------------------- 1 | package storage_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/hex" 7 | "io" 8 | "net/http" 9 | "net/http/httptest" 10 | "testing" 11 | "time" 12 | 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/require" 15 | 16 | "github.com/platacard/cacheprog/internal/app/cacheprog" 17 | "github.com/platacard/cacheprog/internal/infra/storage" 18 | ) 19 | 20 | func TestHTTP_Get(t *testing.T) { 21 | t.Run("success", func(t *testing.T) { 22 | // Create a copy of the response body for testing 23 | bodyContent := []byte("hello world") 24 | 25 | // Setup test server that simulates successful response 26 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 27 | // Verify request 28 | assert.Equal(t, "/cache/01020304", r.URL.Path) 29 | assert.Equal(t, http.MethodGet, r.Method) 30 | 31 | // Set response headers 32 | outputID := []byte("test-output-id") 33 | w.Header().Set(storage.OutputIDHeader, hex.EncodeToString(outputID)) 34 | w.Header().Set("Last-Modified", time.Date(2023, 1, 1, 12, 0, 0, 0, time.UTC).Format(http.TimeFormat)) 35 | w.Header().Set("Content-Length", "11") 36 | w.Header().Set(storage.CompressionAlgorithmHeader, "gzip") 37 | w.Header().Set(storage.UncompressedSizeHeader, "20") 38 | 39 | // Write response body 40 | w.WriteHeader(http.StatusOK) 41 | w.Write(bodyContent) 42 | })) 43 | defer server.Close() 44 | 45 | // Create a mock HTTP client that returns our prepared response 46 | origClient := http.DefaultClient 47 | client := storage.NewHTTP(origClient, server.URL, nil) 48 | 49 | // Create test request 50 | request := &cacheprog.GetRequest{ 51 | ActionID: []byte{1, 2, 3, 4}, 52 | } 53 | 54 | // Call the method 55 | response, err := client.Get(context.Background(), request) 56 | 57 | // Verify response 58 | require.NoError(t, err) 59 | require.NotNil(t, response) 60 | 61 | assert.Equal(t, "test-output-id", string(response.OutputID)) 62 | assert.Equal(t, time.Date(2023, 1, 1, 12, 0, 0, 0, time.UTC), response.ModTime) 63 | assert.Equal(t, int64(11), response.Size) 64 | assert.Equal(t, "gzip", response.CompressionAlgorithm) 65 | assert.Equal(t, int64(20), response.UncompressedSize) 66 | 67 | // Don't try to read response.Body as it's already closed due to defer in HTTP.Get 68 | // Instead, just verify that it was properly set up initially 69 | }) 70 | 71 | t.Run("not found", func(t *testing.T) { 72 | // Setup test server that simulates not found response 73 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 74 | // Verify request 75 | assert.Equal(t, "/cache/01020304", r.URL.Path) 76 | assert.Equal(t, http.MethodGet, r.Method) 77 | 78 | // Return not found 79 | w.WriteHeader(http.StatusNotFound) 80 | w.Write([]byte("not found")) 81 | })) 82 | defer server.Close() 83 | 84 | // Initialize HTTP client with test server endpoint 85 | client := storage.NewHTTP(http.DefaultClient, server.URL, nil) 86 | 87 | // Create test request 88 | request := &cacheprog.GetRequest{ 89 | ActionID: []byte{1, 2, 3, 4}, 90 | } 91 | 92 | // Call the method 93 | response, err := client.Get(context.Background(), request) 94 | 95 | // Verify error is ErrNotFound 96 | assert.Nil(t, response) 97 | assert.ErrorIs(t, err, cacheprog.ErrNotFound) 98 | }) 99 | } 100 | 101 | func TestHTTP_Put(t *testing.T) { 102 | t.Run("success", func(t *testing.T) { 103 | // Setup test server that simulates successful response 104 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 105 | // Verify request 106 | assert.Equal(t, "/cache/01020304", r.URL.Path) 107 | assert.Equal(t, http.MethodPut, r.Method) 108 | assert.Equal(t, hex.EncodeToString([]byte("test-output-id")), r.Header.Get(storage.OutputIDHeader)) 109 | assert.Equal(t, hex.EncodeToString([]byte("md5-test")), r.Header.Get(storage.MD5SumHeader)) 110 | assert.Equal(t, hex.EncodeToString([]byte("sha256-test")), r.Header.Get(storage.Sha256SumHeader)) 111 | assert.Equal(t, "gzip", r.Header.Get(storage.CompressionAlgorithmHeader)) 112 | assert.Equal(t, "20", r.Header.Get(storage.UncompressedSizeHeader)) 113 | assert.Equal(t, int64(11), r.ContentLength) 114 | 115 | // Read request body 116 | body, err := io.ReadAll(r.Body) 117 | require.NoError(t, err) 118 | assert.Equal(t, "hello world", string(body)) 119 | 120 | // Write success response 121 | w.WriteHeader(http.StatusOK) 122 | })) 123 | defer server.Close() 124 | 125 | // Initialize HTTP client with test server endpoint 126 | client := storage.NewHTTP(http.DefaultClient, server.URL, nil) 127 | 128 | // Create test request 129 | request := &cacheprog.PutRequest{ 130 | ActionID: []byte{1, 2, 3, 4}, 131 | OutputID: []byte("test-output-id"), 132 | Size: 11, 133 | Body: bytes.NewReader([]byte("hello world")), 134 | MD5Sum: []byte("md5-test"), 135 | Sha256Sum: []byte("sha256-test"), 136 | CompressionAlgorithm: "gzip", 137 | UncompressedSize: 20, 138 | } 139 | 140 | // Call the method 141 | response, err := client.Put(context.Background(), request) 142 | 143 | // Verify response 144 | require.NoError(t, err) 145 | require.NotNil(t, response) 146 | }) 147 | } 148 | -------------------------------------------------------------------------------- /internal/infra/debugging/debugging.go: -------------------------------------------------------------------------------- 1 | package debugging 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log/slog" 7 | "os" 8 | "runtime" 9 | "runtime/pprof" 10 | "runtime/trace" 11 | 12 | "github.com/felixge/fgprof" 13 | ) 14 | 15 | type DebugConfig struct { 16 | CPUProfilePath string // writes CPU profile to the specified path, if set 17 | MemProfilePath string // writes memory profile to the specified path, if set 18 | TraceProfilePath string // writes trace (for `go tool trace`) to the specified path, if set 19 | FgprofPath string // writes fgprof profile to the specified path, if set 20 | } 21 | 22 | func ConfigureDebugging(cfg DebugConfig) (stop func() error, err error) { 23 | cpuStop, err := configureCPUProfiling(cfg.CPUProfilePath) 24 | if err != nil { 25 | return nil, fmt.Errorf("failed to configure CPU profiling: %w", err) 26 | } 27 | 28 | memStop, err := configureMemProfiling(cfg.MemProfilePath) 29 | if err != nil { 30 | _ = cpuStop() 31 | return nil, fmt.Errorf("failed to configure memory profiling: %w", err) 32 | } 33 | 34 | traceStop, err := configureTraceProfiling(cfg.TraceProfilePath) 35 | if err != nil { 36 | _ = cpuStop() 37 | _ = memStop() 38 | return nil, fmt.Errorf("failed to configure trace profiling: %w", err) 39 | } 40 | 41 | fgprofStop, err := configureFgprofProfiling(cfg.FgprofPath) 42 | if err != nil { 43 | _ = cpuStop() 44 | _ = memStop() 45 | _ = traceStop() 46 | return nil, fmt.Errorf("failed to configure fgprof profiling: %w", err) 47 | } 48 | 49 | return func() error { 50 | cpuErr := cpuStop() 51 | memErr := memStop() 52 | traceErr := traceStop() 53 | fgprofErr := fgprofStop() 54 | return errors.Join(cpuErr, memErr, traceErr, fgprofErr) 55 | }, nil 56 | } 57 | 58 | func configureCPUProfiling(path string) (stop func() error, err error) { 59 | if path == "" { 60 | return func() error { return nil }, nil 61 | } 62 | 63 | f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_SYNC, 0644) 64 | if err != nil { 65 | return nil, fmt.Errorf("failed to open CPU profile file: %w", err) 66 | } 67 | 68 | if err = pprof.StartCPUProfile(f); err != nil { 69 | return nil, fmt.Errorf("failed to start CPU profiling: %w", err) 70 | } 71 | 72 | return func() error { 73 | pprof.StopCPUProfile() 74 | return f.Close() 75 | }, nil 76 | } 77 | 78 | func configureMemProfiling(path string) (stop func() error, err error) { 79 | if path == "" { 80 | return func() error { return nil }, nil 81 | } 82 | 83 | f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_SYNC, 0644) 84 | if err != nil { 85 | return nil, fmt.Errorf("failed to open memory profile file: %w", err) 86 | } 87 | 88 | return func() error { 89 | var ms runtime.MemStats 90 | runtime.ReadMemStats(&ms) 91 | 92 | slog.Debug("Mem stats", 93 | "alloc", memoryBytesValue(ms.Alloc), 94 | "total_alloc", memoryBytesValue(ms.TotalAlloc), 95 | "sys", memoryBytesValue(ms.Sys), 96 | "num_gc", ms.NumGC, 97 | "heap_alloc", memoryBytesValue(ms.HeapAlloc), 98 | "heap_sys", memoryBytesValue(ms.HeapSys), 99 | "heap_idle", memoryBytesValue(ms.HeapIdle), 100 | "heap_released", memoryBytesValue(ms.HeapReleased), 101 | "heap_inuse", memoryBytesValue(ms.HeapInuse), 102 | "stack_inuse", memoryBytesValue(ms.StackInuse), 103 | "stack_sys", memoryBytesValue(ms.StackSys), 104 | "mspan_inuse", memoryBytesValue(ms.MSpanInuse), 105 | "mspan_sys", memoryBytesValue(ms.MSpanSys), 106 | "buck_hash_sys", memoryBytesValue(ms.BuckHashSys), 107 | "gc_sys", memoryBytesValue(ms.GCSys), 108 | "other_sys", memoryBytesValue(ms.OtherSys), 109 | "mallocs", ms.Mallocs, 110 | "frees", ms.Frees, 111 | "heap_objects", ms.HeapObjects, 112 | "gc_cpu_fraction", fmt.Sprintf("%.2f", ms.GCCPUFraction), 113 | ) 114 | 115 | if err = pprof.WriteHeapProfile(f); err != nil { 116 | return fmt.Errorf("failed to write memory profile: %w", err) 117 | } 118 | return f.Close() 119 | }, nil 120 | } 121 | 122 | func configureTraceProfiling(path string) (stop func() error, err error) { 123 | if path == "" { 124 | return func() error { return nil }, nil 125 | } 126 | 127 | f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_SYNC, 0644) 128 | if err != nil { 129 | return nil, fmt.Errorf("failed to open trace profile file: %w", err) 130 | } 131 | 132 | if err = trace.Start(f); err != nil { 133 | return nil, fmt.Errorf("failed to start trace profiling: %w", err) 134 | } 135 | 136 | return func() error { 137 | trace.Stop() 138 | return f.Close() 139 | }, nil 140 | } 141 | 142 | func configureFgprofProfiling(path string) (stop func() error, err error) { 143 | if path == "" { 144 | return func() error { return nil }, nil 145 | } 146 | 147 | f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_SYNC, 0644) 148 | if err != nil { 149 | return nil, fmt.Errorf("failed to open fgprof profile file: %w", err) 150 | } 151 | 152 | end := fgprof.Start(f, fgprof.FormatPprof) 153 | 154 | return func() error { 155 | endErr := end() 156 | closeErr := f.Close() 157 | return errors.Join(endErr, closeErr) 158 | }, nil 159 | } 160 | 161 | type memoryBytesValue uint64 162 | 163 | func (m memoryBytesValue) LogValue() slog.Value { 164 | const Kb = 1024 165 | const Mb = Kb * 1024 166 | 167 | if m < Kb { 168 | return slog.StringValue(fmt.Sprintf("%d bytes", m)) 169 | } 170 | if m < Mb { 171 | return slog.StringValue(fmt.Sprintf("%d kb", m/Kb)) 172 | } 173 | return slog.StringValue(fmt.Sprintf("%d mb", m/Mb)) 174 | } 175 | -------------------------------------------------------------------------------- /pkg/httpstorage/server_test.go: -------------------------------------------------------------------------------- 1 | package httpstorage 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "crypto/md5" 7 | "crypto/sha256" 8 | "fmt" 9 | "io" 10 | "net/http" 11 | "net/http/httptest" 12 | "testing" 13 | "time" 14 | 15 | "github.com/stretchr/testify/require" 16 | "go.uber.org/mock/gomock" 17 | 18 | "github.com/platacard/cacheprog/internal/app/cacheprog" 19 | "github.com/platacard/cacheprog/internal/infra/storage" 20 | ) 21 | 22 | func TestServer_Get(t *testing.T) { 23 | t.Run("success", func(t *testing.T) { 24 | ctrl := gomock.NewController(t) 25 | remoteStorage := NewMockRemoteStorage(ctrl) 26 | handler := NewServer(remoteStorage) 27 | 28 | server := httptest.NewServer(handler) 29 | t.Cleanup(server.Close) 30 | 31 | actionID := []byte("test-action") 32 | content := []byte("test-content") 33 | outputID := []byte("test-output-id") 34 | modTime := time.Now().Truncate(time.Second).UTC() 35 | 36 | remoteStorage.EXPECT().Get(gomock.Any(), &cacheprog.GetRequest{ 37 | ActionID: actionID, 38 | }).Return(&cacheprog.GetResponse{ 39 | OutputID: outputID, 40 | ModTime: modTime, 41 | Size: int64(len(content)), 42 | Body: io.NopCloser(bytes.NewReader(content)), 43 | CompressionAlgorithm: "test-compression-algorithm", 44 | UncompressedSize: int64(len(content)) * 2, 45 | }, nil) 46 | 47 | req, err := storage.NewGetRequest(context.Background(), server.URL, &cacheprog.GetRequest{ 48 | ActionID: actionID, 49 | }) 50 | require.NoError(t, err) 51 | 52 | resp, err := http.DefaultClient.Do(req) 53 | t.Cleanup(func() { 54 | if resp != nil && resp.Body != nil { 55 | resp.Body.Close() 56 | } 57 | }) 58 | require.NoError(t, err) 59 | require.Equal(t, http.StatusOK, resp.StatusCode) 60 | 61 | parsedResp, err := storage.ParseGetResponse(resp) 62 | require.NoError(t, err) 63 | require.Equal(t, outputID, parsedResp.OutputID) 64 | require.Equal(t, modTime, parsedResp.ModTime) 65 | require.Equal(t, int64(len(content)), parsedResp.Size) 66 | require.Equal(t, "test-compression-algorithm", parsedResp.CompressionAlgorithm) 67 | require.Equal(t, int64(len(content))*2, parsedResp.UncompressedSize) 68 | 69 | body, err := io.ReadAll(parsedResp.Body) 70 | require.NoError(t, err) 71 | require.Equal(t, content, body) 72 | }) 73 | 74 | t.Run("not found", func(t *testing.T) { 75 | ctrl := gomock.NewController(t) 76 | remoteStorage := NewMockRemoteStorage(ctrl) 77 | handler := NewServer(remoteStorage) 78 | 79 | server := httptest.NewServer(handler) 80 | t.Cleanup(server.Close) 81 | 82 | remoteStorage.EXPECT().Get(gomock.Any(), &cacheprog.GetRequest{ 83 | ActionID: []byte("test-action"), 84 | }).Return(nil, cacheprog.ErrNotFound) 85 | 86 | req, err := storage.NewGetRequest(context.Background(), server.URL, &cacheprog.GetRequest{ 87 | ActionID: []byte("test-action"), 88 | }) 89 | require.NoError(t, err) 90 | 91 | resp, err := http.DefaultClient.Do(req) 92 | t.Cleanup(func() { 93 | if resp != nil && resp.Body != nil { 94 | resp.Body.Close() 95 | } 96 | }) 97 | require.NoError(t, err) 98 | require.Equal(t, http.StatusNotFound, resp.StatusCode) 99 | }) 100 | } 101 | 102 | func TestServer_Put(t *testing.T) { 103 | t.Run("success", func(t *testing.T) { 104 | ctrl := gomock.NewController(t) 105 | remoteStorage := NewMockRemoteStorage(ctrl) 106 | handler := NewServer(remoteStorage) 107 | 108 | server := httptest.NewServer(handler) 109 | t.Cleanup(server.Close) 110 | 111 | actionID := []byte("test-action") 112 | content := []byte("test-content") 113 | outputID := []byte("test-output-id") 114 | size := int64(len(content)) 115 | md5sum := md5.Sum(content) 116 | sha256sum := sha256.Sum256(content) 117 | compressionAlgorithm := "test-compression-algorithm" 118 | uncompressedSize := int64(len(content)) * 2 119 | 120 | remoteStorage.EXPECT().Put(gomock.Any(), &putRequestMatcher{ 121 | putRequest: &cacheprog.PutRequest{ 122 | ActionID: actionID, 123 | OutputID: outputID, 124 | Size: size, 125 | MD5Sum: md5sum[:], 126 | Sha256Sum: sha256sum[:], 127 | CompressionAlgorithm: compressionAlgorithm, 128 | UncompressedSize: uncompressedSize, 129 | }, 130 | body: content, 131 | }).Return(&cacheprog.PutResponse{}, nil) 132 | 133 | req, err := storage.NewPutRequest(context.Background(), server.URL, &cacheprog.PutRequest{ 134 | ActionID: actionID, 135 | OutputID: outputID, 136 | Size: size, 137 | Body: bytes.NewReader(content), 138 | MD5Sum: md5sum[:], 139 | Sha256Sum: sha256sum[:], 140 | CompressionAlgorithm: compressionAlgorithm, 141 | UncompressedSize: uncompressedSize, 142 | }) 143 | require.NoError(t, err) 144 | 145 | resp, err := http.DefaultClient.Do(req) 146 | t.Cleanup(func() { 147 | if resp != nil && resp.Body != nil { 148 | resp.Body.Close() 149 | } 150 | }) 151 | require.NoError(t, err) 152 | require.Equal(t, http.StatusOK, resp.StatusCode) 153 | }) 154 | } 155 | 156 | type putRequestMatcher struct { 157 | putRequest *cacheprog.PutRequest 158 | body []byte 159 | } 160 | 161 | func (m *putRequestMatcher) Matches(x any) bool { 162 | req, ok := x.(*cacheprog.PutRequest) 163 | if !ok { 164 | return false 165 | } 166 | 167 | metaMatches := bytes.Equal(req.ActionID, m.putRequest.ActionID) && 168 | bytes.Equal(req.OutputID, m.putRequest.OutputID) && 169 | req.Size == m.putRequest.Size && 170 | bytes.Equal(req.MD5Sum, m.putRequest.MD5Sum) && 171 | bytes.Equal(req.Sha256Sum, m.putRequest.Sha256Sum) && 172 | req.CompressionAlgorithm == m.putRequest.CompressionAlgorithm && 173 | req.UncompressedSize == m.putRequest.UncompressedSize 174 | 175 | bodyBytes, err := io.ReadAll(req.Body) 176 | if err != nil { 177 | return false 178 | } 179 | 180 | return metaMatches && bytes.Equal(bodyBytes, m.body) 181 | } 182 | 183 | func (m *putRequestMatcher) String() string { 184 | return fmt.Sprintf("putRequest: %v, body: %v", m.putRequest, m.body) 185 | } 186 | -------------------------------------------------------------------------------- /.github/workflows/pull_request.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | push: 8 | branches: 9 | - main 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | lint: 16 | name: Linter 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - name: Checkout code 21 | uses: actions/checkout@v4 22 | 23 | - name: Set up Go 24 | uses: actions/setup-go@v5 25 | with: 26 | go-version: '1.25' 27 | cache: true 28 | cache-dependency-path: | 29 | go.sum 30 | functests/go.sum 31 | 32 | - name: Run golangci-lint 33 | uses: golangci/golangci-lint-action@v7 34 | with: 35 | version: v2.6 36 | 37 | unit-tests: 38 | name: Unit Tests 39 | runs-on: ubuntu-latest 40 | 41 | steps: 42 | - name: Checkout code 43 | uses: actions/checkout@v4 44 | 45 | - name: Set up Go 46 | uses: actions/setup-go@v5 47 | with: 48 | go-version: '1.25' 49 | cache: true 50 | cache-dependency-path: | 51 | go.sum 52 | functests/go.sum 53 | 54 | - name: Install gotestsum 55 | run: go install gotest.tools/gotestsum@latest 56 | 57 | - name: Create coverage directory 58 | run: mkdir -p coverage 59 | 60 | - name: Run unit tests 61 | run: | 62 | gotestsum \ 63 | --format testname \ 64 | --jsonfile unit-tests.json \ 65 | -- \ 66 | -race \ 67 | -covermode=atomic \ 68 | -coverprofile=coverage/unit.out \ 69 | -coverpkg=./... \ 70 | ./... 71 | 72 | - name: Upload coverage artifacts 73 | uses: actions/upload-artifact@v4 74 | with: 75 | name: unit-coverage 76 | path: coverage/unit.out 77 | 78 | - name: Upload test results 79 | uses: actions/upload-artifact@v4 80 | with: 81 | name: unit-test-results 82 | path: unit-tests.json 83 | 84 | e2e-tests: 85 | name: E2E Tests 86 | runs-on: ubuntu-latest 87 | 88 | steps: 89 | - name: Checkout code 90 | uses: actions/checkout@v4 91 | 92 | - name: Set up Go 93 | uses: actions/setup-go@v5 94 | with: 95 | go-version: '1.25' 96 | cache: true 97 | cache-dependency-path: | 98 | go.sum 99 | functests/go.sum 100 | 101 | - name: Install gotestsum 102 | run: go install gotest.tools/gotestsum@latest 103 | 104 | - name: Create coverage directory 105 | run: mkdir -p coverage/e2e 106 | 107 | - name: Run e2e tests 108 | working-directory: functests 109 | env: 110 | GOCOVERDIR: ${{ github.workspace }}/coverage/e2e 111 | run: | 112 | gotestsum \ 113 | --format testname \ 114 | --jsonfile ../e2e-tests.json \ 115 | -- \ 116 | -race \ 117 | -count=1 \ 118 | ./... 119 | 120 | - name: Convert coverage 121 | run: | 122 | if [ -d coverage/e2e ] && [ "$(ls -A coverage/e2e)" ]; then 123 | go tool covdata textfmt -i=coverage/e2e -o=coverage/e2e.out 124 | else 125 | touch coverage/e2e.out 126 | fi 127 | 128 | - name: Upload coverage artifacts 129 | uses: actions/upload-artifact@v4 130 | with: 131 | name: e2e-coverage 132 | path: coverage/e2e.out 133 | 134 | - name: Upload test results 135 | uses: actions/upload-artifact@v4 136 | with: 137 | name: e2e-test-results 138 | path: e2e-tests.json 139 | 140 | coverage: 141 | name: Publish Coverage 142 | runs-on: ubuntu-latest 143 | needs: [unit-tests, e2e-tests] 144 | permissions: 145 | contents: write 146 | pull-requests: write 147 | 148 | steps: 149 | - name: Checkout code 150 | uses: actions/checkout@v4 151 | 152 | - name: Create coverage directory 153 | run: mkdir -p coverage 154 | 155 | - name: Download unit test coverage 156 | uses: actions/download-artifact@v4 157 | with: 158 | name: unit-coverage 159 | path: coverage 160 | 161 | - name: Download e2e test coverage 162 | uses: actions/download-artifact@v4 163 | with: 164 | name: e2e-coverage 165 | path: coverage 166 | 167 | - name: Publish coverage report 168 | id: coverage 169 | uses: vladopajic/go-test-coverage@v2 170 | continue-on-error: true 171 | with: 172 | profile: coverage/unit.out,coverage/e2e.out 173 | git-token: ${{ github.ref_name == 'main' && secrets.GITHUB_TOKEN || '' }} 174 | git-branch: badges 175 | 176 | # Post coverage report as comment 177 | - name: Find pull request ID 178 | run: | 179 | PR_DATA=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ 180 | "https://api.github.com/repos/${{ github.repository }}/pulls?head=${{ github.repository_owner }}:${{ github.head_ref }}&state=open") 181 | PR_ID=$(echo "$PR_DATA" | jq -r '.[0].number') 182 | 183 | if [ "$PR_ID" != "null" ]; then 184 | echo "pull_request_id=$PR_ID" >> $GITHUB_ENV 185 | else 186 | echo "No open pull request found for this branch." 187 | fi 188 | - name: Post coverage report 189 | if: env.pull_request_id 190 | uses: thollander/actions-comment-pull-request@v3 191 | with: 192 | github-token: ${{ secrets.GITHUB_TOKEN }} 193 | comment-tag: coverage-report 194 | pr-number: ${{ env.pull_request_id }} 195 | message: | 196 | go-test-coverage report: 197 | ``` 198 | ${{ fromJSON(steps.coverage.outputs.report) }}``` 199 | 200 | - name: Finally check coverage 201 | if: steps.coverage.outcome == 'failure' 202 | shell: bash 203 | run: echo "coverage check failed" && exit 1 204 | 205 | -------------------------------------------------------------------------------- /internal/infra/storage/http_contract.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "encoding/hex" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | "path" 11 | "strconv" 12 | "strings" 13 | "time" 14 | 15 | "github.com/platacard/cacheprog/internal/app/cacheprog" 16 | ) 17 | 18 | const ( 19 | ActionIDKey = "actionID" 20 | 21 | CachePath = "cache" 22 | 23 | HeaderPrefix = "X-Cacheprog-" 24 | OutputIDHeader = HeaderPrefix + "OutputID" 25 | MD5SumHeader = HeaderPrefix + "MD5Sum" 26 | Sha256SumHeader = HeaderPrefix + "Sha256Sum" 27 | CompressionAlgorithmHeader = HeaderPrefix + "CompressionAlgorithm" 28 | UncompressedSizeHeader = HeaderPrefix + "UncompressedSize" 29 | ) 30 | 31 | func ParsePutRequest(r *http.Request) (*cacheprog.PutRequest, error) { 32 | actionID := r.PathValue(ActionIDKey) 33 | if actionID == "" { 34 | return nil, errors.New("actionID is required") 35 | } 36 | 37 | actionIDBytes, err := hex.DecodeString(actionID) 38 | if err != nil { 39 | return nil, fmt.Errorf("failed to decode actionID: %w", err) 40 | } 41 | 42 | putRequest := &cacheprog.PutRequest{ 43 | ActionID: actionIDBytes, 44 | } 45 | 46 | putRequest.Size = r.ContentLength 47 | if putRequest.Size < 0 { 48 | return nil, errors.New("content-length is unknown") 49 | } 50 | 51 | putRequest.OutputID, err = hex.DecodeString(r.Header.Get(OutputIDHeader)) 52 | if err != nil { 53 | return nil, fmt.Errorf("failed to parse %s header: %w", OutputIDHeader, err) 54 | } 55 | putRequest.MD5Sum, err = hex.DecodeString(r.Header.Get(MD5SumHeader)) 56 | if err != nil { 57 | return nil, fmt.Errorf("failed to parse %s header: %w", MD5SumHeader, err) 58 | } 59 | putRequest.Sha256Sum, err = hex.DecodeString(r.Header.Get(Sha256SumHeader)) 60 | if err != nil { 61 | return nil, fmt.Errorf("failed to parse %s header: %w", Sha256SumHeader, err) 62 | } 63 | putRequest.CompressionAlgorithm = r.Header.Get(CompressionAlgorithmHeader) 64 | putRequest.UncompressedSize, err = strconv.ParseInt(r.Header.Get(UncompressedSizeHeader), 10, 64) 65 | if err != nil { 66 | return nil, fmt.Errorf("failed to parse %s header: %w", UncompressedSizeHeader, err) 67 | } 68 | 69 | putRequest.Body = r.Body 70 | 71 | return putRequest, nil 72 | } 73 | 74 | func NewPutRequest(ctx context.Context, baseURL string, putRequest *cacheprog.PutRequest) (*http.Request, error) { 75 | req, err := http.NewRequestWithContext(ctx, http.MethodPut, addPathToURL(baseURL, CachePath, hex.EncodeToString(putRequest.ActionID)), nil) 76 | if err != nil { 77 | return nil, fmt.Errorf("failed to create request: %w", err) 78 | } 79 | 80 | req.ContentLength = putRequest.Size 81 | if putRequest.Size == 0 { 82 | req.Body = http.NoBody 83 | } else { 84 | req.Body = io.NopCloser(putRequest.Body) 85 | } 86 | 87 | req.Header.Set(OutputIDHeader, hex.EncodeToString(putRequest.OutputID)) 88 | req.Header.Set(MD5SumHeader, hex.EncodeToString(putRequest.MD5Sum)) 89 | req.Header.Set(Sha256SumHeader, hex.EncodeToString(putRequest.Sha256Sum)) 90 | req.Header.Set(CompressionAlgorithmHeader, putRequest.CompressionAlgorithm) 91 | req.Header.Set(UncompressedSizeHeader, strconv.FormatInt(putRequest.UncompressedSize, 10)) 92 | 93 | return req, nil 94 | } 95 | 96 | func ParseGetRequest(r *http.Request) (*cacheprog.GetRequest, error) { 97 | actionID := r.PathValue(ActionIDKey) 98 | if actionID == "" { 99 | return nil, errors.New("actionID is required") 100 | } 101 | 102 | actionIDBytes, err := hex.DecodeString(actionID) 103 | if err != nil { 104 | return nil, fmt.Errorf("failed to decode actionID: %w", err) 105 | } 106 | 107 | return &cacheprog.GetRequest{ActionID: actionIDBytes}, nil 108 | } 109 | 110 | func NewGetRequest(ctx context.Context, baseURL string, getRequest *cacheprog.GetRequest) (*http.Request, error) { 111 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, addPathToURL(baseURL, CachePath, hex.EncodeToString(getRequest.ActionID)), nil) 112 | if err != nil { 113 | return nil, fmt.Errorf("failed to create request: %w", err) 114 | } 115 | 116 | return req, nil 117 | } 118 | 119 | func WriteGetResponse(w http.ResponseWriter, getResponse *cacheprog.GetResponse) error { 120 | defer getResponse.Body.Close() 121 | w.Header().Set("Content-Length", strconv.FormatInt(getResponse.Size, 10)) 122 | w.Header().Set("Last-Modified", getResponse.ModTime.Format(http.TimeFormat)) 123 | w.Header().Set(OutputIDHeader, hex.EncodeToString(getResponse.OutputID)) 124 | w.Header().Set(CompressionAlgorithmHeader, getResponse.CompressionAlgorithm) 125 | w.Header().Set(UncompressedSizeHeader, strconv.FormatInt(getResponse.UncompressedSize, 10)) 126 | w.WriteHeader(http.StatusOK) 127 | _, err := io.Copy(w, getResponse.Body) 128 | return err 129 | } 130 | 131 | func ParseGetResponse(r *http.Response) (*cacheprog.GetResponse, error) { 132 | if r.StatusCode == http.StatusNotFound { 133 | return nil, cacheprog.ErrNotFound 134 | } 135 | 136 | if r.StatusCode != http.StatusOK { 137 | body, _ := io.ReadAll(r.Body) 138 | return nil, fmt.Errorf("unexpected status code: %d, body: %s", r.StatusCode, string(body)) 139 | } 140 | 141 | if r.ContentLength < 0 { 142 | return nil, errors.New("content-length is unknown") 143 | } 144 | 145 | var ( 146 | getResponse cacheprog.GetResponse 147 | err error 148 | ) 149 | 150 | getResponse.OutputID, err = hex.DecodeString(r.Header.Get(OutputIDHeader)) 151 | if err != nil { 152 | return nil, fmt.Errorf("failed to parse %s header: %w", OutputIDHeader, err) 153 | } 154 | getResponse.ModTime, err = time.Parse(http.TimeFormat, r.Header.Get("Last-Modified")) 155 | if err != nil { 156 | return nil, fmt.Errorf("failed to parse %s header: %w", "Last-Modified", err) 157 | } 158 | getResponse.CompressionAlgorithm = r.Header.Get(CompressionAlgorithmHeader) 159 | getResponse.UncompressedSize, err = strconv.ParseInt(r.Header.Get(UncompressedSizeHeader), 10, 64) 160 | if err != nil { 161 | return nil, fmt.Errorf("failed to parse %s header: %w", UncompressedSizeHeader, err) 162 | } 163 | getResponse.Size = r.ContentLength 164 | 165 | getResponse.Body = r.Body 166 | 167 | return &getResponse, nil 168 | } 169 | 170 | func addPathToURL(baseURL string, pathItems ...string) string { 171 | return strings.TrimSuffix(baseURL, "/") + "/" + path.Join(pathItems...) 172 | } 173 | -------------------------------------------------------------------------------- /internal/infra/cacheproto/reader_test.go: -------------------------------------------------------------------------------- 1 | package cacheproto 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "encoding/json" 7 | "io" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestReader_ReadRequest(t *testing.T) { 15 | t.Run("get request", func(t *testing.T) { 16 | input := `{"ID":1,"Command":"get","ActionID":"YWJjMTIz"}` + "\n\n" 17 | reader := NewReader(bytes.NewBufferString(input)) 18 | 19 | req, err := reader.ReadRequest() 20 | require.NoError(t, err) 21 | 22 | assert.Equal(t, int64(1), req.ID) 23 | assert.Equal(t, CmdGet, req.Command) 24 | assert.Equal(t, []byte("abc123"), req.ActionID) 25 | assert.Nil(t, req.Body) 26 | assert.Zero(t, req.BodySize) 27 | }) 28 | 29 | t.Run("put request with body", func(t *testing.T) { 30 | body := []byte("hello world") 31 | bodyBase64 := base64.StdEncoding.EncodeToString(body) 32 | 33 | req := Request{ 34 | ID: 2, 35 | Command: CmdPut, 36 | ActionID: []byte("def456"), 37 | BodySize: int64(len(body)), 38 | } 39 | reqJSON, err := json.Marshal(req) 40 | require.NoError(t, err) 41 | 42 | input := string(reqJSON) + "\n\n\"" + bodyBase64 + "\"\n" 43 | reader := NewReader(bytes.NewBufferString(input)) 44 | 45 | gotReq, err := reader.ReadRequest() 46 | require.NoError(t, err) 47 | 48 | assert.Equal(t, int64(2), gotReq.ID) 49 | assert.Equal(t, CmdPut, gotReq.Command) 50 | assert.Equal(t, []byte("def456"), gotReq.ActionID) 51 | assert.Equal(t, int64(len(body)), gotReq.BodySize) 52 | 53 | // Read and verify body 54 | gotBody, err := io.ReadAll(gotReq.Body) 55 | require.NoError(t, err) 56 | assert.Equal(t, body, gotBody) 57 | }) 58 | 59 | t.Run("put request with empty body", func(t *testing.T) { 60 | input := `{"ID":3,"Command":"put","ActionID":"Z2hpNzg5","BodySize":0}` + "\n\n" 61 | reader := NewReader(bytes.NewBufferString(input)) 62 | 63 | req, err := reader.ReadRequest() 64 | require.NoError(t, err) 65 | 66 | assert.Equal(t, int64(3), req.ID) 67 | assert.Equal(t, CmdPut, req.Command) 68 | assert.Equal(t, []byte("ghi789"), req.ActionID) 69 | assert.Equal(t, int64(0), req.BodySize) 70 | 71 | // Verify empty body 72 | gotBody, err := io.ReadAll(req.Body) 73 | require.NoError(t, err) 74 | assert.Empty(t, gotBody) 75 | }) 76 | 77 | t.Run("error cases", func(t *testing.T) { 78 | tests := []struct { 79 | name string 80 | input string 81 | want string 82 | }{ 83 | { 84 | name: "invalid json", 85 | input: "invalid json\n\n", 86 | want: "unmarshal request:", 87 | }, 88 | { 89 | name: "missing newline after json", 90 | input: `{"ID":1,"Command":"get"}`, 91 | want: "read line:", 92 | }, 93 | { 94 | name: "missing empty line", 95 | input: `{"ID":1,"Command":"get"}` + "\n", 96 | want: "read empty line:", 97 | }, 98 | { 99 | name: "invalid body start for put", 100 | input: `{"ID":1,"Command":"put","BodySize":1}` + "\n\nX", 101 | want: "unexpected body begin:", 102 | }, 103 | } 104 | 105 | for _, tt := range tests { 106 | t.Run(tt.name, func(t *testing.T) { 107 | reader := NewReader(bytes.NewBufferString(tt.input)) 108 | _, err := reader.ReadRequest() 109 | assert.ErrorContains(t, err, tt.want) 110 | }) 111 | } 112 | }) 113 | } 114 | 115 | func FuzzReader_ReadRequest(f *testing.F) { 116 | // Add initial corpus 117 | f.Add([]byte(`{"ID":1,"Command":"get","ActionID":"abc123"}`)) 118 | f.Add([]byte(`{"ID":2,"Command":"put","ActionID":"def456","BodySize":11}`)) 119 | f.Add([]byte(`{"ID":3,"Command":"close"}`)) 120 | 121 | f.Fuzz(func(t *testing.T, data []byte) { 122 | // Skip empty input 123 | if len(data) == 0 { 124 | return 125 | } 126 | 127 | // Create input with proper line endings 128 | input := string(data) + "\n\n" 129 | 130 | // If it looks like a put request with body, try to add a valid body 131 | if bytes.Contains(data, []byte(`"Command":"put"`)) && bytes.Contains(data, []byte(`"BodySize"`)) { 132 | // Add a base64-encoded body 133 | body := []byte("test body") 134 | bodyBase64 := base64.StdEncoding.EncodeToString(body) 135 | input = input + "\"" + bodyBase64 + "\"\n" 136 | } 137 | 138 | reader := NewReader(bytes.NewBufferString(input)) 139 | req, err := reader.ReadRequest() 140 | 141 | // If parsing succeeds, verify basic invariants 142 | if err == nil { 143 | // Verify ID is not negative 144 | if req.ID < 0 { 145 | t.Errorf("negative ID: %d", req.ID) 146 | } 147 | 148 | // Verify Command is one of the valid commands 149 | validCmd := req.Command == CmdGet || req.Command == CmdPut || req.Command == CmdClose 150 | if !validCmd { 151 | t.Errorf("invalid command: %s", req.Command) 152 | } 153 | 154 | // For put requests with body, verify we can read the body 155 | if req.Command == CmdPut && req.BodySize > 0 { 156 | if req.Body == nil { 157 | t.Error("body is nil for put request with BodySize > 0") 158 | } else { 159 | body, err := io.ReadAll(req.Body) 160 | if err != nil { 161 | t.Errorf("failed to read body: %v", err) 162 | } 163 | if int64(len(body)) != req.BodySize { 164 | t.Errorf("body size mismatch: got %d, want %d", len(body), req.BodySize) 165 | } 166 | } 167 | } 168 | } 169 | }) 170 | } 171 | 172 | // Add a more targeted fuzzer for base64-encoded bodies 173 | func FuzzReader_ReadRequestBody(f *testing.F) { 174 | // Add initial corpus 175 | f.Add([]byte("hello world"), int64(11)) 176 | f.Add([]byte{}, int64(0)) 177 | f.Add([]byte{0xFF, 0x00, 0xFF}, int64(3)) 178 | 179 | f.Fuzz(func(t *testing.T, body []byte, size int64) { 180 | // Skip invalid size 181 | if size < 0 || size != int64(len(body)) { 182 | return 183 | } 184 | 185 | // Create a valid put request with the fuzzed body 186 | req := Request{ 187 | ID: 1, 188 | Command: CmdPut, 189 | ActionID: []byte("test"), 190 | BodySize: size, 191 | } 192 | reqJSON, err := json.Marshal(req) 193 | if err != nil { 194 | t.Skip() 195 | } 196 | 197 | // Encode body in base64 198 | bodyBase64 := base64.StdEncoding.EncodeToString(body) 199 | input := string(reqJSON) + "\n\n\"" + bodyBase64 + "\"\n" 200 | 201 | reader := NewReader(bytes.NewBufferString(input)) 202 | gotReq, err := reader.ReadRequest() 203 | if err != nil { 204 | return // It's okay if some inputs are rejected 205 | } 206 | 207 | // Verify the body was correctly read 208 | gotBody, err := io.ReadAll(gotReq.Body) 209 | if err != nil { 210 | t.Errorf("failed to read body: %v", err) 211 | } 212 | if !bytes.Equal(gotBody, body) { 213 | t.Errorf("body mismatch: got %v, want %v", gotBody, body) 214 | } 215 | if int64(len(gotBody)) != size { 216 | t.Errorf("body size mismatch: got %d, want %d", len(gotBody), size) 217 | } 218 | }) 219 | } 220 | -------------------------------------------------------------------------------- /internal/app/app.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "log/slog" 9 | "net/http" 10 | "net/url" 11 | "os" 12 | "runtime" 13 | "runtime/debug" 14 | "slices" 15 | "strings" 16 | 17 | "github.com/KimMachineGun/automemlimit/memlimit" 18 | arg "github.com/alexflint/go-arg" 19 | 20 | "github.com/platacard/cacheprog/internal/infra/debugging" 21 | "github.com/platacard/cacheprog/internal/infra/logging" 22 | "github.com/platacard/cacheprog/internal/infra/metrics" 23 | ) 24 | 25 | type Args struct { 26 | version string 27 | 28 | DebugArgs 29 | LoggingArgs 30 | 31 | CacheprogApp *CacheprogAppArgs `arg:"subcommand:direct" help:"[default] run as GOCACHEPROG program"` 32 | ProxyApp *ProxyAppArgs `arg:"subcommand:proxy" help:"run as storage and optionally metrics proxy"` 33 | 34 | UseVMHistograms bool `arg:"--use-vm-histograms,env:USE_VM_HISTOGRAMS" placeholder:"true/false" help:"Use VictoriaMetrics-style histograms instead of Prometheus-style histograms"` 35 | } 36 | 37 | type DebugArgs struct { 38 | CPUProfilePath string `arg:"--cpu-profile,env:CPU_PROFILE_PATH" placeholder:"PATH" help:"Path to write CPU profile"` 39 | MemProfilePath string `arg:"--mem-profile,env:MEM_PROFILE_PATH" placeholder:"PATH" help:"Path to write memory profile"` 40 | TraceProfilePath string `arg:"--trace-profile,env:TRACE_PROFILE_PATH" placeholder:"PATH" help:"Path to write trace profile"` 41 | FgprofPath string `arg:"--fgprof,env:FGPROF_PATH" placeholder:"PATH" help:"Path to write fgprof profile"` 42 | } 43 | 44 | func (d *DebugArgs) configureDebugging() (func() error, error) { 45 | return debugging.ConfigureDebugging(debugging.DebugConfig{ 46 | CPUProfilePath: d.CPUProfilePath, 47 | MemProfilePath: d.MemProfilePath, 48 | TraceProfilePath: d.TraceProfilePath, 49 | FgprofPath: d.FgprofPath, 50 | }) 51 | } 52 | 53 | type LoggingArgs struct { 54 | Level slog.Level `arg:"--log-level,env:LOG_LEVEL" placeholder:"LEVEL" default:"INFO" help:"Logging level. Available: DEBUG, INFO, WARN, ERROR"` 55 | Output string `arg:"--log-output,env:LOG_OUTPUT" placeholder:"PATH" help:"Logging output. Path to file, stderr if not provided"` 56 | } 57 | 58 | func (l *LoggingArgs) createLogger() (*slog.Logger, func() error, error) { 59 | if l.Output == "" { 60 | return logging.ConfigureLogger(logging.Config{ 61 | Level: l.Level, 62 | Output: os.Stderr, 63 | }), func() error { return nil }, nil 64 | } 65 | 66 | outFile, err := os.OpenFile(l.Output, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) 67 | if err != nil { 68 | return nil, nil, fmt.Errorf("failed to open log output file: %w", err) 69 | } 70 | 71 | return logging.ConfigureLogger(logging.Config{ 72 | Level: l.Level, 73 | Output: outFile, 74 | }), 75 | func() error { 76 | return errors.Join(outFile.Sync(), outFile.Close()) 77 | }, 78 | nil 79 | } 80 | 81 | func (a *Args) Run(ctx context.Context) error { 82 | metrics.EnableVMHistograms(a.UseVMHistograms) 83 | 84 | logger, stopLogger, err := a.createLogger() 85 | if err != nil { 86 | return fmt.Errorf("failed to create logger: %w", err) 87 | } 88 | defer stopLogger() //nolint:errcheck // error handling is not important here 89 | 90 | slog.SetDefault(logger) 91 | 92 | if _, err := memlimit.SetGoMemLimitWithOpts(memlimit.WithLogger(logger)); err != nil { 93 | slog.WarnContext(ctx, "Failed to configure memory limit", logging.Error(err)) 94 | } 95 | 96 | stopDebugging, err := a.configureDebugging() 97 | if err != nil { 98 | return fmt.Errorf("failed to configure debugging: %w", err) 99 | } 100 | defer stopDebugging() //nolint:errcheck // error handling is not important here 101 | 102 | switch { 103 | case a.CacheprogApp != nil: 104 | return a.CacheprogApp.Run(ctx) 105 | case a.ProxyApp != nil: 106 | return a.ProxyApp.Run(ctx) 107 | default: 108 | return fmt.Errorf("no command provided") 109 | } 110 | } 111 | 112 | func (a *Args) Version() string { 113 | var sb strings.Builder 114 | sb.WriteString("cacheprog version ") 115 | sb.WriteString(a.version) 116 | 117 | if runtimeVersion := runtime.Version(); runtimeVersion != "" { 118 | _, _ = fmt.Fprintf(&sb, " built with %s", runtimeVersion) 119 | } 120 | 121 | var vcsType, vcsRevision, vcsTime string 122 | if buildInfo, ok := debug.ReadBuildInfo(); ok { 123 | for _, setting := range buildInfo.Settings { 124 | if setting.Key == "vcs" { 125 | vcsType = setting.Value 126 | } 127 | if setting.Key == "vcs.revision" { 128 | vcsRevision = setting.Value 129 | } 130 | if setting.Key == "vcs.time" { 131 | vcsTime = setting.Value 132 | } 133 | } 134 | } 135 | 136 | // for "git" vcs clip revision to 7 characters (default git behavior) 137 | if vcsRevision != "" { 138 | if vcsType == "git" { 139 | vcsRevision = vcsRevision[:7] 140 | } 141 | _, _ = fmt.Fprintf(&sb, " from %s", vcsRevision) 142 | } 143 | if vcsTime != "" { 144 | _, _ = fmt.Fprintf(&sb, " on %s", vcsTime) 145 | } 146 | return sb.String() 147 | } 148 | 149 | func RunApp(ctx context.Context, version string, args ...string) int { 150 | appArgs := Args{version: version} 151 | 152 | parser, err := arg.NewParser(arg.Config{ 153 | EnvPrefix: "CACHEPROG_", 154 | Out: os.Stderr, 155 | }, &appArgs) 156 | if err != nil { 157 | fmt.Println(err) 158 | return 2 159 | } 160 | 161 | // unfortunately, arg package does not support 'root' subcommand, so do following: 162 | // if we didn't found any subcommand name in args, then we need to run 'direct' subcommand 163 | // by forcefully appending it to args and parse again 164 | err = parser.Parse(args) 165 | if err == nil { 166 | subcommandNames := parser.SubcommandNames() 167 | subcommandNamesMap := make(map[string]struct{}, len(subcommandNames)) 168 | for _, name := range subcommandNames { 169 | subcommandNamesMap[name] = struct{}{} 170 | } 171 | if !slices.ContainsFunc(args, func(arg string) bool { 172 | _, ok := subcommandNamesMap[arg] 173 | return ok 174 | }) { 175 | args = append([]string{"direct"}, args...) 176 | } 177 | 178 | err = parser.Parse(args) 179 | } 180 | 181 | switch { 182 | case errors.Is(err, nil): 183 | // pass 184 | case errors.Is(err, arg.ErrHelp): 185 | parser.WriteHelp(os.Stderr) 186 | return 0 187 | case errors.Is(err, arg.ErrVersion): 188 | fmt.Printf("%s\n", appArgs.Version()) 189 | return 0 190 | default: 191 | fmt.Fprintln(os.Stderr, err) 192 | parser.WriteHelp(os.Stderr) 193 | return 1 194 | } 195 | 196 | if err = appArgs.Run(ctx); err != nil { 197 | fmt.Fprintln(os.Stderr, err) 198 | return 1 199 | } 200 | 201 | return 0 202 | } 203 | 204 | func urlOrEmpty(u *url.URL) string { 205 | if u == nil { 206 | return "" 207 | } 208 | return u.String() 209 | } 210 | 211 | type httpHeader map[string]string 212 | 213 | func (h *httpHeader) UnmarshalText(text []byte) error { 214 | if *h == nil { 215 | *h = make(map[string]string) 216 | } 217 | k, v, found := bytes.Cut(text, []byte(":")) 218 | if !found { 219 | return fmt.Errorf("invalid http header: %s", string(text)) 220 | } 221 | (*h)[string(k)] = string(v) 222 | return nil 223 | } 224 | 225 | func headerValuesToHTTP(hs []httpHeader) http.Header { 226 | hh := make(http.Header) 227 | for _, h := range hs { 228 | for k, v := range h { 229 | hh.Add(k, v) 230 | } 231 | } 232 | return hh 233 | } 234 | -------------------------------------------------------------------------------- /internal/app/cacheprog/mocks_world_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: handler.go 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -destination=mocks_world_test.go -package=cacheprog -source=handler.go 7 | // 8 | 9 | // Package cacheprog is a generated GoMock package. 10 | package cacheprog 11 | 12 | import ( 13 | context "context" 14 | reflect "reflect" 15 | 16 | gomock "go.uber.org/mock/gomock" 17 | ) 18 | 19 | // MockRemoteStorage is a mock of RemoteStorage interface. 20 | type MockRemoteStorage struct { 21 | ctrl *gomock.Controller 22 | recorder *MockRemoteStorageMockRecorder 23 | isgomock struct{} 24 | } 25 | 26 | // MockRemoteStorageMockRecorder is the mock recorder for MockRemoteStorage. 27 | type MockRemoteStorageMockRecorder struct { 28 | mock *MockRemoteStorage 29 | } 30 | 31 | // NewMockRemoteStorage creates a new mock instance. 32 | func NewMockRemoteStorage(ctrl *gomock.Controller) *MockRemoteStorage { 33 | mock := &MockRemoteStorage{ctrl: ctrl} 34 | mock.recorder = &MockRemoteStorageMockRecorder{mock} 35 | return mock 36 | } 37 | 38 | // EXPECT returns an object that allows the caller to indicate expected use. 39 | func (m *MockRemoteStorage) EXPECT() *MockRemoteStorageMockRecorder { 40 | return m.recorder 41 | } 42 | 43 | // Get mocks base method. 44 | func (m *MockRemoteStorage) Get(ctx context.Context, request *GetRequest) (*GetResponse, error) { 45 | m.ctrl.T.Helper() 46 | ret := m.ctrl.Call(m, "Get", ctx, request) 47 | ret0, _ := ret[0].(*GetResponse) 48 | ret1, _ := ret[1].(error) 49 | return ret0, ret1 50 | } 51 | 52 | // Get indicates an expected call of Get. 53 | func (mr *MockRemoteStorageMockRecorder) Get(ctx, request any) *gomock.Call { 54 | mr.mock.ctrl.T.Helper() 55 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockRemoteStorage)(nil).Get), ctx, request) 56 | } 57 | 58 | // Put mocks base method. 59 | func (m *MockRemoteStorage) Put(ctx context.Context, request *PutRequest) (*PutResponse, error) { 60 | m.ctrl.T.Helper() 61 | ret := m.ctrl.Call(m, "Put", ctx, request) 62 | ret0, _ := ret[0].(*PutResponse) 63 | ret1, _ := ret[1].(error) 64 | return ret0, ret1 65 | } 66 | 67 | // Put indicates an expected call of Put. 68 | func (mr *MockRemoteStorageMockRecorder) Put(ctx, request any) *gomock.Call { 69 | mr.mock.ctrl.T.Helper() 70 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockRemoteStorage)(nil).Put), ctx, request) 71 | } 72 | 73 | // MockLocalStorage is a mock of LocalStorage interface. 74 | type MockLocalStorage struct { 75 | ctrl *gomock.Controller 76 | recorder *MockLocalStorageMockRecorder 77 | isgomock struct{} 78 | } 79 | 80 | // MockLocalStorageMockRecorder is the mock recorder for MockLocalStorage. 81 | type MockLocalStorageMockRecorder struct { 82 | mock *MockLocalStorage 83 | } 84 | 85 | // NewMockLocalStorage creates a new mock instance. 86 | func NewMockLocalStorage(ctrl *gomock.Controller) *MockLocalStorage { 87 | mock := &MockLocalStorage{ctrl: ctrl} 88 | mock.recorder = &MockLocalStorageMockRecorder{mock} 89 | return mock 90 | } 91 | 92 | // EXPECT returns an object that allows the caller to indicate expected use. 93 | func (m *MockLocalStorage) EXPECT() *MockLocalStorageMockRecorder { 94 | return m.recorder 95 | } 96 | 97 | // GetLocal mocks base method. 98 | func (m *MockLocalStorage) GetLocal(ctx context.Context, request *LocalGetRequest) (*LocalGetResponse, error) { 99 | m.ctrl.T.Helper() 100 | ret := m.ctrl.Call(m, "GetLocal", ctx, request) 101 | ret0, _ := ret[0].(*LocalGetResponse) 102 | ret1, _ := ret[1].(error) 103 | return ret0, ret1 104 | } 105 | 106 | // GetLocal indicates an expected call of GetLocal. 107 | func (mr *MockLocalStorageMockRecorder) GetLocal(ctx, request any) *gomock.Call { 108 | mr.mock.ctrl.T.Helper() 109 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLocal", reflect.TypeOf((*MockLocalStorage)(nil).GetLocal), ctx, request) 110 | } 111 | 112 | // GetLocalObject mocks base method. 113 | func (m *MockLocalStorage) GetLocalObject(ctx context.Context, request *LocalObjectGetRequest) (*LocalObjectGetResponse, error) { 114 | m.ctrl.T.Helper() 115 | ret := m.ctrl.Call(m, "GetLocalObject", ctx, request) 116 | ret0, _ := ret[0].(*LocalObjectGetResponse) 117 | ret1, _ := ret[1].(error) 118 | return ret0, ret1 119 | } 120 | 121 | // GetLocalObject indicates an expected call of GetLocalObject. 122 | func (mr *MockLocalStorageMockRecorder) GetLocalObject(ctx, request any) *gomock.Call { 123 | mr.mock.ctrl.T.Helper() 124 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLocalObject", reflect.TypeOf((*MockLocalStorage)(nil).GetLocalObject), ctx, request) 125 | } 126 | 127 | // PutLocal mocks base method. 128 | func (m *MockLocalStorage) PutLocal(ctx context.Context, request *LocalPutRequest) (*LocalPutResponse, error) { 129 | m.ctrl.T.Helper() 130 | ret := m.ctrl.Call(m, "PutLocal", ctx, request) 131 | ret0, _ := ret[0].(*LocalPutResponse) 132 | ret1, _ := ret[1].(error) 133 | return ret0, ret1 134 | } 135 | 136 | // PutLocal indicates an expected call of PutLocal. 137 | func (mr *MockLocalStorageMockRecorder) PutLocal(ctx, request any) *gomock.Call { 138 | mr.mock.ctrl.T.Helper() 139 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutLocal", reflect.TypeOf((*MockLocalStorage)(nil).PutLocal), ctx, request) 140 | } 141 | 142 | // MockCompressionCodec is a mock of CompressionCodec interface. 143 | type MockCompressionCodec struct { 144 | ctrl *gomock.Controller 145 | recorder *MockCompressionCodecMockRecorder 146 | isgomock struct{} 147 | } 148 | 149 | // MockCompressionCodecMockRecorder is the mock recorder for MockCompressionCodec. 150 | type MockCompressionCodecMockRecorder struct { 151 | mock *MockCompressionCodec 152 | } 153 | 154 | // NewMockCompressionCodec creates a new mock instance. 155 | func NewMockCompressionCodec(ctrl *gomock.Controller) *MockCompressionCodec { 156 | mock := &MockCompressionCodec{ctrl: ctrl} 157 | mock.recorder = &MockCompressionCodecMockRecorder{mock} 158 | return mock 159 | } 160 | 161 | // EXPECT returns an object that allows the caller to indicate expected use. 162 | func (m *MockCompressionCodec) EXPECT() *MockCompressionCodecMockRecorder { 163 | return m.recorder 164 | } 165 | 166 | // Compress mocks base method. 167 | func (m *MockCompressionCodec) Compress(req *CompressRequest) (*CompressResponse, error) { 168 | m.ctrl.T.Helper() 169 | ret := m.ctrl.Call(m, "Compress", req) 170 | ret0, _ := ret[0].(*CompressResponse) 171 | ret1, _ := ret[1].(error) 172 | return ret0, ret1 173 | } 174 | 175 | // Compress indicates an expected call of Compress. 176 | func (mr *MockCompressionCodecMockRecorder) Compress(req any) *gomock.Call { 177 | mr.mock.ctrl.T.Helper() 178 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Compress", reflect.TypeOf((*MockCompressionCodec)(nil).Compress), req) 179 | } 180 | 181 | // Decompress mocks base method. 182 | func (m *MockCompressionCodec) Decompress(req *DecompressRequest) (*DecompressResponse, error) { 183 | m.ctrl.T.Helper() 184 | ret := m.ctrl.Call(m, "Decompress", req) 185 | ret0, _ := ret[0].(*DecompressResponse) 186 | ret1, _ := ret[1].(error) 187 | return ret0, ret1 188 | } 189 | 190 | // Decompress indicates an expected call of Decompress. 191 | func (mr *MockCompressionCodecMockRecorder) Decompress(req any) *gomock.Call { 192 | mr.mock.ctrl.T.Helper() 193 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Decompress", reflect.TypeOf((*MockCompressionCodec)(nil).Decompress), req) 194 | } 195 | -------------------------------------------------------------------------------- /internal/app/app_cacheprog.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log/slog" 7 | "net/url" 8 | "os" 9 | "time" 10 | 11 | "github.com/platacard/cacheprog/internal/app/cacheprog" 12 | "github.com/platacard/cacheprog/internal/infra/cacheproto" 13 | "github.com/platacard/cacheprog/internal/infra/compression" 14 | "github.com/platacard/cacheprog/internal/infra/metrics" 15 | "github.com/platacard/cacheprog/internal/infra/storage" 16 | ) 17 | 18 | type CacheprogAppArgs struct { 19 | RemoteStorageArgs 20 | MetricsPushArgs 21 | 22 | RootDirectory string `arg:"--root-directory,env:ROOT_DIRECTORY" placeholder:"PATH" help:"Root directory to local storage of objects. Must be read-write accessible by user and read-accessible by 'go' compiler. If not provided, subdirectory in system temporary directory will be used."` 23 | MaxConcurrentRemoteGets int `arg:"--max-concurrent-remote-gets,env:MAX_CONCURRENT_REMOTE_GETS" placeholder:"NUM" help:"Max number of concurrent remote gets, unlimited if not provided"` 24 | MaxConcurrentRemotePuts int `arg:"--max-concurrent-remote-puts,env:MAX_CONCURRENT_REMOTE_PUTS" placeholder:"NUM" help:"Max number of concurrent remote puts, unlimited if not provided"` 25 | MaxBackgroundWait time.Duration `arg:"--max-background-wait,env:MAX_BACKGROUND_WAIT" placeholder:"DURATION" default:"10s" help:"Max time to wait for waiting of background operations to complete"` 26 | MinRemotePutSize int64 `arg:"--min-remote-put-size,env:MIN_REMOTE_PUT_SIZE" placeholder:"SIZE" help:"Min size of object to push to remote storage, no size limit if not provided"` 27 | DisableGet bool `arg:"--disable-get,env:DISABLE_GET" help:"Disable getting objects from any storage, useful to force rebuild of the project and rewrite cache"` 28 | } 29 | 30 | type RemoteStorageArgs struct { 31 | S3Args 32 | HTTPStorageArgs 33 | 34 | RemoteStorageType string `arg:"--remote-storage-type,env:REMOTE_STORAGE_TYPE" placeholder:"TYPE" default:"disabled" help:"Remote storage type. Available: s3, http, disabled"` 35 | } 36 | 37 | type S3Args struct { 38 | Endpoint *url.URL `arg:"--s3-endpoint,env:S3_ENDPOINT" placeholder:"URL" help:"Endpoint for S3-compatible storages, use schemes minio+http://, minio+https://, etc. for minio compatibility."` 39 | ForcePathStyle bool `arg:"--s3-force-path-style,env:S3_FORCE_PATH_STYLE" placeholder:"true/false" help:"Forces path style endpoints, useful for some S3-compatible storages."` 40 | Region string `arg:"--s3-region,env:S3_REGION" placeholder:"REGION" help:"S3 region name. If not provided will be detected automatically via GetBucketLocation API."` 41 | Bucket string `arg:"--s3-bucket,env:S3_BUCKET" placeholder:"BUCKET" help:"S3 bucket name."` 42 | Prefix string `arg:"--s3-prefix,env:S3_PREFIX" placeholder:"PREFIX" help:"Prefix for S3 keys, useful to run multiple apps on same bucket. Templated, GOOS, GOARCH and env. are available. Template format: {% GOOS %}"` 43 | Expiration time.Duration `arg:"--s3-expiration,env:S3_EXPIRATION" placeholder:"DURATION" help:"Sets expiration for each S3 object during Put, 0 - no expiration."` 44 | CredentialsEndpoint string `arg:"--s3-credentials-endpoint,env:S3_CREDENTIALS_ENDPOINT" placeholder:"URL" help:"Credentials endpoint for S3-compatible storages."` 45 | AccessKeyID string `arg:"--s3-access-key-id,env:S3_ACCESS_KEY_ID" placeholder:"ID" help:"S3 access key id."` 46 | AccessKeySecret string `arg:"--s3-access-key-secret,env:S3_ACCESS_KEY_SECRET" placeholder:"SECRET" help:"S3 access key secret."` 47 | SessionToken string `arg:"--s3-session-token,env:S3_SESSION_TOKEN" placeholder:"TOKEN" help:"S3 session token."` 48 | } 49 | 50 | type HTTPStorageArgs struct { 51 | BaseURL *url.URL `arg:"--http-storage-base-url,env:HTTP_STORAGE_BASE_URL" placeholder:"URL" help:"Base URL for HTTP storage."` 52 | ExtraHeaders []httpHeader `arg:"--http-storage-extra-headers,env:HTTP_STORAGE_EXTRA_HEADERS" placeholder:"[key:value]" help:"Extra headers to be added to each request."` 53 | } 54 | 55 | type MetricsPushArgs struct { 56 | Endpoint *url.URL `arg:"--metrics-push-endpoint,env:METRICS_PUSH_ENDPOINT" placeholder:"URL" help:"Metrics endpoint, metrics will be pushed if provided"` 57 | Method string `arg:"--metrics-push-method,env:METRICS_PUSH_METHOD" placeholder:"METHOD" default:"GET" help:"HTTP method to use for sending metrics"` 58 | ExtraLabels map[string]string `arg:"--metrics-push-extra-labels,env:METRICS_PUSH_EXTRA_LABELS" placeholder:"[key=value]" help:"Extra labels to be added to each metric, format: key=value"` 59 | ExtraHeaders []httpHeader `arg:"--metrics-push-extra-headers,env:METRICS_PUSH_EXTRA_HEADERS" placeholder:"[key:value]" help:"Extra headers to be added to each request."` 60 | } 61 | 62 | func (r *RemoteStorageArgs) configureRemoteStorage() (cacheprog.RemoteStorage, error) { 63 | switch r.RemoteStorageType { 64 | case "s3": 65 | return storage.ConfigureS3(storage.S3Config{ 66 | KeyPrefix: r.Prefix, 67 | Expiration: r.Expiration, 68 | Bucket: r.Bucket, 69 | Region: r.Region, 70 | Endpoint: urlOrEmpty(r.Endpoint), 71 | ForcePathStyle: r.ForcePathStyle, 72 | CredentialsEndpoint: r.CredentialsEndpoint, 73 | AccessKeyID: r.AccessKeyID, 74 | AccessKeySecret: r.AccessKeySecret, 75 | SessionToken: r.SessionToken, 76 | }) 77 | case "http": 78 | return storage.ConfigureHTTP(urlOrEmpty(r.BaseURL), headerValuesToHTTP(r.ExtraHeaders)) 79 | case "disabled": 80 | return nil, nil 81 | default: 82 | return nil, fmt.Errorf("invalid remote storage type: %s", r.RemoteStorageType) 83 | } 84 | } 85 | 86 | func (a *CacheprogAppArgs) Run(ctx context.Context) error { 87 | defer func() { 88 | if err := metrics.PushMetrics(ctx, metrics.PushConfig{ 89 | Endpoint: urlOrEmpty(a.Endpoint), 90 | ExtraLabels: a.ExtraLabels, 91 | ExtraHeaders: headerValuesToHTTP(a.ExtraHeaders), 92 | Method: a.Method, 93 | }); err != nil { 94 | slog.Warn("Failed to push metrics", "error", err) 95 | } 96 | }() 97 | 98 | defer metrics.ObserveOverallRunTime()() 99 | 100 | remoteStorage, err := a.configureRemoteStorage() 101 | if err != nil { 102 | return fmt.Errorf("failed to configure remote storage: %w", err) 103 | } 104 | 105 | if remoteStorage != nil { 106 | remoteStorage = cacheprog.ObservingRemoteStorage{RemoteStorage: remoteStorage} 107 | } 108 | 109 | diskStorage, err := storage.ConfigureDisk(a.RootDirectory) 110 | if err != nil { 111 | return fmt.Errorf("failed to configure disk storage: %w", err) 112 | } 113 | 114 | h := cacheprog.NewHandler(cacheprog.HandlerOptions{ 115 | RemoteStorage: remoteStorage, 116 | MaxConcurrentRemoteGets: a.MaxConcurrentRemoteGets, 117 | MaxConcurrentRemotePuts: a.MaxConcurrentRemotePuts, 118 | LocalStorage: cacheprog.ObservingLocalStorage{LocalStorage: diskStorage}, 119 | CloseTimeout: a.MaxBackgroundWait, 120 | CompressionCodec: compression.NewCodec(), 121 | DisableGet: a.DisableGet, 122 | }) 123 | defer func() { 124 | statistics := h.GetStatistics() 125 | slog.Info("cacheprog statistics", 126 | "get_calls", statistics.GetCalls, 127 | "get_hits", statistics.GetHits, 128 | "get_hit_ratio", fmt.Sprintf("%.2f", float64(statistics.GetHits)/float64(statistics.GetCalls)), 129 | "put_calls", statistics.PutCalls, 130 | ) 131 | }() 132 | 133 | server := cacheproto.NewServer(cacheproto.ServerOptions{ 134 | Reader: os.Stdin, 135 | Writer: os.Stdout, 136 | Handler: h, 137 | }) 138 | 139 | defer context.AfterFunc(ctx, server.Stop)() 140 | 141 | slog.Info("Starting cacheprog") 142 | 143 | return server.Run() 144 | } 145 | -------------------------------------------------------------------------------- /internal/infra/storage/disk_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/hex" 7 | "encoding/json" 8 | "fmt" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | "testing" 13 | "time" 14 | 15 | "github.com/stretchr/testify/assert" 16 | "github.com/stretchr/testify/require" 17 | "go.uber.org/mock/gomock" 18 | 19 | "github.com/platacard/cacheprog/internal/app/cacheprog" 20 | ) 21 | 22 | func TestNewDisk(t *testing.T) { 23 | testFileMatch := gomock.Cond(func(in string) bool { 24 | return strings.HasPrefix(in, ".cacheprog_test_") 25 | }) 26 | 27 | t.Run("success", func(t *testing.T) { 28 | ctrl := gomock.NewController(t) 29 | diskMock := NewMockDiskRoot(ctrl) 30 | 31 | testFileMock := NewMockDiskFile(ctrl) 32 | diskMock.EXPECT().OpenFile(testFileMatch, os.O_WRONLY|os.O_CREATE|os.O_EXCL, os.FileMode(0666)).Return(testFileMock, nil) 33 | testFileMock.EXPECT().Write(gomock.Any()).DoAndReturn(func(p []byte) (int, error) { 34 | return len(p), nil 35 | }).AnyTimes() 36 | testFileMock.EXPECT().Close().Return(nil) 37 | diskMock.EXPECT().Remove(testFileMatch).Return(nil) 38 | 39 | _, err := NewDisk(diskMock) 40 | require.NoError(t, err) 41 | }) 42 | 43 | t.Run("real disk", func(t *testing.T) { 44 | _, err := NewSystemDiskRoot(t.TempDir()) 45 | require.NoError(t, err) 46 | }) 47 | 48 | t.Run("retries on existing file", func(t *testing.T) { 49 | ctrl := gomock.NewController(t) 50 | diskMock := NewMockDiskRoot(ctrl) 51 | 52 | testFileMock := NewMockDiskFile(ctrl) 53 | diskMock.EXPECT().OpenFile(testFileMatch, os.O_WRONLY|os.O_CREATE|os.O_EXCL, os.FileMode(0666)).Return(nil, os.ErrExist) 54 | diskMock.EXPECT().OpenFile(testFileMatch, os.O_WRONLY|os.O_CREATE|os.O_EXCL, os.FileMode(0666)).Return(testFileMock, nil) 55 | testFileMock.EXPECT().Write(gomock.Any()).DoAndReturn(func(p []byte) (int, error) { 56 | return len(p), nil 57 | }).AnyTimes() 58 | testFileMock.EXPECT().Close().Return(nil) 59 | diskMock.EXPECT().Remove(testFileMatch).Return(nil) 60 | 61 | _, err := NewDisk(diskMock) 62 | require.NoError(t, err) 63 | }) 64 | 65 | t.Run("error on open file", func(t *testing.T) { 66 | ctrl := gomock.NewController(t) 67 | diskMock := NewMockDiskRoot(ctrl) 68 | 69 | diskMock.EXPECT().OpenFile(testFileMatch, os.O_WRONLY|os.O_CREATE|os.O_EXCL, os.FileMode(0666)).Return(nil, fmt.Errorf("open file error")) 70 | diskMock.EXPECT().Remove(testFileMatch).Return(nil) 71 | 72 | _, err := NewDisk(diskMock) 73 | require.Error(t, err) 74 | }) 75 | 76 | t.Run("error on write", func(t *testing.T) { 77 | ctrl := gomock.NewController(t) 78 | diskMock := NewMockDiskRoot(ctrl) 79 | 80 | testFileMock := NewMockDiskFile(ctrl) 81 | diskMock.EXPECT().OpenFile(testFileMatch, os.O_WRONLY|os.O_CREATE|os.O_EXCL, os.FileMode(0666)).Return(testFileMock, nil) 82 | testFileMock.EXPECT().Write(gomock.Any()).Return(0, fmt.Errorf("write error")) 83 | diskMock.EXPECT().Remove(testFileMatch).Return(nil) 84 | 85 | _, err := NewDisk(diskMock) 86 | require.Error(t, err) 87 | }) 88 | 89 | t.Run("error on close", func(t *testing.T) { 90 | ctrl := gomock.NewController(t) 91 | diskMock := NewMockDiskRoot(ctrl) 92 | 93 | testFileMock := NewMockDiskFile(ctrl) 94 | diskMock.EXPECT().OpenFile(testFileMatch, os.O_WRONLY|os.O_CREATE|os.O_EXCL, os.FileMode(0666)).Return(testFileMock, nil) 95 | testFileMock.EXPECT().Write(gomock.Any()).DoAndReturn(func(p []byte) (int, error) { 96 | return len(p), nil 97 | }).AnyTimes() 98 | testFileMock.EXPECT().Close().Return(fmt.Errorf("close error")) 99 | diskMock.EXPECT().Remove(testFileMatch).Return(nil) 100 | 101 | _, err := NewDisk(diskMock) 102 | require.Error(t, err) 103 | }) 104 | 105 | t.Run("error on remove", func(t *testing.T) { 106 | ctrl := gomock.NewController(t) 107 | diskMock := NewMockDiskRoot(ctrl) 108 | 109 | testFileMock := NewMockDiskFile(ctrl) 110 | diskMock.EXPECT().OpenFile(testFileMatch, os.O_WRONLY|os.O_CREATE|os.O_EXCL, os.FileMode(0666)).Return(testFileMock, nil) 111 | testFileMock.EXPECT().Write(gomock.Any()).DoAndReturn(func(p []byte) (int, error) { 112 | return len(p), nil 113 | }).AnyTimes() 114 | testFileMock.EXPECT().Close().Return(nil) 115 | diskMock.EXPECT().Remove(testFileMatch).Return(fmt.Errorf("remove error")) 116 | 117 | _, err := NewDisk(diskMock) 118 | require.Error(t, err) 119 | }) 120 | } 121 | 122 | func TestDisk_GetLocal(t *testing.T) { 123 | actionID := []byte("test-action-id") 124 | outputID := []byte("test-output-id") 125 | size := int64(100) 126 | modTime := time.Now().UTC() 127 | testData := bytes.Repeat([]byte("a"), int(size)) 128 | 129 | t.Run("success", func(t *testing.T) { 130 | // Setup real disk 131 | root, err := NewSystemDiskRoot(t.TempDir()) 132 | require.NoError(t, err) 133 | disk, err := NewDisk(root) 134 | require.NoError(t, err) 135 | 136 | // Write test data first using PutLocal 137 | _, err = disk.PutLocal(context.Background(), &cacheprog.LocalPutRequest{ 138 | ActionID: actionID, 139 | OutputID: outputID, 140 | Size: size, 141 | Body: bytes.NewReader(testData), 142 | }) 143 | require.NoError(t, err) 144 | 145 | // Test GetLocal 146 | resp, err := disk.GetLocal(context.Background(), &cacheprog.LocalGetRequest{ 147 | ActionID: actionID, 148 | }) 149 | 150 | require.NoError(t, err) 151 | assert.Equal(t, outputID, resp.OutputID) 152 | assert.Equal(t, size, resp.Size) 153 | assert.WithinDuration(t, modTime, resp.ModTime, 2*time.Second) 154 | assert.NotEmpty(t, resp.DiskPath) 155 | 156 | // Verify file contents 157 | data, err := os.ReadFile(resp.DiskPath) 158 | require.NoError(t, err) 159 | assert.Equal(t, testData, data) 160 | }) 161 | 162 | t.Run("not found", func(t *testing.T) { 163 | root, err := NewSystemDiskRoot(t.TempDir()) 164 | require.NoError(t, err) 165 | disk, err := NewDisk(root) 166 | require.NoError(t, err) 167 | 168 | _, err = disk.GetLocal(context.Background(), &cacheprog.LocalGetRequest{ 169 | ActionID: actionID, 170 | }) 171 | 172 | assert.ErrorIs(t, err, cacheprog.ErrNotFound) 173 | }) 174 | } 175 | 176 | func TestDisk_PutLocal(t *testing.T) { 177 | actionID := []byte("test-action-id") 178 | outputID := []byte("test-output-id") 179 | data := []byte("test-data") 180 | size := int64(len(data)) 181 | 182 | t.Run("success", func(t *testing.T) { 183 | root, err := NewSystemDiskRoot(t.TempDir()) 184 | require.NoError(t, err) 185 | disk, err := NewDisk(root) 186 | require.NoError(t, err) 187 | 188 | resp, err := disk.PutLocal(context.Background(), &cacheprog.LocalPutRequest{ 189 | ActionID: actionID, 190 | OutputID: outputID, 191 | Size: size, 192 | Body: bytes.NewReader(data), 193 | }) 194 | 195 | require.NoError(t, err) 196 | assert.NotEmpty(t, resp.DiskPath) 197 | 198 | // Verify the file was written correctly 199 | storedData, err := os.ReadFile(resp.DiskPath) 200 | require.NoError(t, err) 201 | assert.Equal(t, data, storedData) 202 | 203 | // Verify metadata 204 | metaPath := filepath.Join(filepath.Dir(resp.DiskPath), hex.EncodeToString(actionID)+"-meta") 205 | metaData, err := os.ReadFile(metaPath) 206 | require.NoError(t, err) 207 | 208 | var meta metaEntry 209 | err = json.Unmarshal(metaData, &meta) 210 | require.NoError(t, err) 211 | assert.Equal(t, outputID, meta.OutputID) 212 | assert.Equal(t, size, meta.Size) 213 | assert.NotZero(t, meta.Time) 214 | }) 215 | 216 | t.Run("wrong size", func(t *testing.T) { 217 | root, err := NewSystemDiskRoot(t.TempDir()) 218 | require.NoError(t, err) 219 | disk, err := NewDisk(root) 220 | require.NoError(t, err) 221 | 222 | _, err = disk.PutLocal(context.Background(), &cacheprog.LocalPutRequest{ 223 | ActionID: actionID, 224 | OutputID: outputID, 225 | Size: size + 1, // Incorrect size 226 | Body: bytes.NewReader(data), 227 | }) 228 | 229 | require.Error(t, err) 230 | assert.Contains(t, err.Error(), "write to file") 231 | }) 232 | } 233 | -------------------------------------------------------------------------------- /internal/infra/storage/mocks_world_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: interfaces.go 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -destination=mocks_world_test.go -package=storage -source=interfaces.go 7 | // 8 | 9 | // Package storage is a generated GoMock package. 10 | package storage 11 | 12 | import ( 13 | context "context" 14 | os "os" 15 | reflect "reflect" 16 | 17 | s3 "github.com/aws/aws-sdk-go-v2/service/s3" 18 | gomock "go.uber.org/mock/gomock" 19 | ) 20 | 21 | // MockS3Client is a mock of S3Client interface. 22 | type MockS3Client struct { 23 | ctrl *gomock.Controller 24 | recorder *MockS3ClientMockRecorder 25 | isgomock struct{} 26 | } 27 | 28 | // MockS3ClientMockRecorder is the mock recorder for MockS3Client. 29 | type MockS3ClientMockRecorder struct { 30 | mock *MockS3Client 31 | } 32 | 33 | // NewMockS3Client creates a new mock instance. 34 | func NewMockS3Client(ctrl *gomock.Controller) *MockS3Client { 35 | mock := &MockS3Client{ctrl: ctrl} 36 | mock.recorder = &MockS3ClientMockRecorder{mock} 37 | return mock 38 | } 39 | 40 | // EXPECT returns an object that allows the caller to indicate expected use. 41 | func (m *MockS3Client) EXPECT() *MockS3ClientMockRecorder { 42 | return m.recorder 43 | } 44 | 45 | // GetObject mocks base method. 46 | func (m *MockS3Client) GetObject(arg0 context.Context, arg1 *s3.GetObjectInput, arg2 ...func(*s3.Options)) (*s3.GetObjectOutput, error) { 47 | m.ctrl.T.Helper() 48 | varargs := []any{arg0, arg1} 49 | for _, a := range arg2 { 50 | varargs = append(varargs, a) 51 | } 52 | ret := m.ctrl.Call(m, "GetObject", varargs...) 53 | ret0, _ := ret[0].(*s3.GetObjectOutput) 54 | ret1, _ := ret[1].(error) 55 | return ret0, ret1 56 | } 57 | 58 | // GetObject indicates an expected call of GetObject. 59 | func (mr *MockS3ClientMockRecorder) GetObject(arg0, arg1 any, arg2 ...any) *gomock.Call { 60 | mr.mock.ctrl.T.Helper() 61 | varargs := append([]any{arg0, arg1}, arg2...) 62 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObject", reflect.TypeOf((*MockS3Client)(nil).GetObject), varargs...) 63 | } 64 | 65 | // HeadObject mocks base method. 66 | func (m *MockS3Client) HeadObject(arg0 context.Context, arg1 *s3.HeadObjectInput, arg2 ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { 67 | m.ctrl.T.Helper() 68 | varargs := []any{arg0, arg1} 69 | for _, a := range arg2 { 70 | varargs = append(varargs, a) 71 | } 72 | ret := m.ctrl.Call(m, "HeadObject", varargs...) 73 | ret0, _ := ret[0].(*s3.HeadObjectOutput) 74 | ret1, _ := ret[1].(error) 75 | return ret0, ret1 76 | } 77 | 78 | // HeadObject indicates an expected call of HeadObject. 79 | func (mr *MockS3ClientMockRecorder) HeadObject(arg0, arg1 any, arg2 ...any) *gomock.Call { 80 | mr.mock.ctrl.T.Helper() 81 | varargs := append([]any{arg0, arg1}, arg2...) 82 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeadObject", reflect.TypeOf((*MockS3Client)(nil).HeadObject), varargs...) 83 | } 84 | 85 | // PutObject mocks base method. 86 | func (m *MockS3Client) PutObject(arg0 context.Context, arg1 *s3.PutObjectInput, arg2 ...func(*s3.Options)) (*s3.PutObjectOutput, error) { 87 | m.ctrl.T.Helper() 88 | varargs := []any{arg0, arg1} 89 | for _, a := range arg2 { 90 | varargs = append(varargs, a) 91 | } 92 | ret := m.ctrl.Call(m, "PutObject", varargs...) 93 | ret0, _ := ret[0].(*s3.PutObjectOutput) 94 | ret1, _ := ret[1].(error) 95 | return ret0, ret1 96 | } 97 | 98 | // PutObject indicates an expected call of PutObject. 99 | func (mr *MockS3ClientMockRecorder) PutObject(arg0, arg1 any, arg2 ...any) *gomock.Call { 100 | mr.mock.ctrl.T.Helper() 101 | varargs := append([]any{arg0, arg1}, arg2...) 102 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutObject", reflect.TypeOf((*MockS3Client)(nil).PutObject), varargs...) 103 | } 104 | 105 | // MockDiskFile is a mock of DiskFile interface. 106 | type MockDiskFile struct { 107 | ctrl *gomock.Controller 108 | recorder *MockDiskFileMockRecorder 109 | isgomock struct{} 110 | } 111 | 112 | // MockDiskFileMockRecorder is the mock recorder for MockDiskFile. 113 | type MockDiskFileMockRecorder struct { 114 | mock *MockDiskFile 115 | } 116 | 117 | // NewMockDiskFile creates a new mock instance. 118 | func NewMockDiskFile(ctrl *gomock.Controller) *MockDiskFile { 119 | mock := &MockDiskFile{ctrl: ctrl} 120 | mock.recorder = &MockDiskFileMockRecorder{mock} 121 | return mock 122 | } 123 | 124 | // EXPECT returns an object that allows the caller to indicate expected use. 125 | func (m *MockDiskFile) EXPECT() *MockDiskFileMockRecorder { 126 | return m.recorder 127 | } 128 | 129 | // Close mocks base method. 130 | func (m *MockDiskFile) Close() error { 131 | m.ctrl.T.Helper() 132 | ret := m.ctrl.Call(m, "Close") 133 | ret0, _ := ret[0].(error) 134 | return ret0 135 | } 136 | 137 | // Close indicates an expected call of Close. 138 | func (mr *MockDiskFileMockRecorder) Close() *gomock.Call { 139 | mr.mock.ctrl.T.Helper() 140 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockDiskFile)(nil).Close)) 141 | } 142 | 143 | // Read mocks base method. 144 | func (m *MockDiskFile) Read(p []byte) (int, error) { 145 | m.ctrl.T.Helper() 146 | ret := m.ctrl.Call(m, "Read", p) 147 | ret0, _ := ret[0].(int) 148 | ret1, _ := ret[1].(error) 149 | return ret0, ret1 150 | } 151 | 152 | // Read indicates an expected call of Read. 153 | func (mr *MockDiskFileMockRecorder) Read(p any) *gomock.Call { 154 | mr.mock.ctrl.T.Helper() 155 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Read", reflect.TypeOf((*MockDiskFile)(nil).Read), p) 156 | } 157 | 158 | // Seek mocks base method. 159 | func (m *MockDiskFile) Seek(offset int64, whence int) (int64, error) { 160 | m.ctrl.T.Helper() 161 | ret := m.ctrl.Call(m, "Seek", offset, whence) 162 | ret0, _ := ret[0].(int64) 163 | ret1, _ := ret[1].(error) 164 | return ret0, ret1 165 | } 166 | 167 | // Seek indicates an expected call of Seek. 168 | func (mr *MockDiskFileMockRecorder) Seek(offset, whence any) *gomock.Call { 169 | mr.mock.ctrl.T.Helper() 170 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Seek", reflect.TypeOf((*MockDiskFile)(nil).Seek), offset, whence) 171 | } 172 | 173 | // Stat mocks base method. 174 | func (m *MockDiskFile) Stat() (os.FileInfo, error) { 175 | m.ctrl.T.Helper() 176 | ret := m.ctrl.Call(m, "Stat") 177 | ret0, _ := ret[0].(os.FileInfo) 178 | ret1, _ := ret[1].(error) 179 | return ret0, ret1 180 | } 181 | 182 | // Stat indicates an expected call of Stat. 183 | func (mr *MockDiskFileMockRecorder) Stat() *gomock.Call { 184 | mr.mock.ctrl.T.Helper() 185 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stat", reflect.TypeOf((*MockDiskFile)(nil).Stat)) 186 | } 187 | 188 | // Write mocks base method. 189 | func (m *MockDiskFile) Write(p []byte) (int, error) { 190 | m.ctrl.T.Helper() 191 | ret := m.ctrl.Call(m, "Write", p) 192 | ret0, _ := ret[0].(int) 193 | ret1, _ := ret[1].(error) 194 | return ret0, ret1 195 | } 196 | 197 | // Write indicates an expected call of Write. 198 | func (mr *MockDiskFileMockRecorder) Write(p any) *gomock.Call { 199 | mr.mock.ctrl.T.Helper() 200 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockDiskFile)(nil).Write), p) 201 | } 202 | 203 | // MockDiskRoot is a mock of DiskRoot interface. 204 | type MockDiskRoot struct { 205 | ctrl *gomock.Controller 206 | recorder *MockDiskRootMockRecorder 207 | isgomock struct{} 208 | } 209 | 210 | // MockDiskRootMockRecorder is the mock recorder for MockDiskRoot. 211 | type MockDiskRootMockRecorder struct { 212 | mock *MockDiskRoot 213 | } 214 | 215 | // NewMockDiskRoot creates a new mock instance. 216 | func NewMockDiskRoot(ctrl *gomock.Controller) *MockDiskRoot { 217 | mock := &MockDiskRoot{ctrl: ctrl} 218 | mock.recorder = &MockDiskRootMockRecorder{mock} 219 | return mock 220 | } 221 | 222 | // EXPECT returns an object that allows the caller to indicate expected use. 223 | func (m *MockDiskRoot) EXPECT() *MockDiskRootMockRecorder { 224 | return m.recorder 225 | } 226 | 227 | // FullPath mocks base method. 228 | func (m *MockDiskRoot) FullPath(name string) string { 229 | m.ctrl.T.Helper() 230 | ret := m.ctrl.Call(m, "FullPath", name) 231 | ret0, _ := ret[0].(string) 232 | return ret0 233 | } 234 | 235 | // FullPath indicates an expected call of FullPath. 236 | func (mr *MockDiskRootMockRecorder) FullPath(name any) *gomock.Call { 237 | mr.mock.ctrl.T.Helper() 238 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FullPath", reflect.TypeOf((*MockDiskRoot)(nil).FullPath), name) 239 | } 240 | 241 | // Mkdir mocks base method. 242 | func (m *MockDiskRoot) Mkdir(name string, perm os.FileMode) error { 243 | m.ctrl.T.Helper() 244 | ret := m.ctrl.Call(m, "Mkdir", name, perm) 245 | ret0, _ := ret[0].(error) 246 | return ret0 247 | } 248 | 249 | // Mkdir indicates an expected call of Mkdir. 250 | func (mr *MockDiskRootMockRecorder) Mkdir(name, perm any) *gomock.Call { 251 | mr.mock.ctrl.T.Helper() 252 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Mkdir", reflect.TypeOf((*MockDiskRoot)(nil).Mkdir), name, perm) 253 | } 254 | 255 | // OpenFile mocks base method. 256 | func (m *MockDiskRoot) OpenFile(name string, flag int, perm os.FileMode) (DiskFile, error) { 257 | m.ctrl.T.Helper() 258 | ret := m.ctrl.Call(m, "OpenFile", name, flag, perm) 259 | ret0, _ := ret[0].(DiskFile) 260 | ret1, _ := ret[1].(error) 261 | return ret0, ret1 262 | } 263 | 264 | // OpenFile indicates an expected call of OpenFile. 265 | func (mr *MockDiskRootMockRecorder) OpenFile(name, flag, perm any) *gomock.Call { 266 | mr.mock.ctrl.T.Helper() 267 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenFile", reflect.TypeOf((*MockDiskRoot)(nil).OpenFile), name, flag, perm) 268 | } 269 | 270 | // Remove mocks base method. 271 | func (m *MockDiskRoot) Remove(name string) error { 272 | m.ctrl.T.Helper() 273 | ret := m.ctrl.Call(m, "Remove", name) 274 | ret0, _ := ret[0].(error) 275 | return ret0 276 | } 277 | 278 | // Remove indicates an expected call of Remove. 279 | func (mr *MockDiskRootMockRecorder) Remove(name any) *gomock.Call { 280 | mr.mock.ctrl.T.Helper() 281 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockDiskRoot)(nil).Remove), name) 282 | } 283 | 284 | // Rename mocks base method. 285 | func (m *MockDiskRoot) Rename(oldpath, newpath string) error { 286 | m.ctrl.T.Helper() 287 | ret := m.ctrl.Call(m, "Rename", oldpath, newpath) 288 | ret0, _ := ret[0].(error) 289 | return ret0 290 | } 291 | 292 | // Rename indicates an expected call of Rename. 293 | func (mr *MockDiskRootMockRecorder) Rename(oldpath, newpath any) *gomock.Call { 294 | mr.mock.ctrl.T.Helper() 295 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Rename", reflect.TypeOf((*MockDiskRoot)(nil).Rename), oldpath, newpath) 296 | } 297 | -------------------------------------------------------------------------------- /functests/env/env.go: -------------------------------------------------------------------------------- 1 | package env 2 | 3 | import ( 4 | "cmp" 5 | "context" 6 | "fmt" 7 | "io" 8 | "net" 9 | "net/http/httptest" 10 | "net/url" 11 | "os" 12 | "os/exec" 13 | "path/filepath" 14 | "runtime" 15 | "strconv" 16 | "strings" 17 | "testing" 18 | 19 | "github.com/docker/docker/pkg/stdcopy" 20 | "github.com/docker/go-connections/nat" 21 | "github.com/rogpeppe/go-internal/gotooltest" 22 | "github.com/rogpeppe/go-internal/testenv" 23 | "github.com/rogpeppe/go-internal/testscript" 24 | "github.com/stretchr/testify/require" 25 | "github.com/testcontainers/testcontainers-go/modules/compose" 26 | "github.com/testcontainers/testcontainers-go/wait" 27 | ) 28 | 29 | type Env struct { 30 | projectRoot string 31 | stack compose.ComposeStack // nil for short tests 32 | coverageDir string 33 | cacheprogPath string 34 | recordingServer *recordingServer 35 | } 36 | 37 | func NewEnv(t testing.TB) *Env { 38 | t.Helper() 39 | 40 | _, file, _, ok := runtime.Caller(0) 41 | require.True(t, ok) 42 | 43 | projectRoot, err := filepath.Abs(filepath.Join(file, "..", "..", "..")) 44 | require.NoError(t, err) 45 | 46 | env := &Env{ 47 | projectRoot: projectRoot, 48 | recordingServer: newRecordingServer(), 49 | } 50 | 51 | // compile cacheprog locally once 52 | binDir := t.TempDir() 53 | 54 | env.cacheprogPath = filepath.Join(binDir, "cacheprog") 55 | if runtime.GOOS == "windows" { 56 | env.cacheprogPath += ".exe" 57 | } 58 | 59 | // compile cacheprog locally, with race and coverage enabled 60 | compiler := testenv.GoToolPath(t) 61 | args := []string{ 62 | "build", 63 | "-C", projectRoot, 64 | "-o", env.cacheprogPath, 65 | } 66 | if RaceEnabled { 67 | args = append(args, "-race") 68 | } 69 | if covDir := os.Getenv("GOCOVERDIR"); covDir != "" { 70 | args = append(args, "-cover") 71 | env.coverageDir = covDir 72 | } 73 | args = append(args, "./cmd/cacheprog") 74 | cmd := testenv.CleanCmdEnv(exec.Command(compiler, args...)) 75 | output, err := cmd.CombinedOutput() 76 | require.NoError(t, err, "failed to build cacheprog: %s", string(output)) 77 | 78 | if testing.Short() { 79 | return env 80 | } 81 | 82 | env.runInProjectRoot(t, func() { 83 | t.Helper() 84 | 85 | var err error 86 | composeStack, err := compose.NewDockerComposeWith( 87 | compose.WithStackFiles(filepath.Join("deployments", "compose", "docker-compose.yml")), 88 | compose.WithLogger(&testLogger{t: t}), 89 | ) 90 | require.NoError(t, err) 91 | 92 | env.stack = composeStack. 93 | WithOsEnv(). 94 | WaitForService("minio", wait.ForHealthCheck()). // wait for minio to be ready 95 | WaitForService("mc", wait.ForHealthCheck()) // wait for bucket to be created 96 | 97 | require.NoError(t, env.stack.Up(t.Context(), compose.WithRecreate("force"), compose.RemoveOrphans(true))) 98 | 99 | t.Cleanup(func() { 100 | _ = env.stack.Down(context.Background(), compose.RemoveOrphans(true)) 101 | }) 102 | }) 103 | 104 | return env 105 | } 106 | 107 | func (e *Env) getServiceExposedAddress(ctx context.Context, serviceName string, proto, port string) (string, error) { 108 | container, err := e.stack.ServiceContainer(ctx, serviceName) 109 | if err != nil { 110 | return "", fmt.Errorf("failed to get service container: %w", err) 111 | } 112 | 113 | portObj, err := nat.NewPort(proto, port) 114 | if err != nil { 115 | return "", fmt.Errorf("failed to create port: %w", err) 116 | } 117 | 118 | endpoint, err := container.PortEndpoint(ctx, portObj, "") 119 | if err != nil { 120 | return "", fmt.Errorf("failed to get endpoint: %w", err) 121 | } 122 | 123 | return endpoint, nil 124 | } 125 | 126 | func (e *Env) GetTestScriptParams(t testing.TB) testscript.Params { 127 | t.Helper() 128 | p := testscript.Params{ 129 | Dir: "testscripts", 130 | WorkdirRoot: t.TempDir(), 131 | Setup: func(env *testscript.Env) error { 132 | if e.coverageDir != "" { 133 | env.Setenv("GOCOVERDIR", e.coverageDir) 134 | } 135 | 136 | return nil 137 | }, 138 | Cmds: map[string]func(ts *testscript.TestScript, neg bool, args []string){ 139 | // 1st arg is a name of the env var 140 | // 2nd arg (optional) is a stderr capture file, shell required 141 | "set-cacheprog": func(ts *testscript.TestScript, neg bool, args []string) { 142 | if len(args) < 1 { 143 | ts.Fatalf("set-cacheprog expects at least 1 argument, got %d", len(args)) 144 | } 145 | 146 | if len(args) == 1 { 147 | ts.Setenv(args[0], e.cacheprogPath) 148 | } else { 149 | // TODO: maybe we should add log redirection as option to cacheprog itself 150 | ts.Setenv(args[0], fmt.Sprintf("%s --log-output=%q", e.cacheprogPath, args[1])) 151 | } 152 | }, 153 | // 1st arg is a name of the env var 154 | "allocate-port": func(ts *testscript.TestScript, neg bool, args []string) { 155 | if len(args) < 1 { 156 | ts.Fatalf("allocate-port expects at least 1 argument, got %d", len(args)) 157 | } 158 | 159 | listener, err := net.Listen("tcp", ":0") 160 | ts.Check(err) 161 | ts.Setenv(args[0], strconv.Itoa(listener.Addr().(*net.TCPAddr).Port)) 162 | ts.Check(listener.Close()) 163 | }, 164 | "recording-server": e.recordingServerCmd, 165 | "install-go-binary": e.installGoBinaryCmd, 166 | }, 167 | Condition: func(cond string) (bool, error) { 168 | switch cond { 169 | case "cgo": 170 | return testenv.HasCGO(), nil 171 | case "race": 172 | return RaceEnabled, nil 173 | default: 174 | return false, fmt.Errorf("unknown condition: %s", cond) 175 | } 176 | }, 177 | } 178 | require.NoError(t, gotooltest.Setup(&p)) 179 | 180 | e.configureMinio(&p) 181 | 182 | return p 183 | } 184 | 185 | func (e *Env) configureMinio(params *testscript.Params) { 186 | if e.stack == nil { 187 | return 188 | } 189 | origSetup := params.Setup 190 | params.Setup = func(env *testscript.Env) error { 191 | err := origSetup(env) 192 | if err != nil { 193 | return err 194 | } 195 | 196 | minioAddress, err := e.getServiceExposedAddress(context.Background(), "toxiproxy", "tcp", "9000") 197 | if err != nil { 198 | return fmt.Errorf("failed to get minio address: %w", err) 199 | } 200 | 201 | // configuration for s3 backend to avoid repetition in scripts 202 | env.Setenv("CACHEPROG_S3_ENDPOINT", (&url.URL{ 203 | Scheme: "minio+http", 204 | Host: minioAddress, 205 | }).String()) 206 | env.Setenv("CACHEPROG_S3_BUCKET", "files-bucket") 207 | env.Setenv("CACHEPROG_S3_FORCE_PATH_STYLE", "true") 208 | env.Setenv("CACHEPROG_S3_ACCESS_KEY_ID", cmp.Or(os.Getenv("MINIO_ROOT_USER"), "minioadmin")) 209 | env.Setenv("CACHEPROG_S3_ACCESS_KEY_SECRET", cmp.Or(os.Getenv("MINIO_ROOT_PASSWORD"), "minioadmin")) 210 | env.Setenv("CACHEPROG_S3_SESSION_TOKEN", "") 211 | env.Setenv("MINIO_ALIAS", "myminio") // defined in docker-compose.yml 212 | 213 | return nil 214 | } 215 | 216 | // propagate minio client to the scripts 217 | params.Cmds["mc"] = e.mcCmd 218 | } 219 | 220 | func (e *Env) mcCmd(ts *testscript.TestScript, neg bool, args []string) { 221 | mcContainer, err := e.stack.ServiceContainer(context.Background(), "mc") 222 | ts.Check(err) 223 | 224 | exitCode, out, err := mcContainer.Exec(context.Background(), 225 | append([]string{"mc"}, args...), 226 | ) 227 | ts.Check(err) 228 | 229 | _, err = stdcopy.StdCopy(ts.Stdout(), ts.Stderr(), out) 230 | ts.Check(err) 231 | 232 | if exitCode != 0 && !neg { 233 | ts.Fatalf("mc command failed with exit code %d", exitCode) 234 | } 235 | 236 | if exitCode == 0 && neg { 237 | ts.Fatalf("mc command succeeded, but expected to fail") 238 | } 239 | } 240 | 241 | func (e *Env) recordingServerCmd(ts *testscript.TestScript, neg bool, args []string) { 242 | if len(args) < 1 { 243 | ts.Fatalf("command expects at least 1 argument, got %d", len(args)) 244 | } 245 | 246 | cmd := args[0] 247 | if cmd == "start" { 248 | if len(args) < 2 { 249 | ts.Fatalf("command start expects at least 2 arguments, got %d", len(args)) 250 | } 251 | 252 | addressEnvVar := args[1] 253 | 254 | server := httptest.NewServer(e.recordingServer) 255 | ts.Defer(func() { 256 | server.Close() 257 | }) 258 | 259 | ts.Setenv(addressEnvVar, server.URL) 260 | return 261 | } 262 | 263 | if cmd != "get" { 264 | ts.Fatalf("unknown command: %s", cmd) 265 | return 266 | } 267 | 268 | if len(args) < 3 { 269 | ts.Fatalf("command get expects at least 3 arguments, got %d", len(args)) 270 | } 271 | 272 | recordKey := args[1] 273 | whatToGet := args[2] 274 | 275 | request, ok := e.recordingServer.GetRequest(recordKey) 276 | if !ok && !neg { 277 | ts.Fatalf("request not found for key: %s", recordKey) 278 | return 279 | } 280 | if !ok && neg { 281 | ts.Fatalf("request found for key: %s", recordKey) 282 | return 283 | } 284 | 285 | var err error 286 | 287 | switch whatToGet { 288 | case "body": 289 | _, err = ts.Stdout().Write(request.Body) 290 | case "method": 291 | _, err = io.WriteString(ts.Stdout(), request.Method) 292 | case "path": 293 | _, err = io.WriteString(ts.Stdout(), request.Path) 294 | case "query": 295 | _, err = io.WriteString(ts.Stdout(), request.Query.Encode()) 296 | case "headers": 297 | err = request.Headers.Write(ts.Stdout()) 298 | default: 299 | ts.Fatalf("unknown what to get: %s", whatToGet) 300 | } 301 | 302 | ts.Check(err) 303 | } 304 | 305 | func (e *Env) installGoBinaryCmd(ts *testscript.TestScript, neg bool, args []string) { 306 | if len(args) < 2 { 307 | ts.Fatalf("command expects at least 2 argument, got %d", len(args)) 308 | } 309 | 310 | importPath := args[0] 311 | targetDir := ts.MkAbs(args[1]) 312 | 313 | compiler, err := testenv.GoTool() 314 | ts.Check(err) 315 | 316 | compilerArgs := []string{"install"} 317 | if testing.Verbose() { 318 | compilerArgs = append(compilerArgs, "-v") 319 | } 320 | compilerArgs = append(compilerArgs, importPath) 321 | 322 | cmd := testenv.CleanCmdEnv(exec.Command(compiler, compilerArgs...)) 323 | // forcefully disable cgo 324 | for _, env := range os.Environ() { 325 | if strings.HasPrefix(env, "CGO_ENABLED=") { 326 | continue 327 | } 328 | cmd.Env = append(cmd.Env, env) 329 | } 330 | cmd.Env = append(cmd.Env, "GOBIN="+targetDir, "CGO_ENABLED=0") 331 | cmd.Stdout = ts.Stdout() 332 | cmd.Stderr = ts.Stderr() 333 | 334 | ts.Check(cmd.Run()) 335 | } 336 | 337 | type testLogger struct { 338 | t testing.TB 339 | } 340 | 341 | func (l *testLogger) Printf(format string, v ...any) { 342 | l.t.Helper() 343 | l.t.Logf(format, v...) 344 | } 345 | 346 | func (e *Env) runInProjectRoot(t testing.TB, fn func()) { 347 | t.Helper() 348 | 349 | originalDir, err := os.Getwd() 350 | require.NoError(t, err, "failed to get current directory") 351 | 352 | err = os.Chdir(e.projectRoot) 353 | require.NoError(t, err, "failed to change to project root") 354 | // test if it is real project root, must contain go.mod 355 | _, err = os.Stat("go.mod") 356 | require.NoError(t, err, "go.mod not found in project root") 357 | 358 | defer func() { 359 | t.Helper() 360 | 361 | require.NoError(t, os.Chdir(originalDir), "failed to change back to original directory") 362 | }() 363 | 364 | fn() 365 | } 366 | -------------------------------------------------------------------------------- /internal/infra/storage/s3_test.go: -------------------------------------------------------------------------------- 1 | package storage_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/base64" 7 | "encoding/hex" 8 | "errors" 9 | "fmt" 10 | "io" 11 | "maps" 12 | "testing" 13 | "time" 14 | 15 | "github.com/aws/aws-sdk-go-v2/aws" 16 | "github.com/aws/aws-sdk-go-v2/service/s3" 17 | "github.com/aws/aws-sdk-go-v2/service/s3/types" 18 | "github.com/aws/smithy-go" 19 | "github.com/stretchr/testify/assert" 20 | "github.com/stretchr/testify/require" 21 | "go.uber.org/mock/gomock" 22 | 23 | "github.com/platacard/cacheprog/internal/app/cacheprog" 24 | "github.com/platacard/cacheprog/internal/infra/storage" 25 | ) 26 | 27 | type putObjectInputMatcher struct { 28 | bucket string 29 | key string 30 | content []byte 31 | contentLength int64 32 | checksumAlgorithm types.ChecksumAlgorithm 33 | checksumSHA256 []byte 34 | metadata map[string]string 35 | } 36 | 37 | func (m putObjectInputMatcher) Matches(x any) bool { 38 | input, ok := x.(*s3.PutObjectInput) 39 | if !ok { 40 | return false 41 | } 42 | 43 | // Read the body content 44 | content, err := io.ReadAll(input.Body) 45 | if err != nil { 46 | return false 47 | } 48 | // Reset the reader for subsequent reads 49 | input.Body = bytes.NewReader(content) 50 | 51 | // Check all required fields 52 | if aws.ToString(input.Bucket) != m.bucket || 53 | aws.ToString(input.Key) != m.key || 54 | !bytes.Equal(content, m.content) || 55 | aws.ToInt64(input.ContentLength) != m.contentLength || 56 | input.ChecksumAlgorithm != m.checksumAlgorithm || 57 | aws.ToString(input.ChecksumSHA256) != base64.StdEncoding.EncodeToString(m.checksumSHA256) { 58 | return false 59 | } 60 | 61 | // Check metadata 62 | if !maps.Equal(input.Metadata, m.metadata) { 63 | return false 64 | } 65 | 66 | return true 67 | } 68 | 69 | func (m putObjectInputMatcher) String() string { 70 | return fmt.Sprintf("PutObjectInput with bucket=%q, key=%q, metadata=%v", m.bucket, m.key, m.metadata) 71 | } 72 | 73 | func TestS3(t *testing.T) { 74 | t.Run("Get", func(t *testing.T) { 75 | t.Run("success", func(t *testing.T) { 76 | ctrl := gomock.NewController(t) 77 | 78 | mockClient := storage.NewMockS3Client(ctrl) 79 | s3Storage := storage.NewS3(storage.S3Params{ 80 | Client: mockClient, 81 | Bucket: "test-bucket", 82 | Prefix: "test-prefix", 83 | Lifetime: time.Hour, 84 | }) 85 | 86 | actionID := []byte("test-action-id") 87 | outputID := []byte("test-output-id") 88 | lastModified := time.Now() 89 | contentLength := int64(100) 90 | body := io.NopCloser(bytes.NewReader([]byte("test-content"))) 91 | metadata := map[string]string{ 92 | "output_id": base64.StdEncoding.EncodeToString(outputID), 93 | "compression_algorithm": "test-algorithm", 94 | "uncompressed_size": "100", 95 | } 96 | 97 | mockClient.EXPECT().GetObject(gomock.Any(), &s3.GetObjectInput{ 98 | Bucket: aws.String("test-bucket"), 99 | Key: aws.String("test-prefix/" + hex.EncodeToString(actionID)), 100 | }).Return(&s3.GetObjectOutput{ 101 | LastModified: &lastModified, 102 | ContentLength: aws.Int64(contentLength), 103 | Body: body, 104 | Metadata: metadata, 105 | }, nil) 106 | 107 | resp, err := s3Storage.Get(context.Background(), &cacheprog.GetRequest{ 108 | ActionID: actionID, 109 | }) 110 | 111 | require.NoError(t, err) 112 | assert.Equal(t, outputID, resp.OutputID) 113 | assert.Equal(t, lastModified, resp.ModTime) 114 | assert.Equal(t, contentLength, resp.Size) 115 | assert.Equal(t, body, resp.Body) 116 | assert.Equal(t, "test-algorithm", resp.CompressionAlgorithm) 117 | assert.Equal(t, contentLength, resp.UncompressedSize) 118 | }) 119 | 120 | t.Run("success, not compressed", func(t *testing.T) { 121 | ctrl := gomock.NewController(t) 122 | 123 | mockClient := storage.NewMockS3Client(ctrl) 124 | s3Storage := storage.NewS3(storage.S3Params{ 125 | Client: mockClient, 126 | Bucket: "test-bucket", 127 | Prefix: "test-prefix", 128 | Lifetime: time.Hour, 129 | }) 130 | 131 | actionID := []byte("test-action-id") 132 | outputID := []byte("test-output-id") 133 | lastModified := time.Now() 134 | contentLength := int64(100) 135 | body := io.NopCloser(bytes.NewReader([]byte("test-content"))) 136 | metadata := map[string]string{ 137 | "output_id": base64.StdEncoding.EncodeToString(outputID), 138 | } 139 | 140 | mockClient.EXPECT().GetObject(gomock.Any(), &s3.GetObjectInput{ 141 | Bucket: aws.String("test-bucket"), 142 | Key: aws.String("test-prefix/" + hex.EncodeToString(actionID)), 143 | }).Return(&s3.GetObjectOutput{ 144 | LastModified: &lastModified, 145 | ContentLength: aws.Int64(contentLength), 146 | Body: body, 147 | Metadata: metadata, 148 | }, nil) 149 | 150 | resp, err := s3Storage.Get(context.Background(), &cacheprog.GetRequest{ 151 | ActionID: actionID, 152 | }) 153 | 154 | require.NoError(t, err) 155 | assert.Equal(t, outputID, resp.OutputID) 156 | assert.Equal(t, lastModified, resp.ModTime) 157 | assert.Equal(t, contentLength, resp.Size) 158 | assert.Equal(t, body, resp.Body) 159 | assert.Empty(t, resp.CompressionAlgorithm) 160 | assert.Equal(t, contentLength, resp.UncompressedSize) 161 | }) 162 | 163 | t.Run("not found", func(t *testing.T) { 164 | ctrl := gomock.NewController(t) 165 | 166 | mockClient := storage.NewMockS3Client(ctrl) 167 | s3Storage := storage.NewS3(storage.S3Params{ 168 | Client: mockClient, 169 | Bucket: "test-bucket", 170 | Prefix: "test-prefix", 171 | Lifetime: time.Hour, 172 | }) 173 | 174 | actionID := []byte("test-action-id") 175 | 176 | mockClient.EXPECT().GetObject(gomock.Any(), gomock.Any()).Return(nil, &smithy.GenericAPIError{ 177 | Code: "NoSuchKey", 178 | }) 179 | 180 | _, err := s3Storage.Get(context.Background(), &cacheprog.GetRequest{ 181 | ActionID: actionID, 182 | }) 183 | 184 | assert.ErrorIs(t, err, cacheprog.ErrNotFound) 185 | }) 186 | 187 | t.Run("missing output ID", func(t *testing.T) { 188 | ctrl := gomock.NewController(t) 189 | t.Cleanup(ctrl.Finish) 190 | 191 | mockClient := storage.NewMockS3Client(ctrl) 192 | s3Storage := storage.NewS3(storage.S3Params{ 193 | Client: mockClient, 194 | Bucket: "test-bucket", 195 | Prefix: "test-prefix", 196 | Lifetime: time.Hour, 197 | }) 198 | 199 | actionID := []byte("test-action-id") 200 | 201 | mockClient.EXPECT().GetObject(gomock.Any(), gomock.Any()).Return(&s3.GetObjectOutput{ 202 | Metadata: map[string]string{}, 203 | }, nil) 204 | 205 | _, err := s3Storage.Get(context.Background(), &cacheprog.GetRequest{ 206 | ActionID: actionID, 207 | }) 208 | 209 | assert.Error(t, err) 210 | }) 211 | }) 212 | 213 | t.Run("Put", func(t *testing.T) { 214 | t.Run("success new object", func(t *testing.T) { 215 | ctrl := gomock.NewController(t) 216 | 217 | mockClient := storage.NewMockS3Client(ctrl) 218 | s3Storage := storage.NewS3(storage.S3Params{ 219 | Client: mockClient, 220 | Bucket: "test-bucket", 221 | Prefix: "test-prefix", 222 | Lifetime: time.Hour, 223 | }) 224 | 225 | actionID := []byte("test-action-id") 226 | outputID := []byte("test-output-id") 227 | content := []byte("test-content") 228 | md5Sum := []byte("test-md5") 229 | sha256Sum := []byte("test-sha256") 230 | 231 | mockClient.EXPECT().HeadObject(gomock.Any(), &s3.HeadObjectInput{ 232 | Bucket: aws.String("test-bucket"), 233 | Key: aws.String("test-prefix/" + hex.EncodeToString(actionID)), 234 | IfMatch: aws.String(hex.EncodeToString(md5Sum)), 235 | }).Return(nil, &smithy.GenericAPIError{Code: "NoSuchKey"}) 236 | 237 | mockClient.EXPECT().PutObject( 238 | gomock.Any(), 239 | putObjectInputMatcher{ 240 | bucket: "test-bucket", 241 | key: "test-prefix/" + hex.EncodeToString(actionID), 242 | content: content, 243 | contentLength: int64(len(content)), 244 | checksumAlgorithm: types.ChecksumAlgorithmSha256, 245 | checksumSHA256: sha256Sum, 246 | metadata: map[string]string{ 247 | "output_id": base64.StdEncoding.EncodeToString(outputID), 248 | "compression_algorithm": "test-algorithm", 249 | "uncompressed_size": "100", 250 | }, 251 | }, 252 | gomock.Any(), 253 | ).Return(&s3.PutObjectOutput{}, nil) 254 | 255 | _, err := s3Storage.Put(context.Background(), &cacheprog.PutRequest{ 256 | ActionID: actionID, 257 | OutputID: outputID, 258 | Body: bytes.NewReader(content), 259 | Size: int64(len(content)), 260 | MD5Sum: md5Sum, 261 | Sha256Sum: sha256Sum, 262 | CompressionAlgorithm: "test-algorithm", 263 | UncompressedSize: 100, 264 | }) 265 | 266 | require.NoError(t, err) 267 | }) 268 | 269 | t.Run("object exists with same metadata", func(t *testing.T) { 270 | ctrl := gomock.NewController(t) 271 | 272 | mockClient := storage.NewMockS3Client(ctrl) 273 | s3Storage := storage.NewS3(storage.S3Params{ 274 | Client: mockClient, 275 | Bucket: "test-bucket", 276 | Prefix: "test-prefix", 277 | Lifetime: time.Hour, 278 | }) 279 | 280 | actionID := []byte("test-action-id") 281 | outputID := []byte("test-output-id") 282 | content := []byte("test-content") 283 | md5Sum := []byte("test-md5") 284 | sha256Sum := []byte("test-sha256") 285 | metadata := map[string]string{ 286 | "output_id": base64.StdEncoding.EncodeToString(outputID), 287 | "compression_algorithm": "test-algorithm", 288 | "uncompressed_size": "100", 289 | } 290 | 291 | mockClient.EXPECT().HeadObject(gomock.Any(), gomock.Any()).Return(&s3.HeadObjectOutput{ 292 | Metadata: metadata, 293 | }, nil) 294 | 295 | _, err := s3Storage.Put(context.Background(), &cacheprog.PutRequest{ 296 | ActionID: actionID, 297 | OutputID: outputID, 298 | Body: bytes.NewReader(content), 299 | Size: int64(len(content)), 300 | MD5Sum: md5Sum, 301 | Sha256Sum: sha256Sum, 302 | CompressionAlgorithm: "test-algorithm", 303 | UncompressedSize: 100, 304 | }) 305 | 306 | require.NoError(t, err) 307 | }) 308 | 309 | t.Run("put error", func(t *testing.T) { 310 | ctrl := gomock.NewController(t) 311 | 312 | mockClient := storage.NewMockS3Client(ctrl) 313 | s3Storage := storage.NewS3(storage.S3Params{ 314 | Client: mockClient, 315 | Bucket: "test-bucket", 316 | Prefix: "test-prefix", 317 | Lifetime: time.Hour, 318 | }) 319 | 320 | actionID := []byte("test-action-id") 321 | outputID := []byte("test-output-id") 322 | content := []byte("test-content") 323 | md5Sum := []byte("test-md5") 324 | sha256Sum := []byte("test-sha256") 325 | 326 | mockClient.EXPECT().HeadObject(gomock.Any(), gomock.Any()).Return(nil, errors.New("head error")) 327 | mockClient.EXPECT().PutObject(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("put error")) 328 | 329 | _, err := s3Storage.Put(context.Background(), &cacheprog.PutRequest{ 330 | ActionID: actionID, 331 | OutputID: outputID, 332 | Body: bytes.NewReader(content), 333 | Size: int64(len(content)), 334 | MD5Sum: md5Sum, 335 | Sha256Sum: sha256Sum, 336 | CompressionAlgorithm: "test-algorithm", 337 | UncompressedSize: 100, 338 | }) 339 | 340 | assert.Error(t, err) 341 | }) 342 | }) 343 | } 344 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/KimMachineGun/automemlimit v0.7.5 h1:RkbaC0MwhjL1ZuBKunGDjE/ggwAX43DwZrJqVwyveTk= 2 | github.com/KimMachineGun/automemlimit v0.7.5/go.mod h1:QZxpHaGOQoYvFhv/r4u3U0JTC2ZcOwbSr11UZF46UBM= 3 | github.com/VictoriaMetrics/metrics v1.40.2 h1:OVSjKcQEx6JAwGeu8/KQm9Su5qJ72TMEW4xYn5vw3Ac= 4 | github.com/VictoriaMetrics/metrics v1.40.2/go.mod h1:XE4uudAAIRaJE614Tl5HMrtoEU6+GDZO4QTnNSsZRuA= 5 | github.com/alexflint/go-arg v1.6.0 h1:wPP9TwTPO54fUVQl4nZoxbFfKCcy5E6HBCumj1XVRSo= 6 | github.com/alexflint/go-arg v1.6.0/go.mod h1:A7vTJzvjoaSTypg4biM5uYNTkJ27SkNTArtYXnlqVO8= 7 | github.com/alexflint/go-scalar v1.2.0 h1:WR7JPKkeNpnYIOfHRa7ivM21aWAdHD0gEWHCx+WQBRw= 8 | github.com/alexflint/go-scalar v1.2.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= 9 | github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc= 10 | github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE= 11 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0= 12 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y= 13 | github.com/aws/aws-sdk-go-v2/config v1.32.1 h1:iODUDLgk3q8/flEC7ymhmxjfoAnBDwEEYEVyKZ9mzjU= 14 | github.com/aws/aws-sdk-go-v2/config v1.32.1/go.mod h1:xoAgo17AGrPpJBSLg81W+ikM0cpOZG8ad04T2r+d5P0= 15 | github.com/aws/aws-sdk-go-v2/credentials v1.19.1 h1:JeW+EwmtTE0yXFK8SmklrFh/cGTTXsQJumgMZNlbxfM= 16 | github.com/aws/aws-sdk-go-v2/credentials v1.19.1/go.mod h1:BOoXiStwTF+fT2XufhO0Efssbi1CNIO/ZXpZu87N0pw= 17 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 h1:WZVR5DbDgxzA0BJeudId89Kmgy6DIU4ORpxwsVHz0qA= 18 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14/go.mod h1:Dadl9QO0kHgbrH1GRqGiZdYtW5w+IXXaBNCHTIaheM4= 19 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 h1:PZHqQACxYb8mYgms4RZbhZG0a7dPW06xOjmaH0EJC/I= 20 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14/go.mod h1:VymhrMJUWs69D8u0/lZ7jSB6WgaG/NqHi3gX0aYf6U0= 21 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 h1:bOS19y6zlJwagBfHxs0ESzr1XCOU2KXJCWcq3E2vfjY= 22 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14/go.mod h1:1ipeGBMAxZ0xcTm6y6paC2C/J6f6OO7LBODV9afuAyM= 23 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= 24 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= 25 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 h1:ITi7qiDSv/mSGDSWNpZ4k4Ve0DQR6Ug2SJQ8zEHoDXg= 26 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14/go.mod h1:k1xtME53H1b6YpZt74YmwlONMWf4ecM+lut1WQLAF/U= 27 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o= 28 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo= 29 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5 h1:Hjkh7kE6D81PgrHlE/m9gx+4TyyeLHuY8xJs7yXN5C4= 30 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.5/go.mod h1:nPRXgyCfAurhyaTMoBMwRBYBhaHI4lNPAnJmjM0Tslc= 31 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 h1:FIouAnCE46kyYqyhs0XEBDFFSREtdnr8HQuLPQPLCrY= 32 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14/go.mod h1:UTwDc5COa5+guonQU8qBikJo1ZJ4ln2r1MkF7Dqag1E= 33 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 h1:FzQE21lNtUor0Fb7QNgnEyiRCBlolLTX/Z1j65S7teM= 34 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14/go.mod h1:s1ydyWG9pm3ZwmmYN21HKyG9WzAZhYVW85wMHs5FV6w= 35 | github.com/aws/aws-sdk-go-v2/service/s3 v1.92.0 h1:8FshVvnV2sr9kOSAbOnc/vwVmmAwMjOedKH6JW2ddPM= 36 | github.com/aws/aws-sdk-go-v2/service/s3 v1.92.0/go.mod h1:wYNqY3L02Z3IgRYxOBPH9I1zD9Cjh9hI5QOy/eOjQvw= 37 | github.com/aws/aws-sdk-go-v2/service/signin v1.0.1 h1:BDgIUYGEo5TkayOWv/oBLPphWwNm/A91AebUjAu5L5g= 38 | github.com/aws/aws-sdk-go-v2/service/signin v1.0.1/go.mod h1:iS6EPmNeqCsGo+xQmXv0jIMjyYtQfnwg36zl2FwEouk= 39 | github.com/aws/aws-sdk-go-v2/service/sso v1.30.4 h1:U//SlnkE1wOQiIImxzdY5PXat4Wq+8rlfVEw4Y7J8as= 40 | github.com/aws/aws-sdk-go-v2/service/sso v1.30.4/go.mod h1:av+ArJpoYf3pgyrj6tcehSFW+y9/QvAY8kMooR9bZCw= 41 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.9 h1:LU8S9W/mPDAU9q0FjCLi0TrCheLMGwzbRpvUMwYspcA= 42 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.9/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug= 43 | github.com/aws/aws-sdk-go-v2/service/sts v1.41.1 h1:GdGmKtG+/Krag7VfyOXV17xjTCz0i9NT+JnqLTOI5nA= 44 | github.com/aws/aws-sdk-go-v2/service/sts v1.41.1/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso= 45 | github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM= 46 | github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= 47 | github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= 48 | github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= 49 | github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= 50 | github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= 51 | github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= 52 | github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= 53 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 54 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 55 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 56 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 57 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 58 | github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY= 59 | github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= 60 | github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= 61 | github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= 62 | github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= 63 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 64 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 65 | github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= 66 | github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= 67 | github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= 68 | github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= 69 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 70 | github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= 71 | github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= 72 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 73 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 74 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 75 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 76 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 77 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 78 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 79 | github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= 80 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 81 | github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= 82 | github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= 83 | github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= 84 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 85 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 86 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 87 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 88 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 89 | github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= 90 | github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= 91 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 92 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 93 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 94 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 95 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 96 | github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 97 | github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 98 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 99 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 100 | github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= 101 | github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= 102 | github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= 103 | github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= 104 | github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= 105 | github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= 106 | go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= 107 | go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= 108 | golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= 109 | golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= 110 | golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= 111 | golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 112 | golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 113 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 114 | golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= 115 | golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 116 | golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= 117 | golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= 118 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 119 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 120 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 121 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 122 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 123 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 124 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /functests/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/platacard/cacheprog/functests 2 | 3 | go 1.25.0 4 | 5 | require ( 6 | github.com/docker/docker v28.4.0+incompatible 7 | github.com/docker/go-connections v0.6.0 8 | github.com/rogpeppe/go-internal v1.14.1 9 | github.com/stretchr/testify v1.11.1 10 | github.com/testcontainers/testcontainers-go v0.39.0 11 | github.com/testcontainers/testcontainers-go/modules/compose v0.39.0 12 | ) 13 | 14 | require ( 15 | dario.cat/mergo v1.0.2 // indirect 16 | github.com/AlecAivazis/survey/v2 v2.3.7 // indirect 17 | github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect 18 | github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e // indirect 19 | github.com/Masterminds/semver/v3 v3.4.0 // indirect 20 | github.com/Microsoft/go-winio v0.6.2 // indirect 21 | github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect 22 | github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect 23 | github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect 24 | github.com/aws/aws-sdk-go-v2/config v1.27.27 // indirect 25 | github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect 26 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect 27 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect 28 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect 29 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect 30 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect 31 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect 32 | github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect 33 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect 34 | github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect 35 | github.com/aws/smithy-go v1.20.3 // indirect 36 | github.com/beorn7/perks v1.0.1 // indirect 37 | github.com/buger/goterm v1.0.4 // indirect 38 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 39 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 40 | github.com/compose-spec/compose-go/v2 v2.8.1 // indirect 41 | github.com/containerd/console v1.0.5 // indirect 42 | github.com/containerd/containerd/api v1.9.0 // indirect 43 | github.com/containerd/containerd/v2 v2.1.4 // indirect 44 | github.com/containerd/continuity v0.4.5 // indirect 45 | github.com/containerd/errdefs v1.0.0 // indirect 46 | github.com/containerd/errdefs/pkg v0.3.0 // indirect 47 | github.com/containerd/log v0.1.0 // indirect 48 | github.com/containerd/platforms v1.0.0-rc.1 // indirect 49 | github.com/containerd/ttrpc v1.2.7 // indirect 50 | github.com/containerd/typeurl/v2 v2.2.3 // indirect 51 | github.com/cpuguy83/dockercfg v0.3.2 // indirect 52 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 53 | github.com/distribution/reference v0.6.0 // indirect 54 | github.com/docker/buildx v0.26.1 // indirect 55 | github.com/docker/cli v28.3.3+incompatible // indirect 56 | github.com/docker/cli-docs-tool v0.10.0 // indirect 57 | github.com/docker/compose/v2 v2.39.2 // indirect 58 | github.com/docker/distribution v2.8.3+incompatible // indirect 59 | github.com/docker/docker-credential-helpers v0.9.3 // indirect 60 | github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect 61 | github.com/docker/go-metrics v0.0.1 // indirect 62 | github.com/docker/go-units v0.5.0 // indirect 63 | github.com/ebitengine/purego v0.8.4 // indirect 64 | github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 // indirect 65 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 66 | github.com/felixge/httpsnoop v1.0.4 // indirect 67 | github.com/fsnotify/fsevents v0.2.0 // indirect 68 | github.com/fvbommel/sortorder v1.1.0 // indirect 69 | github.com/fxamacker/cbor/v2 v2.7.0 // indirect 70 | github.com/go-logr/logr v1.4.3 // indirect 71 | github.com/go-logr/stdr v1.2.2 // indirect 72 | github.com/go-ole/go-ole v1.2.6 // indirect 73 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 74 | github.com/go-openapi/jsonreference v0.20.2 // indirect 75 | github.com/go-openapi/swag v0.23.0 // indirect 76 | github.com/go-viper/mapstructure/v2 v2.4.0 // indirect 77 | github.com/gofrs/flock v0.12.1 // indirect 78 | github.com/gogo/protobuf v1.3.2 // indirect 79 | github.com/golang-jwt/jwt/v5 v5.2.2 // indirect 80 | github.com/golang/protobuf v1.5.4 // indirect 81 | github.com/google/gnostic-models v0.6.8 // indirect 82 | github.com/google/go-cmp v0.7.0 // indirect 83 | github.com/google/gofuzz v1.2.0 // indirect 84 | github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect 85 | github.com/google/uuid v1.6.0 // indirect 86 | github.com/gorilla/mux v1.8.1 // indirect 87 | github.com/gorilla/websocket v1.5.0 // indirect 88 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 // indirect 89 | github.com/hashicorp/errwrap v1.1.0 // indirect 90 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 91 | github.com/hashicorp/go-multierror v1.1.1 // indirect 92 | github.com/hashicorp/go-version v1.7.0 // indirect 93 | github.com/in-toto/in-toto-golang v0.9.0 // indirect 94 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 95 | github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf // indirect 96 | github.com/jonboulle/clockwork v0.5.0 // indirect 97 | github.com/josharian/intern v1.0.0 // indirect 98 | github.com/json-iterator/go v1.1.12 // indirect 99 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect 100 | github.com/klauspost/compress v1.18.0 // indirect 101 | github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect 102 | github.com/magiconair/properties v1.8.10 // indirect 103 | github.com/mailru/easyjson v0.7.7 // indirect 104 | github.com/mattn/go-colorable v0.1.13 // indirect 105 | github.com/mattn/go-isatty v0.0.20 // indirect 106 | github.com/mattn/go-runewidth v0.0.16 // indirect 107 | github.com/mattn/go-shellwords v1.0.12 // indirect 108 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect 109 | github.com/miekg/pkcs11 v1.1.1 // indirect 110 | github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect 111 | github.com/mitchellh/mapstructure v1.5.0 // indirect 112 | github.com/moby/buildkit v0.23.0-rc1.0.20250618182037-9b91d20367db // indirect 113 | github.com/moby/docker-image-spec v1.3.1 // indirect 114 | github.com/moby/go-archive v0.1.0 // indirect 115 | github.com/moby/locker v1.0.1 // indirect 116 | github.com/moby/patternmatcher v0.6.0 // indirect 117 | github.com/moby/spdystream v0.5.0 // indirect 118 | github.com/moby/sys/atomicwriter v0.1.0 // indirect 119 | github.com/moby/sys/capability v0.4.0 // indirect 120 | github.com/moby/sys/mountinfo v0.7.2 // indirect 121 | github.com/moby/sys/sequential v0.6.0 // indirect 122 | github.com/moby/sys/signal v0.7.1 // indirect 123 | github.com/moby/sys/symlink v0.3.0 // indirect 124 | github.com/moby/sys/user v0.4.0 // indirect 125 | github.com/moby/sys/userns v0.1.0 // indirect 126 | github.com/moby/term v0.5.2 // indirect 127 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 128 | github.com/modern-go/reflect2 v1.0.2 // indirect 129 | github.com/morikuni/aec v1.0.0 // indirect 130 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 131 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect 132 | github.com/opencontainers/go-digest v1.0.0 // indirect 133 | github.com/opencontainers/image-spec v1.1.1 // indirect 134 | github.com/pelletier/go-toml v1.9.5 // indirect 135 | github.com/pkg/errors v0.9.1 // indirect 136 | github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect 137 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 138 | github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect 139 | github.com/prometheus/client_golang v1.22.0 // indirect 140 | github.com/prometheus/client_model v0.6.1 // indirect 141 | github.com/prometheus/common v0.62.0 // indirect 142 | github.com/prometheus/procfs v0.15.1 // indirect 143 | github.com/rivo/uniseg v0.2.0 // indirect 144 | github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect 145 | github.com/secure-systems-lab/go-securesystemslib v0.6.0 // indirect 146 | github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b // indirect 147 | github.com/shibumi/go-pathspec v1.3.0 // indirect 148 | github.com/shirou/gopsutil/v4 v4.25.6 // indirect 149 | github.com/sirupsen/logrus v1.9.3 // indirect 150 | github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect 151 | github.com/spf13/cobra v1.9.1 // indirect 152 | github.com/spf13/pflag v1.0.7 // indirect 153 | github.com/theupdateframework/notary v0.7.0 // indirect 154 | github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375 // indirect 155 | github.com/tklauser/go-sysconf v0.3.12 // indirect 156 | github.com/tklauser/numcpus v0.6.1 // indirect 157 | github.com/tonistiigi/dchapes-mode v0.0.0-20250318174251-73d941a28323 // indirect 158 | github.com/tonistiigi/fsutil v0.0.0-20250605211040-586307ad452f // indirect 159 | github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0 // indirect 160 | github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect 161 | github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab // indirect 162 | github.com/x448/float16 v0.8.4 // indirect 163 | github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect 164 | github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect 165 | github.com/xeipuuv/gojsonschema v1.2.0 // indirect 166 | github.com/xhit/go-str2duration/v2 v2.1.0 // indirect 167 | github.com/yusufpapurcu/wmi v1.2.4 // indirect 168 | github.com/zclconf/go-cty v1.16.2 // indirect 169 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 170 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect 171 | go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0 // indirect 172 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect 173 | go.opentelemetry.io/otel v1.36.0 // indirect 174 | go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0 // indirect 175 | go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 // indirect 176 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect 177 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0 // indirect 178 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 // indirect 179 | go.opentelemetry.io/otel/metric v1.36.0 // indirect 180 | go.opentelemetry.io/otel/sdk v1.36.0 // indirect 181 | go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect 182 | go.opentelemetry.io/otel/trace v1.36.0 // indirect 183 | go.opentelemetry.io/proto/otlp v1.5.0 // indirect 184 | go.uber.org/mock v0.5.2 // indirect 185 | go.yaml.in/yaml/v3 v3.0.4 // indirect 186 | golang.org/x/crypto v0.38.0 // indirect 187 | golang.org/x/net v0.40.0 // indirect 188 | golang.org/x/oauth2 v0.30.0 // indirect 189 | golang.org/x/sync v0.16.0 // indirect 190 | golang.org/x/sys v0.36.0 // indirect 191 | golang.org/x/term v0.32.0 // indirect 192 | golang.org/x/text v0.25.0 // indirect 193 | golang.org/x/time v0.11.0 // indirect 194 | golang.org/x/tools v0.32.0 // indirect 195 | google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a // indirect 196 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect 197 | google.golang.org/grpc v1.74.2 // indirect 198 | google.golang.org/protobuf v1.36.6 // indirect 199 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 200 | gopkg.in/inf.v0 v0.9.1 // indirect 201 | gopkg.in/ini.v1 v1.67.0 // indirect 202 | gopkg.in/yaml.v3 v3.0.1 // indirect 203 | k8s.io/api v0.32.3 // indirect 204 | k8s.io/apimachinery v0.32.3 // indirect 205 | k8s.io/client-go v0.32.3 // indirect 206 | k8s.io/klog/v2 v2.130.1 // indirect 207 | k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect 208 | k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect 209 | sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect 210 | sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect 211 | sigs.k8s.io/yaml v1.4.0 // indirect 212 | tags.cncf.io/container-device-interface v1.0.1 // indirect 213 | ) 214 | --------------------------------------------------------------------------------