├── pkg ├── vfs │ ├── testdata │ │ ├── fs │ │ │ ├── empty │ │ │ └── a │ │ │ │ └── b │ │ │ │ └── c │ │ │ │ └── d │ │ ├── fs.zip │ │ ├── fs.tar.gz │ │ ├── fs.tar.bz2 │ │ ├── update-fs.sh │ │ └── download-data.sh │ ├── README.md │ ├── write_test.go │ ├── bench_test.go │ ├── doc.go │ ├── open_test.go │ ├── map.go │ ├── rewriter.go │ ├── ro.go │ ├── chroot.go │ ├── write.go │ ├── vfs.go │ ├── open.go │ ├── file_util.go │ ├── mounter.go │ ├── fs.go │ ├── file.go │ ├── mem.go │ ├── util.go │ └── vfs_test.go ├── httpcache │ ├── logger.go │ ├── header.go │ ├── bench_test.go │ ├── LICENSE │ ├── cache_test.go │ ├── cachecontrol_test.go │ ├── validator.go │ ├── README.md │ ├── key_test.go │ ├── key.go │ ├── cachecontrol.go │ ├── util_test.go │ ├── resource.go │ └── cache.go ├── stream.v1 │ ├── sync.go │ ├── LICENSE │ ├── fs.go │ ├── reader.go │ ├── memfs.go │ ├── stream.go │ ├── README.md │ └── stream_test.go ├── system │ ├── gc.go │ └── filesize.go └── httplog │ └── log.go ├── .gitignore ├── example ├── assets │ ├── logo.png │ ├── preview.png │ ├── dockerhub.png │ └── logo.svg ├── basic │ └── docker-compose.yml └── specify-mirrors │ └── docker-compose.yml ├── go.mod ├── go.sum ├── apt-proxy.go ├── docker ├── Dockerfile.gorelease └── Dockerfile ├── internal ├── mirrors │ ├── ubuntu_test.go │ ├── ubuntu.go │ ├── mirrors_test.go │ ├── templates.go │ └── mirrors.go ├── server │ ├── page_test.go │ ├── internal_test.go │ ├── internal.go │ └── proxy.go ├── benchmarks │ └── benchmark.go └── rewriter │ └── rewriter.go ├── SECURITY.md ├── .github └── workflows │ ├── scan.yml │ └── release.yaml ├── define ├── alpine.go ├── centos.go ├── define_test.go ├── debian.go ├── ubuntu.go ├── ubuntu-ports.go ├── mirror_test.go └── define.go ├── state └── global.go ├── .goreleaser.yaml ├── cli ├── cli.go └── daemon.go └── README_CN.md /pkg/vfs/testdata/fs/empty: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkg/vfs/testdata/fs/a/b/c/d: -------------------------------------------------------------------------------- 1 | go -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .aptcache 2 | apt-proxy 3 | coverage.out 4 | -------------------------------------------------------------------------------- /example/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/soulteary/apt-proxy/HEAD/example/assets/logo.png -------------------------------------------------------------------------------- /pkg/vfs/testdata/fs.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/soulteary/apt-proxy/HEAD/pkg/vfs/testdata/fs.zip -------------------------------------------------------------------------------- /example/assets/preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/soulteary/apt-proxy/HEAD/example/assets/preview.png -------------------------------------------------------------------------------- /pkg/vfs/testdata/fs.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/soulteary/apt-proxy/HEAD/pkg/vfs/testdata/fs.tar.gz -------------------------------------------------------------------------------- /example/assets/dockerhub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/soulteary/apt-proxy/HEAD/example/assets/dockerhub.png -------------------------------------------------------------------------------- /pkg/vfs/testdata/fs.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/soulteary/apt-proxy/HEAD/pkg/vfs/testdata/fs.tar.bz2 -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/soulteary/apt-proxy 2 | 3 | go 1.23 4 | 5 | require golang.org/x/sys v0.27.0 6 | 7 | replace github.com/soulteary/apt-proxy => ./ 8 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= 2 | golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 3 | -------------------------------------------------------------------------------- /pkg/vfs/testdata/update-fs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | cd fs 5 | zip -r ../fs.zip * 6 | tar cvvf ../fs.tar * 7 | tar cvvzf ../fs.tar.gz * 8 | tar cvvjf ../fs.tar.bz2 * 9 | cd - 10 | -------------------------------------------------------------------------------- /pkg/vfs/README.md: -------------------------------------------------------------------------------- 1 | # vfs 2 | 3 | vfs implements Virtual File Systems with read-write support in Go (golang) 4 | 5 | [![GoDoc](https://godoc.org/github.com/rainycape/vfs?status.svg)](https://godoc.org/github.com/rainycape/vfs) 6 | -------------------------------------------------------------------------------- /example/basic/docker-compose.yml: -------------------------------------------------------------------------------- 1 | name: apt-cache 2 | 3 | services: 4 | apt-proxy: 5 | image: soulteary/apt-proxy 6 | restart: always 7 | environment: 8 | - TZ=Asia/Shanghai 9 | ports: 10 | - "3142:3142" 11 | -------------------------------------------------------------------------------- /example/specify-mirrors/docker-compose.yml: -------------------------------------------------------------------------------- 1 | name: apt-cache 2 | 3 | services: 4 | apt-proxy: 5 | image: soulteary/apt-proxy 6 | restart: always 7 | command: --ubuntu=cn:tsinghua --debian=cn:tsinghua 8 | environment: 9 | - TZ=Asia/Shanghai 10 | ports: 11 | - "3142:3142" 12 | -------------------------------------------------------------------------------- /apt-proxy.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/soulteary/apt-proxy/cli" 8 | ) 9 | 10 | func main() { 11 | flags, err := cli.ParseFlags() 12 | if err != nil { 13 | fmt.Fprintf(os.Stderr, "Error: %v\n", err) 14 | os.Exit(1) 15 | } 16 | cli.Daemon(flags) 17 | } 18 | -------------------------------------------------------------------------------- /docker/Dockerfile.gorelease: -------------------------------------------------------------------------------- 1 | FROM debian:bullseye as certs 2 | RUN apt update && apt install -y ca-certificates 3 | 4 | FROM scratch 5 | COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 6 | COPY apt-proxy /usr/bin/apt-proxy 7 | EXPOSE 3142/tcp 8 | ENTRYPOINT ["/usr/bin/apt-proxy"] -------------------------------------------------------------------------------- /internal/mirrors/ubuntu_test.go: -------------------------------------------------------------------------------- 1 | package mirrors 2 | 3 | import "testing" 4 | 5 | func TestGetUbuntuMirrorUrlsByGeo(t *testing.T) { 6 | mirrors, err := GetUbuntuMirrorUrlsByGeo() 7 | if err != nil { 8 | t.Fatal(err) 9 | } 10 | if len(mirrors) == 0 { 11 | t.Fatal("get ubuntu get mirrors failed") 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /pkg/vfs/testdata/download-data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | SRC=https://storage.googleapis.com/golang/go1.3.src.tar.gz 4 | if which curl > /dev/null 2>&1; then 5 | curl -O ${SRC} 6 | elif which wget > /dev/null 2&1; then 7 | wget -O `basename ${SRC}` ${SRC} 8 | else 9 | echo "no curl nor wget found" 1>&2 10 | exit 1 11 | fi 12 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | All versions are currently being actively maintained. 6 | 7 | | Version | Supported | 8 | | ------- | ------------------ | 9 | | <= 1.0 | :white_check_mark: | 10 | 11 | ## Reporting a Vulnerability 12 | 13 | If you find any risks and vulnerabilities, you are welcome to directly disclose them in the issue, thank you! 14 | -------------------------------------------------------------------------------- /pkg/httpcache/logger.go: -------------------------------------------------------------------------------- 1 | package httpcache 2 | 3 | import "log" 4 | 5 | const ( 6 | ansiRed = "\x1b[31;1m" 7 | ansiReset = "\x1b[0m" 8 | ) 9 | 10 | var DebugLogging = false 11 | 12 | func debugf(format string, args ...interface{}) { 13 | if DebugLogging { 14 | log.Printf(format, args...) 15 | } 16 | } 17 | 18 | func errorf(format string, args ...interface{}) { 19 | log.Printf(ansiRed+"✗ "+format+ansiReset, args) 20 | } 21 | -------------------------------------------------------------------------------- /internal/mirrors/ubuntu.go: -------------------------------------------------------------------------------- 1 | package mirrors 2 | 3 | import ( 4 | "bufio" 5 | "net/http" 6 | 7 | define "github.com/soulteary/apt-proxy/define" 8 | ) 9 | 10 | func GetUbuntuMirrorUrlsByGeo() (mirrors []string, err error) { 11 | response, err := http.Get(define.UBUNTU_GEO_MIRROR_API) 12 | if err != nil { 13 | return mirrors, err 14 | } 15 | defer response.Body.Close() 16 | 17 | scanner := bufio.NewScanner(response.Body) 18 | for scanner.Scan() { 19 | mirrors = append(mirrors, scanner.Text()) 20 | } 21 | return mirrors, scanner.Err() 22 | } 23 | -------------------------------------------------------------------------------- /pkg/httpcache/header.go: -------------------------------------------------------------------------------- 1 | package httpcache 2 | 3 | import ( 4 | "errors" 5 | "net/http" 6 | "strconv" 7 | "time" 8 | ) 9 | 10 | var errNoHeader = errors.New("Header doesn't exist") 11 | 12 | func timeHeader(key string, h http.Header) (time.Time, error) { 13 | if header := h.Get(key); header != "" { 14 | return http.ParseTime(header) 15 | } else { 16 | return time.Time{}, errNoHeader 17 | } 18 | } 19 | 20 | func intHeader(key string, h http.Header) (int, error) { 21 | if header := h.Get(key); header != "" { 22 | return strconv.Atoi(header) 23 | } else { 24 | return 0, errNoHeader 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /pkg/stream.v1/sync.go: -------------------------------------------------------------------------------- 1 | package stream 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | ) 7 | 8 | type broadcaster struct { 9 | sync.RWMutex 10 | closed uint32 11 | *sync.Cond 12 | } 13 | 14 | func newBroadcaster() *broadcaster { 15 | var b broadcaster 16 | b.Cond = sync.NewCond(b.RWMutex.RLocker()) 17 | return &b 18 | } 19 | 20 | func (b *broadcaster) Wait() { 21 | if b.IsOpen() { 22 | b.Cond.Wait() 23 | } 24 | } 25 | 26 | func (b *broadcaster) IsOpen() bool { 27 | return atomic.LoadUint32(&b.closed) == 0 28 | } 29 | 30 | func (b *broadcaster) Close() error { 31 | atomic.StoreUint32(&b.closed, 1) 32 | b.Cond.Broadcast() 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23.3-alpine3.19 as builder 2 | RUN apk update && \ 3 | apk add ca-certificates 4 | ENV TZ=Asia/Shanghai 5 | RUN apk add tzdata && \ 6 | cp /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 7 | WORKDIR /build 8 | ENV GO111MODULE=on 9 | ENV CGO_ENABLED=0 10 | ENV GOPROXY=https://goproxy.cn 11 | COPY go.mod . 12 | COPY go.sum . 13 | RUN go mod download 14 | COPY . . 15 | RUN go build -ldflags "-w -s" 16 | 17 | FROM alpine:3.16.0 18 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | ENV TZ=Asia/Shanghai 20 | COPY --from=builder /etc/localtime /etc/localtime 21 | COPY --from=builder /etc/timezone /etc/timezone 22 | COPY --from=builder /build/apt-proxy /usr/bin/apt-proxy 23 | EXPOSE 3142/tcp 24 | ENTRYPOINT ["/usr/bin/apt-proxy"] -------------------------------------------------------------------------------- /internal/server/page_test.go: -------------------------------------------------------------------------------- 1 | package server_test 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | server "github.com/soulteary/apt-proxy/internal/server" 8 | ) 9 | 10 | func TestGetBaseTemplate(t *testing.T) { 11 | tpl := server.GetBaseTemplate("11000", "11001", "11002", "11003", "11004") 12 | 13 | if !strings.Contains(tpl, "11000") { 14 | t.Fatal("test get base template failed") 15 | } 16 | 17 | if !strings.Contains(tpl, "11001") { 18 | t.Fatal("test get base template failed") 19 | } 20 | 21 | if !strings.Contains(tpl, "11002") { 22 | t.Fatal("test get base template failed") 23 | } 24 | 25 | if !strings.Contains(tpl, "11003") { 26 | t.Fatal("test get base template failed") 27 | } 28 | 29 | if !strings.Contains(tpl, "11004") { 30 | t.Fatal("test get base template failed") 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /pkg/vfs/write_test.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "path/filepath" 7 | "testing" 8 | ) 9 | 10 | type writeTester struct { 11 | name string 12 | writer func(io.Writer, VFS) error 13 | reader func(io.Reader) (VFS, error) 14 | } 15 | 16 | func TestWrite(t *testing.T) { 17 | var ( 18 | writeTests = []writeTester{ 19 | {"zip", WriteZip, func(r io.Reader) (VFS, error) { return Zip(r, 0) }}, 20 | {"tar", WriteTar, Tar}, 21 | {"tar.gz", WriteTarGzip, TarGzip}, 22 | } 23 | ) 24 | p := filepath.Join("testdata", "fs.zip") 25 | fs, err := Open(p) 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | var buf bytes.Buffer 30 | for _, v := range writeTests { 31 | buf.Reset() 32 | if err := v.writer(&buf, fs); err != nil { 33 | t.Fatalf("error writing %s: %s", v.name, err) 34 | } 35 | newFs, err := v.reader(&buf) 36 | if err != nil { 37 | t.Fatalf("error reading %s: %s", v.name, err) 38 | } 39 | testOpenedVFS(t, newFs) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /pkg/vfs/bench_test.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "io" 7 | "os" 8 | "testing" 9 | ) 10 | 11 | func BenchmarkLoadGoSrc(b *testing.B) { 12 | f := openOptionalTestFile(b, goTestFile) 13 | defer f.Close() 14 | // Decompress to avoid measuring the time to gunzip 15 | zr, err := gzip.NewReader(f) 16 | if err != nil { 17 | b.Fatal(err) 18 | } 19 | defer zr.Close() 20 | data, err := io.ReadAll(zr) 21 | if err != nil { 22 | b.Fatal(err) 23 | } 24 | b.ResetTimer() 25 | for ii := 0; ii < b.N; ii++ { 26 | if _, err := Tar(bytes.NewReader(data)); err != nil { 27 | b.Fatal(err) 28 | } 29 | } 30 | } 31 | 32 | func BenchmarkWalkGoSrc(b *testing.B) { 33 | f := openOptionalTestFile(b, goTestFile) 34 | defer f.Close() 35 | fs, err := TarGzip(f) 36 | if err != nil { 37 | b.Fatal(err) 38 | } 39 | b.ResetTimer() 40 | for ii := 0; ii < b.N; ii++ { 41 | Walk(fs, "/", func(_ VFS, _ string, _ os.FileInfo, _ error) error { return nil }) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /pkg/httpcache/bench_test.go: -------------------------------------------------------------------------------- 1 | package httpcache_test 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/httptest" 7 | "net/http/httputil" 8 | "net/url" 9 | "testing" 10 | 11 | "github.com/soulteary/apt-proxy/pkg/httpcache" 12 | ) 13 | 14 | func BenchmarkCachingFiles(b *testing.B) { 15 | backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 16 | w.Header().Add("Cache-Control", "max-age=100000") 17 | fmt.Fprintf(w, "cache server payload") 18 | })) 19 | defer backend.Close() 20 | 21 | u, err := url.Parse(backend.URL) 22 | if err != nil { 23 | b.Fatal(err) 24 | } 25 | 26 | handler := httpcache.NewHandler(httpcache.NewMemoryCache(), httputil.NewSingleHostReverseProxy(u)) 27 | handler.Shared = true 28 | cacheServer := httptest.NewServer(handler) 29 | defer cacheServer.Close() 30 | 31 | for n := 0; n < b.N; n++ { 32 | client := http.Client{} 33 | resp, err := client.Get(fmt.Sprintf("%s/llamas/%d", cacheServer.URL, n)) 34 | if err != nil { 35 | b.Fatal(err) 36 | } 37 | resp.Body.Close() 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pkg/vfs/doc.go: -------------------------------------------------------------------------------- 1 | // Package vfs implements Virtual File Systems with read-write support. 2 | // 3 | // All implementatations use slash ('/') separated paths, with / representing 4 | // the root directory. This means that to manipulate or construct paths, the 5 | // functions in path package should be used, like path.Join or path.Dir. 6 | // There's also no notion of the current directory nor relative paths. The paths 7 | // /a/b/c and a/b/c are considered to point to the same element. 8 | // 9 | // This package also implements some shorthand functions which might be used with 10 | // any VFS implementation, providing the same functionality than functions in the 11 | // io/ioutil, os and path/filepath packages: 12 | // 13 | // io/ioutil.ReadFile => ReadFile 14 | // io/ioutil.WriteFile => WriteFile 15 | // os.IsExist => IsExist 16 | // os.IsNotExist => IsNotExist 17 | // os.MkdirAll => MkdirAll 18 | // os.RemoveAll => RemoveAll 19 | // path/filepath.Walk => Walk 20 | // 21 | // All VFS implementations are thread safe, so multiple readers and writers might 22 | // operate on them at any time. 23 | package vfs 24 | -------------------------------------------------------------------------------- /pkg/vfs/open_test.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | ) 7 | 8 | func testOpenedVFS(t *testing.T, fs VFS) { 9 | data1, err := ReadFile(fs, "a/b/c/d") 10 | if err != nil { 11 | t.Fatal(err) 12 | } 13 | if string(data1) != "go" { 14 | t.Errorf("expecting a/b/c/d to contain \"go\", it contains %q instead", string(data1)) 15 | } 16 | data2, err := ReadFile(fs, "empty") 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | if len(data2) > 0 { 21 | t.Error("non-empty empty file") 22 | } 23 | } 24 | 25 | func testOpenFilename(t *testing.T, filename string) { 26 | p := filepath.Join("testdata", filename) 27 | fs, err := Open(p) 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | testOpenedVFS(t, fs) 32 | } 33 | 34 | func TestOpenZip(t *testing.T) { 35 | testOpenFilename(t, "fs.zip") 36 | } 37 | 38 | func TestOpenTar(t *testing.T) { 39 | testOpenFilename(t, "fs.tar") 40 | } 41 | 42 | func TestOpenTarGzip(t *testing.T) { 43 | testOpenFilename(t, "fs.tar.gz") 44 | } 45 | 46 | func TestOpenTarBzip2(t *testing.T) { 47 | testOpenFilename(t, "fs.tar.bz2") 48 | } 49 | -------------------------------------------------------------------------------- /.github/workflows/scan.yml: -------------------------------------------------------------------------------- 1 | name: "Security Scan" 2 | 3 | # Run workflow each time code is pushed to your repository and on a schedule. 4 | # The scheduled workflow runs every at 00:00 on Sunday UTC time. 5 | on: 6 | workflow_dispatch: 7 | push: 8 | branches: 9 | - main 10 | paths: 11 | - 'cli/**' 12 | - 'linux/**' 13 | - 'pkgs/**' 14 | - 'proxy/**' 15 | - '*.go' 16 | pull_request: 17 | branches: 18 | - main 19 | schedule: 20 | - cron: '0 0 * * 0' 21 | 22 | jobs: 23 | scan: 24 | runs-on: ubuntu-latest 25 | steps: 26 | - name: Check out code into the Go module directory 27 | uses: actions/checkout@v2 28 | - name: Security Scan 29 | uses: securego/gosec@6cd9e6289db3ae9a81f9d0a4f6f7aacb4bca4410 30 | with: 31 | args: '-no-fail -fmt sarif -out results.sarif ./...' 32 | 33 | - name: Temp sarif workaround 34 | shell: bash 35 | run: | 36 | sed -i "/null/d" results.sarif 37 | 38 | - name: Upload SARIF file 39 | uses: github/codeql-action/upload-sarif@v2 40 | with: 41 | sarif_file: results.sarif -------------------------------------------------------------------------------- /pkg/stream.v1/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Dustin H 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /pkg/httpcache/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2010-2014 Lachlan Donald http://lachlan.me 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /pkg/stream.v1/fs.go: -------------------------------------------------------------------------------- 1 | package stream 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "path/filepath" 7 | ) 8 | 9 | // File is a backing data-source for a Stream. 10 | type File interface { 11 | Name() string // The name used to Create/Open the File 12 | io.Reader // Reader must continue reading after EOF on subsequent calls after more Writes. 13 | io.ReaderAt // Similarly to Reader 14 | io.Writer // Concurrent reading/writing must be supported. 15 | io.Closer // Close should do any cleanup when done with the File. 16 | } 17 | 18 | // FileSystem is used to manage Files 19 | type FileSystem interface { 20 | Create(name string) (File, error) // Create must return a new File for Writing 21 | Open(name string) (File, error) // Open must return an existing File for Reading 22 | Remove(name string) error // Remove deletes an existing File 23 | } 24 | 25 | // StdFileSystem is backed by the os package. 26 | var StdFileSystem FileSystem = stdFS{} 27 | 28 | type stdFS struct{} 29 | 30 | func (fs stdFS) Create(name string) (File, error) { 31 | return os.Create(filepath.Clean(name)) 32 | } 33 | 34 | func (fs stdFS) Open(name string) (File, error) { 35 | return os.Open(filepath.Clean(name)) 36 | } 37 | 38 | func (fs stdFS) Remove(name string) error { 39 | return os.Remove(name) 40 | } 41 | -------------------------------------------------------------------------------- /pkg/system/gc.go: -------------------------------------------------------------------------------- 1 | package system 2 | 3 | // https://github.com/soulteary/hosts-blackhole/blob/main/pkg/system/gc.go 4 | 5 | import ( 6 | "fmt" 7 | "runtime" 8 | "runtime/debug" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | func GetMemoryUsageAndGoroutine() (uint64, string) { 14 | var m runtime.MemStats 15 | runtime.ReadMemStats(&m) 16 | return m.Alloc, strconv.Itoa(runtime.NumGoroutine()) 17 | } 18 | 19 | func Stats(logPrint bool) string { 20 | var m runtime.MemStats 21 | runtime.ReadMemStats(&m) 22 | 23 | stats := []string{ 24 | "Runtime Information:", 25 | fmt.Sprintf(" MEM Alloc = %10v MB", toMB(m.Alloc)), 26 | fmt.Sprintf(" MEM HeapAlloc = %10v MB", toMB(m.HeapAlloc)), 27 | fmt.Sprintf(" MEM Sys = %10v MB", toMB(m.Sys)), 28 | fmt.Sprintf(" MEM NumGC = %10v", m.NumGC), 29 | fmt.Sprintf(" RUN NumCPU = %10d", runtime.NumCPU()), 30 | fmt.Sprintf(" RUN NumGoroutine = %10d", runtime.NumGoroutine()), 31 | } 32 | 33 | if logPrint { 34 | for _, info := range stats { 35 | fmt.Println(info) 36 | } 37 | } 38 | return strings.Join(stats, "\n") 39 | } 40 | 41 | func ManualGC() { 42 | runtime.GC() 43 | debug.FreeOSMemory() 44 | Stats(true) 45 | } 46 | 47 | func toMB(b uint64) uint64 { 48 | const bytesInKB = 1024 49 | return b / bytesInKB / bytesInKB 50 | } 51 | -------------------------------------------------------------------------------- /pkg/vfs/map.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "path" 5 | "sort" 6 | ) 7 | 8 | // Map returns an in-memory file system using the given files argument to 9 | // populate it (which might be nil). Note that the files map does 10 | // not need to contain any directories, they will be created automatically. 11 | // If the files contain conflicting paths (e.g. files named a and a/b, thus 12 | // making "a" both a file and a directory), an error will be returned. 13 | func Map(files map[string]*File) (VFS, error) { 14 | fs := newMemory() 15 | keys := make([]string, 0, len(files)) 16 | for k := range files { 17 | keys = append(keys, k) 18 | } 19 | sort.Strings(keys) 20 | var dir *Dir 21 | var prevDir *Dir 22 | var prevDirPath string 23 | for _, k := range keys { 24 | file := files[k] 25 | if file.Mode == 0 { 26 | file.Mode = 0644 27 | } 28 | fileDir, fileBase := path.Split(k) 29 | if prevDir != nil && fileDir == prevDirPath { 30 | dir = prevDir 31 | } else { 32 | if err := MkdirAll(fs, fileDir, 0755); err != nil { 33 | return nil, err 34 | } 35 | var err error 36 | dir, err = fs.dirEntry(fileDir) 37 | if err != nil { 38 | return nil, err 39 | } 40 | prevDir = dir 41 | prevDirPath = fileDir 42 | } 43 | if err := dir.Add(fileBase, file); err != nil { 44 | return nil, err 45 | } 46 | } 47 | return fs, nil 48 | } 49 | -------------------------------------------------------------------------------- /define/alpine.go: -------------------------------------------------------------------------------- 1 | package define 2 | 3 | import "regexp" 4 | 5 | var ALPINE_HOST_PATTERN = regexp.MustCompile(`/alpine/(.+)$`) 6 | 7 | const ALPINE_BENCHMAKR_URL = "MIRRORS.txt" 8 | 9 | // https://mirrors.alpinelinux.org/ 2022.11.19 10 | // Sites that contain protocol headers, restrict access to resources using that protocol 11 | var ALPINE_OFFICIAL_MIRRORS = []string{ 12 | "mirrors.tuna.tsinghua.edu.cn/alpine/", 13 | "mirrors.ustc.edu.cn/alpine/", 14 | "mirrors.nju.edu.cn/alpine/", 15 | // offline "mirror.lzu.edu.cn/alpine/", 16 | "mirrors.sjtug.sjtu.edu.cn/alpine/", 17 | "mirrors.aliyun.com/alpine/", 18 | // not vaild "mirrors.bfsu.edu.cn/alpine", 19 | // offline "mirrors.neusoft.edu.cn/alpine/", 20 | } 21 | 22 | var ALPINE_CUSTOM_MIRRORS = []string{} 23 | 24 | var BUILDIN_ALPINE_MIRRORS = GenerateBuildInList(ALPINE_OFFICIAL_MIRRORS, ALPINE_CUSTOM_MIRRORS) 25 | 26 | var ALPINE_DEFAULT_CACHE_RULES = []Rule{ 27 | {Pattern: regexp.MustCompile(`APKINDEX.tar.gz$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_ALPINE}, 28 | {Pattern: regexp.MustCompile(`tar.gz$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_ALPINE}, 29 | {Pattern: regexp.MustCompile(`apk$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_ALPINE}, 30 | {Pattern: regexp.MustCompile(`.*`), CacheControl: `max-age=100000`, Rewrite: true, OS: TYPE_LINUX_DISTROS_ALPINE}, 31 | } 32 | -------------------------------------------------------------------------------- /pkg/httpcache/cache_test.go: -------------------------------------------------------------------------------- 1 | package httpcache_test 2 | 3 | import ( 4 | "net/http" 5 | "reflect" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/soulteary/apt-proxy/pkg/httpcache" 10 | ) 11 | 12 | func TestSaveResource(t *testing.T) { 13 | var body = strings.Repeat("llamas", 5000) 14 | var cache = httpcache.NewMemoryCache() 15 | 16 | res := httpcache.NewResourceBytes(http.StatusOK, []byte(body), http.Header{ 17 | "Llamas": []string{"true"}, 18 | }) 19 | 20 | if err := cache.Store(res, "testkey"); err != nil { 21 | t.Fatal(err) 22 | } 23 | 24 | resOut, err := cache.Retrieve("testkey") 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | 29 | if resOut == nil { 30 | t.Fatalf("resOut should not be null") 31 | } 32 | 33 | if !reflect.DeepEqual(res.Header(), resOut.Header()) { 34 | t.Fatalf("header should be equal") 35 | } 36 | 37 | if body != readAllString(resOut) { 38 | t.Fatalf("body should be equal") 39 | } 40 | } 41 | 42 | func TestSaveResourceWithIncorrectContentLength(t *testing.T) { 43 | var body = "llamas" 44 | var cache = httpcache.NewMemoryCache() 45 | 46 | res := httpcache.NewResourceBytes(http.StatusOK, []byte(body), http.Header{ 47 | "Llamas": []string{"true"}, 48 | "Content-Length": []string{"10"}, 49 | }) 50 | 51 | if err := cache.Store(res, "testkey"); err == nil { 52 | t.Fatal("Entry should have generated an error") 53 | } 54 | 55 | _, err := cache.Retrieve("testkey") 56 | if err != httpcache.ErrNotFoundInCache { 57 | t.Fatal("Entry shouldn't have been cached") 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pkg/vfs/rewriter.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | type rewriterFileSystem struct { 9 | fs VFS 10 | rewriter func(string) string 11 | } 12 | 13 | func (fs *rewriterFileSystem) VFS() VFS { 14 | return fs.fs 15 | } 16 | 17 | func (fs *rewriterFileSystem) Open(path string) (RFile, error) { 18 | return fs.fs.Open(fs.rewriter(path)) 19 | } 20 | 21 | func (fs *rewriterFileSystem) OpenFile(path string, flag int, perm os.FileMode) (WFile, error) { 22 | return fs.fs.OpenFile(fs.rewriter(path), flag, perm) 23 | } 24 | 25 | func (fs *rewriterFileSystem) Lstat(path string) (os.FileInfo, error) { 26 | return fs.fs.Lstat(fs.rewriter(path)) 27 | } 28 | 29 | func (fs *rewriterFileSystem) Stat(path string) (os.FileInfo, error) { 30 | return fs.fs.Stat(fs.rewriter(path)) 31 | } 32 | 33 | func (fs *rewriterFileSystem) ReadDir(path string) ([]os.FileInfo, error) { 34 | return fs.fs.ReadDir(fs.rewriter(path)) 35 | } 36 | 37 | func (fs *rewriterFileSystem) Mkdir(path string, perm os.FileMode) error { 38 | return fs.fs.Mkdir(fs.rewriter(path), perm) 39 | } 40 | 41 | func (fs *rewriterFileSystem) Remove(path string) error { 42 | return fs.fs.Remove(fs.rewriter(path)) 43 | } 44 | 45 | func (fs *rewriterFileSystem) String() string { 46 | return fmt.Sprintf("Rewriter %s", fs.fs.String()) 47 | } 48 | 49 | // Rewriter returns a file system which uses the provided function 50 | // to rewrite paths. 51 | func Rewriter(fs VFS, rewriter func(oldPath string) (newPath string)) VFS { 52 | if rewriter == nil { 53 | return fs 54 | } 55 | return &rewriterFileSystem{fs: fs, rewriter: rewriter} 56 | } 57 | -------------------------------------------------------------------------------- /pkg/httpcache/cachecontrol_test.go: -------------------------------------------------------------------------------- 1 | package httpcache_test 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | . "github.com/soulteary/apt-proxy/pkg/httpcache" 8 | ) 9 | 10 | func TestParsingCacheControl(t *testing.T) { 11 | table := []struct { 12 | ccString string 13 | ccStruct CacheControl 14 | }{ 15 | {`public, private="set-cookie", max-age=100`, CacheControl{ 16 | "public": []string{}, 17 | "private": []string{"set-cookie"}, 18 | "max-age": []string{"100"}, 19 | }}, 20 | {` foo="max-age=8, space", public`, CacheControl{ 21 | "public": []string{}, 22 | "foo": []string{"max-age=8, space"}, 23 | }}, 24 | {`s-maxage=86400`, CacheControl{ 25 | "s-maxage": []string{"86400"}, 26 | }}, 27 | {`max-stale`, CacheControl{ 28 | "max-stale": []string{}, 29 | }}, 30 | {`max-stale=60`, CacheControl{ 31 | "max-stale": []string{"60"}, 32 | }}, 33 | {`" max-age=8,max-age=8 "=blah`, CacheControl{ 34 | " max-age=8,max-age=8 ": []string{"blah"}, 35 | }}, 36 | } 37 | 38 | for _, expect := range table { 39 | cc, err := ParseCacheControl(expect.ccString) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | if !reflect.DeepEqual(cc, expect.ccStruct) { 44 | t.Fatalf("cc should be equal") 45 | } 46 | if cc.String() == "" { 47 | t.Fatalf("cc string should not be empty") 48 | } 49 | } 50 | } 51 | 52 | func BenchmarkCacheControlParsing(b *testing.B) { 53 | b.ReportAllocs() 54 | b.ResetTimer() 55 | for i := 0; i < b.N; i++ { 56 | _, err := ParseCacheControl(`public, private="set-cookie", max-age=100`) 57 | if err != nil { 58 | b.Fatal(err) 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /pkg/vfs/ro.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | ) 8 | 9 | var ( 10 | // ErrReadOnlyFileSystem is the error returned by read only file systems 11 | // from calls which would result in a write operation. 12 | ErrReadOnlyFileSystem = errors.New("read-only filesystem") 13 | ) 14 | 15 | type readOnlyFileSystem struct { 16 | fs VFS 17 | } 18 | 19 | func (fs *readOnlyFileSystem) VFS() VFS { 20 | return fs.fs 21 | } 22 | 23 | func (fs *readOnlyFileSystem) Open(path string) (RFile, error) { 24 | return fs.fs.Open(path) 25 | } 26 | 27 | func (fs *readOnlyFileSystem) OpenFile(path string, flag int, perm os.FileMode) (WFile, error) { 28 | if flag&(os.O_CREATE|os.O_WRONLY|os.O_RDWR) != 0 { 29 | return nil, ErrReadOnlyFileSystem 30 | } 31 | return fs.fs.OpenFile(path, flag, perm) 32 | } 33 | 34 | func (fs *readOnlyFileSystem) Lstat(path string) (os.FileInfo, error) { 35 | return fs.fs.Lstat(path) 36 | } 37 | 38 | func (fs *readOnlyFileSystem) Stat(path string) (os.FileInfo, error) { 39 | return fs.fs.Stat(path) 40 | } 41 | 42 | func (fs *readOnlyFileSystem) ReadDir(path string) ([]os.FileInfo, error) { 43 | return fs.fs.ReadDir(path) 44 | } 45 | 46 | func (fs *readOnlyFileSystem) Mkdir(path string, perm os.FileMode) error { 47 | return ErrReadOnlyFileSystem 48 | } 49 | 50 | func (fs *readOnlyFileSystem) Remove(path string) error { 51 | return ErrReadOnlyFileSystem 52 | } 53 | 54 | func (fs *readOnlyFileSystem) String() string { 55 | return fmt.Sprintf("RO %s", fs.fs.String()) 56 | } 57 | 58 | // ReadOnly returns a read-only filesystem wrapping the given fs. 59 | func ReadOnly(fs VFS) VFS { 60 | return &readOnlyFileSystem{fs: fs} 61 | } 62 | -------------------------------------------------------------------------------- /pkg/system/filesize.go: -------------------------------------------------------------------------------- 1 | package system 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "os" 7 | "path/filepath" 8 | 9 | "golang.org/x/sys/unix" 10 | ) 11 | 12 | func DiskAvailable() (uint64, error) { 13 | var stat unix.Statfs_t 14 | wd, err := os.Getwd() 15 | if err != nil { 16 | return 0, err 17 | } 18 | 19 | err = unix.Statfs(wd, &stat) 20 | if err != nil { 21 | return 0, err 22 | } 23 | 24 | return uint64(stat.Bavail) * uint64(stat.Bsize), nil 25 | } 26 | 27 | // uint64 ver https://stackoverflow.com/questions/32482673/how-to-get-directory-total-size 28 | func DirSize(path string) (uint64, error) { 29 | var size int64 30 | err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { 31 | if err != nil { 32 | return err 33 | } 34 | if !info.IsDir() { 35 | size += info.Size() 36 | } 37 | return err 38 | }) 39 | return uint64(size), err 40 | } 41 | 42 | // https://hakk.dev/docs/golang-convert-file-size-human-readable/ 43 | func fileSizeRound(val float64, roundOn float64, places int) float64 { 44 | var round float64 45 | pow := math.Pow(10, float64(places)) 46 | digit := pow * val 47 | _, div := math.Modf(digit) 48 | if div >= roundOn { 49 | round = math.Ceil(digit) 50 | } else { 51 | round = math.Floor(digit) 52 | } 53 | return round / pow 54 | } 55 | 56 | // uint64 ver https://programming.guide/go/formatting-byte-size-to-human-readable-format.html 57 | func ByteCountDecimal(b uint64) string { 58 | const unit = 1000 59 | if b < unit { 60 | return fmt.Sprintf("%d B", b) 61 | } 62 | div, exp := uint64(unit), 0 63 | for n := b / unit; n >= unit; n /= unit { 64 | div *= unit 65 | exp++ 66 | } 67 | return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp]) 68 | } 69 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - 'main' 8 | tags: 9 | - 'v*' 10 | env: 11 | GO_VERSION: "1.23" 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | env: 17 | GO111MODULE: on 18 | DOCKER_CLI_EXPERIMENTAL: "enabled" 19 | steps: 20 | - 21 | name: Checkout 22 | uses: actions/checkout@v3 23 | with: 24 | fetch-depth: 0 25 | - 26 | name: Set up Go 27 | uses: actions/setup-go@v3 28 | with: 29 | go-version: ${{ env.GO_VERSION }} 30 | - 31 | name: Set up QEMU 32 | uses: docker/setup-qemu-action@v1 33 | - 34 | name: Cache Go modules 35 | uses: actions/cache@v4 36 | with: 37 | path: ~/go/pkg/mod 38 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 39 | restore-keys: | 40 | ${{ runner.os }}-go- 41 | - 42 | name: Tests 43 | run: | 44 | go mod tidy 45 | go test -v ./... 46 | # Login against a Docker registry except on PR 47 | # https://github.com/docker/login-action 48 | - name: Login to Docker Hub 49 | if: github.event_name != 'pull_request' 50 | uses: docker/login-action@v2 51 | with: 52 | username: ${{ secrets.DOCKERHUB_USERNAME }} 53 | password: ${{ secrets.DOCKERHUB_TOKEN }} 54 | - 55 | name: Run GoReleaser 56 | uses: goreleaser/goreleaser-action@v3 57 | if: success() && startsWith(github.ref, 'refs/tags/') 58 | with: 59 | version: latest 60 | args: release --clean 61 | env: 62 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /define/centos.go: -------------------------------------------------------------------------------- 1 | package define 2 | 3 | import "regexp" 4 | 5 | var CENTOS_HOST_PATTERN = regexp.MustCompile(`/centos/(.+)$`) 6 | 7 | const CENTOS_BENCHMAKR_URL = "TIME" 8 | 9 | // https://www.centos.org/download/mirrors/ 2022.11.19 10 | // Sites that contain protocol headers, restrict access to resources using that protocol 11 | var CENTOS_OFFICIAL_MIRRORS = []string{ 12 | "mirrors.bfsu.edu.cn/centos/", 13 | "mirrors.cqu.edu.cn/CentOS/", 14 | "http://mirrors.neusoft.edu.cn/centos/", 15 | "mirrors.nju.edu.cn/centos/", 16 | "mirrors.huaweicloud.com/centos/", 17 | "mirror.lzu.edu.cn/centos/", 18 | "http://mirrors.njupt.edu.cn/centos/", 19 | "mirrors.163.com/centos/", 20 | "mirrors.bupt.edu.cn/centos/", 21 | "ftp.sjtu.edu.cn/centos/", 22 | "mirrors.tuna.tsinghua.edu.cn/centos/", 23 | "mirrors.ustc.edu.cn/centos/", 24 | } 25 | 26 | var CENTOS_CUSTOM_MIRRORS = []string{ 27 | "mirrors.aliyun.com/centos/", 28 | } 29 | 30 | var BUILDIN_CENTOS_MIRRORS = GenerateBuildInList(CENTOS_OFFICIAL_MIRRORS, CENTOS_CUSTOM_MIRRORS) 31 | 32 | var CENTOS_DEFAULT_CACHE_RULES = []Rule{ 33 | {Pattern: regexp.MustCompile(`repomd.xml$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_CENTOS}, 34 | {Pattern: regexp.MustCompile(`filelist.gz$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_CENTOS}, 35 | {Pattern: regexp.MustCompile(`dir_sizes$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_CENTOS}, 36 | {Pattern: regexp.MustCompile(`TIME$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_CENTOS}, 37 | {Pattern: regexp.MustCompile(`timestamp.txt$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_CENTOS}, 38 | {Pattern: regexp.MustCompile(`.*`), CacheControl: `max-age=100000`, Rewrite: true, OS: TYPE_LINUX_DISTROS_CENTOS}, 39 | } 40 | -------------------------------------------------------------------------------- /pkg/httpcache/validator.go: -------------------------------------------------------------------------------- 1 | package httpcache 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/httptest" 7 | ) 8 | 9 | type Validator struct { 10 | Handler http.Handler 11 | } 12 | 13 | func (v *Validator) Validate(req *http.Request, res *Resource) bool { 14 | outreq := cloneRequest(req) 15 | resHeaders := res.Header() 16 | 17 | if etag := resHeaders.Get("Etag"); etag != "" { 18 | outreq.Header.Set("If-None-Match", etag) 19 | } else if lastMod := resHeaders.Get("Last-Modified"); lastMod != "" { 20 | outreq.Header.Set("If-Modified-Since", lastMod) 21 | } 22 | 23 | t := Clock() 24 | resp := httptest.NewRecorder() 25 | v.Handler.ServeHTTP(resp, outreq) 26 | resp.Flush() 27 | 28 | if age, err := correctedAge(resp.Header(), t, Clock()); err == nil { 29 | resp.Header().Set("Age", fmt.Sprintf("%.f", age.Seconds())) 30 | } 31 | 32 | if headersEqual(resHeaders, resp.Header()) { 33 | res.header = resp.Header() 34 | res.header.Set(ProxyDateHeader, Clock().Format(http.TimeFormat)) 35 | return true 36 | } 37 | 38 | return false 39 | } 40 | 41 | var validationHeaders = []string{"ETag", "Content-MD5", "Last-Modified", "Content-Length"} 42 | 43 | func headersEqual(h1, h2 http.Header) bool { 44 | for _, header := range validationHeaders { 45 | if value := h2.Get(header); value != "" { 46 | if h1.Get(header) != value { 47 | debugf("%s changed, %q != %q", header, value, h1.Get(header)) 48 | return false 49 | } 50 | } 51 | } 52 | 53 | return true 54 | } 55 | 56 | // cloneRequest returns a clone of the provided *http.Request. 57 | // The clone is a shallow copy of the struct and its Header map. 58 | func cloneRequest(r *http.Request) *http.Request { 59 | r2 := new(http.Request) 60 | *r2 = *r 61 | r2.Header = make(http.Header) 62 | for k, s := range r.Header { 63 | r2.Header[k] = s 64 | } 65 | return r2 66 | } 67 | -------------------------------------------------------------------------------- /internal/server/internal_test.go: -------------------------------------------------------------------------------- 1 | package server_test 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "testing" 7 | 8 | server "github.com/soulteary/apt-proxy/internal/server" 9 | ) 10 | 11 | func TestIsInternalUrls(t *testing.T) { 12 | if server.IsInternalUrls("mirrors.tuna.tsinghua.edu.cn/ubuntu/") { 13 | t.Fatal("test internal url failed") 14 | } 15 | if !server.IsInternalUrls(server.INTERNAL_PAGE_HOME) { 16 | t.Fatal("test internal url failed") 17 | } 18 | if !server.IsInternalUrls(server.INTERNAL_PAGE_PING) { 19 | t.Fatal("test internal url failed") 20 | } 21 | } 22 | 23 | func TestGetInternalResType(t *testing.T) { 24 | if server.GetInternalResType(server.INTERNAL_PAGE_HOME) != server.TYPE_HOME { 25 | t.Fatal("test get internal res type failed") 26 | } 27 | 28 | if server.GetInternalResType(server.INTERNAL_PAGE_PING) != server.TYPE_PING { 29 | t.Fatal("test get internal res type failed") 30 | } 31 | 32 | if server.GetInternalResType("/url-not-found") != server.TYPE_NOT_FOUND { 33 | t.Fatal("test get internal res type failed") 34 | } 35 | } 36 | 37 | func TestRenderInternalUrls(t *testing.T) { 38 | cacheDir := "./.aptcache" 39 | res, code := server.RenderInternalUrls(server.INTERNAL_PAGE_PING, cacheDir) 40 | if code != http.StatusOK { 41 | t.Fatal("test render internal urls failed") 42 | } 43 | if res != "pong" { 44 | t.Fatal("test render internal urls failed") 45 | } 46 | 47 | _, code = server.RenderInternalUrls("/url-not-exists", cacheDir) 48 | if code != http.StatusNotFound { 49 | t.Fatal("test render internal urls failed") 50 | } 51 | 52 | res, code = server.RenderInternalUrls(server.INTERNAL_PAGE_HOME, cacheDir) 53 | fmt.Println(res) 54 | if !(code == http.StatusOK || code == http.StatusBadGateway) { 55 | t.Fatal("test render internal urls failed") 56 | } 57 | if res == "" { 58 | t.Fatal("test render internal urls failed") 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /pkg/stream.v1/reader.go: -------------------------------------------------------------------------------- 1 | package stream 2 | 3 | import "io" 4 | 5 | // Reader is a concurrent-safe Stream Reader. 6 | type Reader struct { 7 | s *Stream 8 | file File 9 | } 10 | 11 | // Name returns the name of the underlying File in the FileSystem. 12 | func (r *Reader) Name() string { return r.file.Name() } 13 | 14 | // ReadAt lets you Read from specific offsets in the Stream. 15 | // ReadAt blocks while waiting for the requested section of the Stream to be written, 16 | // unless the Stream is closed in which case it will always return immediately. 17 | func (r *Reader) ReadAt(p []byte, off int64) (n int, err error) { 18 | r.s.b.RLock() 19 | defer r.s.b.RUnlock() 20 | 21 | var m int 22 | 23 | for { 24 | 25 | m, err = r.file.ReadAt(p[n:], off+int64(n)) 26 | n += m 27 | 28 | if r.s.b.IsOpen() { 29 | 30 | switch { 31 | case n != 0 && err == nil: 32 | return n, err 33 | case err == io.EOF: 34 | r.s.b.Wait() 35 | case err != nil: 36 | return n, err 37 | } 38 | 39 | } else { 40 | return n, err 41 | } 42 | 43 | } 44 | } 45 | 46 | // Read reads from the Stream. If the end of an open Stream is reached, Read 47 | // blocks until more data is written or the Stream is Closed. 48 | func (r *Reader) Read(p []byte) (n int, err error) { 49 | r.s.b.RLock() 50 | defer r.s.b.RUnlock() 51 | 52 | var m int 53 | 54 | for { 55 | 56 | m, err = r.file.Read(p[n:]) 57 | n += m 58 | 59 | if r.s.b.IsOpen() { 60 | 61 | switch { 62 | case n != 0 && err == nil: 63 | return n, err 64 | case err == io.EOF: 65 | r.s.b.Wait() 66 | case err != nil: 67 | return n, err 68 | } 69 | 70 | } else { 71 | return n, err 72 | } 73 | 74 | } 75 | } 76 | 77 | // Close closes this Reader on the Stream. This must be called when done with the 78 | // Reader or else the Stream cannot be Removed. 79 | func (r *Reader) Close() error { 80 | defer r.s.dec() 81 | return r.file.Close() 82 | } 83 | -------------------------------------------------------------------------------- /pkg/httpcache/README.md: -------------------------------------------------------------------------------- 1 | 2 | # httpcache 3 | 4 | `httpcache` provides an [rfc7234][] compliant golang [http.Handler](http://golang.org/pkg/net/http/#Handler). 5 | 6 | [![wercker status](https://app.wercker.com/status/a76986990d27e72ea656bb37bb93f59f/m "wercker status")](https://app.wercker.com/project/bykey/a76986990d27e72ea656bb37bb93f59f) 7 | 8 | [![GoDoc](https://godoc.org/github.com/lox/httpcache?status.svg)](https://godoc.org/github.com/lox/httpcache) 9 | 10 | ## Example 11 | 12 | This example is from the included CLI, it runs a caching proxy on http://localhost:8080. 13 | 14 | ```go 15 | proxy := &httputil.ReverseProxy{ 16 | Director: func(r *http.Request) { 17 | }, 18 | } 19 | 20 | handler := httpcache.NewHandler(httpcache.NewMemoryCache(), proxy) 21 | handler.Shared = true 22 | 23 | log.Printf("proxy listening on http://%s", listen) 24 | log.Fatal(http.ListenAndServe(listen, handler)) 25 | ``` 26 | 27 | ## Implemented 28 | 29 | - All of [rfc7234][], except those listed below 30 | - Disk and Memory storage 31 | - Apache-like logging via `httplog` package 32 | 33 | ## Todo 34 | 35 | - Offline operation 36 | - Size constraints on memory/disk cache and cache eviction 37 | - Correctly handle mixture of HTTP1.0 clients and 1.1 upstreams 38 | - More detail in `Via` header 39 | - Support for weak entities with `If-Match` and `If-None-Match` 40 | - Invalidation based on `Content-Location` and request method 41 | - Better handling of duplicate headers and CacheControl values 42 | 43 | ## Caveats 44 | 45 | - Conditional requests are never cached, this includes `Range` requests 46 | 47 | ## Testing 48 | 49 | Tests are currently conducted via the test suite and verified via the [CoAdvisor tool](http://coad.measurement-factory.com/). 50 | 51 | ## Reading List 52 | 53 | - http://httpwg.github.io/specs/rfc7234.html 54 | - https://www.mnot.net/blog/2011/07/11/what_proxies_must_do 55 | - https://www.mnot.net/blog/2014/06/07/rfc2616_is_dead 56 | 57 | [rfc7234]: http://httpwg.github.io/specs/rfc7234.html 58 | -------------------------------------------------------------------------------- /pkg/httpcache/key_test.go: -------------------------------------------------------------------------------- 1 | package httpcache_test 2 | 3 | import ( 4 | "net/url" 5 | "testing" 6 | 7 | "github.com/soulteary/apt-proxy/pkg/httpcache" 8 | ) 9 | 10 | func mustParseUrl(u string) *url.URL { 11 | ru, err := url.Parse(u) 12 | if err != nil { 13 | panic(err) 14 | } 15 | return ru 16 | } 17 | 18 | func TestKeysDiffer(t *testing.T) { 19 | k1 := httpcache.NewKey("GET", mustParseUrl("http://x.org/test"), nil) 20 | k2 := httpcache.NewKey("GET", mustParseUrl("http://y.org/test"), nil) 21 | 22 | if k1.String() == k2.String() { 23 | t.Fatal("key should be same") 24 | } 25 | } 26 | 27 | func TestRequestKey(t *testing.T) { 28 | r := newRequest("GET", "http://x.org/test") 29 | 30 | k1 := httpcache.NewKey("GET", mustParseUrl("http://x.org/test"), nil) 31 | k2 := httpcache.NewRequestKey(r) 32 | 33 | if k1.String() != k2.String() { 34 | t.Fatal("request key should be same") 35 | } 36 | } 37 | 38 | func TestVaryKey(t *testing.T) { 39 | r := newRequest("GET", "http://x.org/test", "Llamas-1: true", "Llamas-2: false") 40 | 41 | k1 := httpcache.NewRequestKey(r) 42 | k2 := httpcache.NewRequestKey(r).Vary("Llamas-1, Llamas-2", r) 43 | 44 | if k1.String() == k2.String() { 45 | t.Fatal("vary key should be same") 46 | } 47 | } 48 | 49 | func TestRequestKeyWithContentLocation(t *testing.T) { 50 | r := newRequest("GET", "http://x.org/test1", "Content-Location: http://x.org/test2") 51 | 52 | k1 := httpcache.NewKey("GET", mustParseUrl("http://x.org/test2"), nil) 53 | k2 := httpcache.NewRequestKey(r) 54 | 55 | if k1.String() != k2.String() { 56 | t.Fatal("request key should with content location") 57 | } 58 | } 59 | 60 | func TestRequestKeyWithIllegalContentLocation(t *testing.T) { 61 | r := newRequest("GET", "http://x.org/test1", "Content-Location: http://y.org/test2") 62 | 63 | k1 := httpcache.NewKey("GET", mustParseUrl("http://x.org/test1"), nil) 64 | k2 := httpcache.NewRequestKey(r) 65 | 66 | if k1.String() != k2.String() { 67 | t.Fatal("request key should with illegal content location") 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /pkg/vfs/chroot.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | ) 8 | 9 | type chrootFileSystem struct { 10 | root string 11 | fs VFS 12 | } 13 | 14 | func (fs *chrootFileSystem) path(p string) string { 15 | // root always ends with /, if there are double 16 | // slashes they will be fixed by the underlying 17 | // VFS 18 | return fs.root + p 19 | } 20 | 21 | func (fs *chrootFileSystem) VFS() VFS { 22 | return fs.fs 23 | } 24 | 25 | func (fs *chrootFileSystem) Open(path string) (RFile, error) { 26 | return fs.fs.Open(fs.path(path)) 27 | } 28 | 29 | func (fs *chrootFileSystem) OpenFile(path string, flag int, perm os.FileMode) (WFile, error) { 30 | return fs.fs.OpenFile(fs.path(path), flag, perm) 31 | } 32 | 33 | func (fs *chrootFileSystem) Lstat(path string) (os.FileInfo, error) { 34 | return fs.fs.Lstat(fs.path(path)) 35 | } 36 | 37 | func (fs *chrootFileSystem) Stat(path string) (os.FileInfo, error) { 38 | return fs.fs.Stat(fs.path(path)) 39 | } 40 | 41 | func (fs *chrootFileSystem) ReadDir(path string) ([]os.FileInfo, error) { 42 | return fs.fs.ReadDir(fs.path(path)) 43 | } 44 | 45 | func (fs *chrootFileSystem) Mkdir(path string, perm os.FileMode) error { 46 | return fs.fs.Mkdir(fs.path(path), perm) 47 | } 48 | 49 | func (fs *chrootFileSystem) Remove(path string) error { 50 | return fs.fs.Remove(fs.path(path)) 51 | } 52 | 53 | func (fs *chrootFileSystem) String() string { 54 | return fmt.Sprintf("Chroot %s %s", fs.root, fs.fs.String()) 55 | } 56 | 57 | // Chroot returns a new VFS wrapping the given VFS, making the given 58 | // directory the new root ("/"). Note that root must be an existing 59 | // directory in the given file system, otherwise an error is returned. 60 | func Chroot(root string, fs VFS) (VFS, error) { 61 | root = path.Clean("/" + root) 62 | st, err := fs.Stat(root) 63 | if err != nil { 64 | return nil, err 65 | } 66 | if !st.IsDir() { 67 | return nil, fmt.Errorf("%s is not a directory", root) 68 | } 69 | return &chrootFileSystem{root: root + "/", fs: fs}, nil 70 | } 71 | -------------------------------------------------------------------------------- /pkg/vfs/write.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "archive/tar" 5 | "archive/zip" 6 | "compress/gzip" 7 | "io" 8 | "os" 9 | ) 10 | 11 | func copyVFS(fs VFS, copier func(p string, info os.FileInfo, f io.Reader) error) error { 12 | return Walk(fs, "/", func(vfs VFS, p string, info os.FileInfo, err error) error { 13 | if err != nil { 14 | return err 15 | } 16 | if info.IsDir() { 17 | return nil 18 | } 19 | f, err := fs.Open(p) 20 | if err != nil { 21 | return err 22 | } 23 | defer f.Close() 24 | return copier(p[1:], info, f) 25 | }) 26 | } 27 | 28 | // WriteZip writes the given VFS as a zip file to the given io.Writer. 29 | func WriteZip(w io.Writer, fs VFS) error { 30 | zw := zip.NewWriter(w) 31 | err := copyVFS(fs, func(p string, info os.FileInfo, f io.Reader) error { 32 | hdr, err := zip.FileInfoHeader(info) 33 | if err != nil { 34 | return err 35 | } 36 | hdr.Name = p 37 | fw, err := zw.CreateHeader(hdr) 38 | if err != nil { 39 | return err 40 | } 41 | _, err = io.Copy(fw, f) 42 | return err 43 | }) 44 | if err != nil { 45 | return err 46 | } 47 | return zw.Close() 48 | } 49 | 50 | // WriteTar writes the given VFS as a tar file to the given io.Writer. 51 | func WriteTar(w io.Writer, fs VFS) error { 52 | tw := tar.NewWriter(w) 53 | err := copyVFS(fs, func(p string, info os.FileInfo, f io.Reader) error { 54 | hdr, err := tar.FileInfoHeader(info, "") 55 | if err != nil { 56 | return err 57 | } 58 | hdr.Name = p 59 | if err := tw.WriteHeader(hdr); err != nil { 60 | return err 61 | } 62 | _, err = io.Copy(tw, f) 63 | return err 64 | }) 65 | if err != nil { 66 | return err 67 | } 68 | return tw.Close() 69 | } 70 | 71 | // WriteTarGzip writes the given VFS as a tar.gz file to the given io.Writer. 72 | func WriteTarGzip(w io.Writer, fs VFS) error { 73 | gw, err := gzip.NewWriterLevel(w, gzip.BestCompression) 74 | if err != nil { 75 | return err 76 | } 77 | if err := WriteTar(gw, fs); err != nil { 78 | return err 79 | } 80 | return gw.Close() 81 | } 82 | -------------------------------------------------------------------------------- /pkg/httpcache/key.go: -------------------------------------------------------------------------------- 1 | package httpcache 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | "strings" 9 | ) 10 | 11 | // Key represents a unique identifier for a resource in the cache 12 | type Key struct { 13 | method string 14 | header http.Header 15 | u url.URL 16 | vary []string 17 | } 18 | 19 | // NewKey returns a new Key instance 20 | func NewKey(method string, u *url.URL, h http.Header) Key { 21 | return Key{method: method, header: h, u: *u, vary: []string{}} 22 | } 23 | 24 | // NewRequestKey generates a Key for a request 25 | func NewRequestKey(r *http.Request) Key { 26 | URL := r.URL 27 | 28 | if location := r.Header.Get("Content-Location"); location != "" { 29 | u, err := url.Parse(location) 30 | if err == nil { 31 | if !u.IsAbs() { 32 | u = r.URL.ResolveReference(u) 33 | } 34 | if u.Host != r.Host { 35 | debugf("illegal host %q in Content-Location", u.Host) 36 | } else { 37 | debugf("using Content-Location: %q", u.String()) 38 | URL = u 39 | } 40 | } else { 41 | debugf("failed to parse Content-Location %q", location) 42 | } 43 | } 44 | 45 | return NewKey(r.Method, URL, r.Header) 46 | } 47 | 48 | // ForMethod returns a new Key with a given method 49 | func (k Key) ForMethod(method string) Key { 50 | k2 := k 51 | k2.method = method 52 | return k2 53 | } 54 | 55 | // Vary returns a Key that is varied on particular headers in a http.Request 56 | func (k Key) Vary(varyHeader string, r *http.Request) Key { 57 | k2 := k 58 | 59 | for _, header := range strings.Split(varyHeader, ", ") { 60 | k2.vary = append(k2.vary, header+"="+r.Header.Get(header)) 61 | } 62 | 63 | return k2 64 | } 65 | 66 | func (k Key) String() string { 67 | URL := strings.ToLower(canonicalURL(&k.u).String()) 68 | b := &bytes.Buffer{} 69 | b.WriteString(fmt.Sprintf("%s:%s", k.method, URL)) 70 | 71 | if len(k.vary) > 0 { 72 | b.WriteString("::") 73 | for _, v := range k.vary { 74 | b.WriteString(v + ":") 75 | } 76 | } 77 | 78 | return b.String() 79 | } 80 | 81 | func canonicalURL(u *url.URL) *url.URL { 82 | return u 83 | } 84 | -------------------------------------------------------------------------------- /define/define_test.go: -------------------------------------------------------------------------------- 1 | package define_test 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | "testing" 8 | 9 | define "github.com/soulteary/apt-proxy/define" 10 | ) 11 | 12 | func TestRuleToString(t *testing.T) { 13 | r := define.Rule{ 14 | Pattern: regexp.MustCompile(`a$`), 15 | CacheControl: "1", 16 | Rewrite: true, 17 | } 18 | 19 | expect := fmt.Sprintf("%s Cache-Control=%s Rewrite=%#v", r.Pattern.String(), r.CacheControl, r.Rewrite) 20 | if expect != r.String() { 21 | t.Fatal("parse rule to string failed") 22 | } 23 | } 24 | 25 | func TestGenerateAliasFromURL(t *testing.T) { 26 | if define.GenerateAliasFromURL("http://mirrors.cn99.com/ubuntu/") != "cn:cn99" { 27 | t.Fatal("generate alias from url failed") 28 | } 29 | 30 | if define.GenerateAliasFromURL("https://mirrors.tuna.tsinghua.edu.cn/ubuntu/") != "cn:tsinghua" { 31 | t.Fatal("generate alias from url failed") 32 | } 33 | 34 | if define.GenerateAliasFromURL("mirrors.cnnic.cn/ubuntu/") != "cn:cnnic" { 35 | t.Fatal("generate alias from url failed") 36 | } 37 | } 38 | 39 | func TestGenerateBuildInMirorItem(t *testing.T) { 40 | mirror := define.GenerateBuildInMirorItem("http://mirrors.tuna.tsinghua.edu.cn/ubuntu/", true) 41 | if !(mirror.Http == true && mirror.Https == false) || mirror.Official != true { 42 | t.Fatal("generate build-in mirror item failed") 43 | } 44 | mirror = define.GenerateBuildInMirorItem("https://mirrors.tuna.tsinghua.edu.cn/ubuntu/", false) 45 | if !(mirror.Http == false && mirror.Https == true) || mirror.Official != false { 46 | t.Fatal("generate build-in mirror item failed") 47 | } 48 | } 49 | 50 | func TestGenerateBuildInList(t *testing.T) { 51 | mirrors := define.GenerateBuildInList(define.UBUNTU_OFFICIAL_MIRRORS, define.UBUNTU_CUSTOM_MIRRORS) 52 | 53 | count := 0 54 | for _, url := range define.UBUNTU_OFFICIAL_MIRRORS { 55 | if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { 56 | count += 1 57 | } else { 58 | count += 2 59 | } 60 | } 61 | for _, url := range define.UBUNTU_CUSTOM_MIRRORS { 62 | if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { 63 | count += 1 64 | } else { 65 | count += 2 66 | } 67 | } 68 | 69 | if len(mirrors) != count { 70 | t.Fatal("generate build-in mirror list failed") 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /pkg/stream.v1/memfs.go: -------------------------------------------------------------------------------- 1 | package stream 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io" 7 | "sync" 8 | ) 9 | 10 | // ErrNotFoundInMem is returned when an in-memory FileSystem cannot find a file. 11 | var ErrNotFoundInMem = errors.New("not found") 12 | 13 | type memfs struct { 14 | mu sync.RWMutex 15 | files map[string]*memFile 16 | } 17 | 18 | // NewMemFS returns a New in-memory FileSystem 19 | func NewMemFS() FileSystem { 20 | return &memfs{ 21 | files: make(map[string]*memFile), 22 | } 23 | } 24 | 25 | func (fs *memfs) Create(key string) (File, error) { 26 | fs.mu.Lock() 27 | defer fs.mu.Unlock() 28 | 29 | file := &memFile{ 30 | name: key, 31 | r: bytes.NewBuffer(nil), 32 | } 33 | file.memReader.memFile = file 34 | fs.files[key] = file 35 | return file, nil 36 | } 37 | 38 | func (fs *memfs) Open(key string) (File, error) { 39 | fs.mu.RLock() 40 | defer fs.mu.RUnlock() 41 | 42 | if f, ok := fs.files[key]; ok { 43 | return &memReader{memFile: f}, nil 44 | } 45 | return nil, ErrNotFoundInMem 46 | } 47 | 48 | func (fs *memfs) Remove(key string) error { 49 | fs.mu.Lock() 50 | defer fs.mu.Unlock() 51 | delete(fs.files, key) 52 | return nil 53 | } 54 | 55 | type memFile struct { 56 | mu sync.RWMutex 57 | name string 58 | r *bytes.Buffer 59 | memReader 60 | } 61 | 62 | func (f *memFile) Name() string { 63 | return f.name 64 | } 65 | 66 | func (f *memFile) Write(p []byte) (int, error) { 67 | if len(p) > 0 { 68 | f.mu.Lock() 69 | defer f.mu.Unlock() 70 | return f.r.Write(p) 71 | } 72 | return len(p), nil 73 | } 74 | 75 | func (f *memFile) Bytes() []byte { 76 | f.mu.RLock() 77 | defer f.mu.RUnlock() 78 | return f.r.Bytes() 79 | } 80 | 81 | func (f *memFile) Close() error { 82 | return nil 83 | } 84 | 85 | type memReader struct { 86 | *memFile 87 | n int 88 | } 89 | 90 | func (r *memReader) ReadAt(p []byte, off int64) (n int, err error) { 91 | data := r.Bytes() 92 | if int64(len(data)) < off { 93 | return 0, io.EOF 94 | } 95 | n, err = bytes.NewReader(data[off:]).ReadAt(p, 0) 96 | return n, err 97 | } 98 | 99 | func (r *memReader) Read(p []byte) (n int, err error) { 100 | n, err = bytes.NewReader(r.Bytes()[r.n:]).Read(p) 101 | r.n += n 102 | return n, err 103 | } 104 | 105 | func (r *memReader) Close() error { 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /internal/server/internal.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | "path/filepath" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/soulteary/apt-proxy/pkg/system" 11 | ) 12 | 13 | const ( 14 | INTERNAL_PAGE_HOME string = "/" 15 | INTERNAL_PAGE_PING string = "/_/ping/" 16 | ) 17 | 18 | const ( 19 | TYPE_NOT_FOUND int = 0 20 | TYPE_HOME int = 1 21 | TYPE_PING int = 2 22 | ) 23 | 24 | func IsInternalUrls(url string) bool { 25 | u := strings.ToLower(url) 26 | return !(strings.Contains(u, "/ubuntu") || strings.Contains(u, "/debian") || strings.Contains(u, "/centos") || strings.Contains(u, "/alpine")) 27 | } 28 | 29 | func GetInternalResType(url string) int { 30 | if url == INTERNAL_PAGE_HOME { 31 | return TYPE_HOME 32 | } 33 | 34 | if url == INTERNAL_PAGE_PING { 35 | return TYPE_PING 36 | } 37 | 38 | return TYPE_NOT_FOUND 39 | } 40 | 41 | const LABEL_NO_VALID_VALUE = "N/A" 42 | 43 | func RenderInternalUrls(url string, cacheDir string) (string, int) { 44 | switch GetInternalResType(url) { 45 | case TYPE_HOME: 46 | cacheSizeLabel := LABEL_NO_VALID_VALUE 47 | cacheSize, err := system.DirSize(cacheDir) 48 | if err == nil { 49 | cacheSizeLabel = system.ByteCountDecimal(cacheSize) 50 | // } else { 51 | // return "Get Cache Size Failed", http.StatusBadGateway 52 | } 53 | 54 | filesNumberLabel := LABEL_NO_VALID_VALUE 55 | cacheMetaDir := filepath.Join(cacheDir, "header", "v1") 56 | if _, err := os.Stat(cacheMetaDir); !os.IsNotExist(err) { 57 | files, err := os.ReadDir(cacheMetaDir) 58 | if err == nil { 59 | filesNumberLabel = strconv.Itoa(len(files)) 60 | // } else { 61 | // return "Get Cache Meta Dir Failed", http.StatusBadGateway 62 | } 63 | // } else { 64 | // return "Get Cache Meta Failed", http.StatusBadGateway 65 | } 66 | 67 | diskAvailableLabel := LABEL_NO_VALID_VALUE 68 | available, err := system.DiskAvailable() 69 | if err == nil { 70 | diskAvailableLabel = system.ByteCountDecimal(available) 71 | // } else { 72 | // return "Get Disk Available Failed", http.StatusBadGateway 73 | } 74 | 75 | memoryUsageLabel := LABEL_NO_VALID_VALUE 76 | memoryUsage, goroutine := system.GetMemoryUsageAndGoroutine() 77 | memoryUsageLabel = system.ByteCountDecimal(memoryUsage) 78 | 79 | return GetBaseTemplate(cacheSizeLabel, filesNumberLabel, diskAvailableLabel, memoryUsageLabel, goroutine), 200 80 | case TYPE_PING: 81 | return "pong", http.StatusOK 82 | } 83 | return "Not Found", http.StatusNotFound 84 | } 85 | -------------------------------------------------------------------------------- /define/debian.go: -------------------------------------------------------------------------------- 1 | package define 2 | 3 | import "regexp" 4 | 5 | const ( 6 | DEBIAN_BENCHMAKR_URL = "dists/bullseye/main/binary-amd64/Release" 7 | ) 8 | 9 | var DEBIAN_HOST_PATTERN = regexp.MustCompile(`/debian(-security)?/(.+)$`) 10 | 11 | // https://www.debian.org/mirror/list 2022.11.19 12 | // Sites that contain protocol headers, restrict access to resources using that protocol 13 | var DEBIAN_OFFICIAL_MIRRORS = []string{ 14 | "http://ftp.cn.debian.org/debian/", 15 | "mirror.bjtu.edu.cn/debian/", 16 | "mirrors.163.com/debian/", 17 | "mirrors.bfsu.edu.cn/debian/", 18 | "mirrors.huaweicloud.com/debian/", 19 | "http://mirrors.neusoft.edu.cn/debian/", 20 | "mirrors.tuna.tsinghua.edu.cn/debian/", 21 | "mirrors.ustc.edu.cn/debian/", 22 | } 23 | 24 | var DEBIAN_CUSTOM_MIRRORS = []string{ 25 | "repo.huaweicloud.com/debian/", 26 | "mirrors.cloud.tencent.com/debian/", 27 | "mirrors.hit.edu.cn/debian/", 28 | "mirrors.aliyun.com/debian/", 29 | "mirror.lzu.edu.cn/debian/", 30 | "mirror.nju.edu.cn/debian/", 31 | } 32 | 33 | var BUILDIN_DEBIAN_MIRRORS = GenerateBuildInList(DEBIAN_OFFICIAL_MIRRORS, DEBIAN_CUSTOM_MIRRORS) 34 | 35 | var DEBIAN_DEFAULT_CACHE_RULES = []Rule{ 36 | {Pattern: regexp.MustCompile(`deb$`), CacheControl: `max-age=100000`, Rewrite: true, OS: TYPE_LINUX_DISTROS_DEBIAN}, 37 | {Pattern: regexp.MustCompile(`udeb$`), CacheControl: `max-age=100000`, Rewrite: true, OS: TYPE_LINUX_DISTROS_DEBIAN}, 38 | {Pattern: regexp.MustCompile(`InRelease$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_DEBIAN}, 39 | {Pattern: regexp.MustCompile(`DiffIndex$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_DEBIAN}, 40 | {Pattern: regexp.MustCompile(`PackagesIndex$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_DEBIAN}, 41 | {Pattern: regexp.MustCompile(`Packages\.(bz2|gz|lzma)$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_DEBIAN}, 42 | {Pattern: regexp.MustCompile(`SourcesIndex$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_DEBIAN}, 43 | {Pattern: regexp.MustCompile(`Sources\.(bz2|gz|lzma)$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_DEBIAN}, 44 | {Pattern: regexp.MustCompile(`Release(\.gpg)?$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_DEBIAN}, 45 | {Pattern: regexp.MustCompile(`Translation-(en|fr)\.(gz|bz2|bzip2|lzma)$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_DEBIAN}, 46 | // Add file file hash 47 | {Pattern: regexp.MustCompile(`/by-hash/`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_DEBIAN}, 48 | } 49 | -------------------------------------------------------------------------------- /pkg/httpcache/cachecontrol.go: -------------------------------------------------------------------------------- 1 | package httpcache 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net/http" 7 | "sort" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | const ( 13 | CacheControlHeader = "Cache-Control" 14 | ) 15 | 16 | type CacheControl map[string][]string 17 | 18 | func ParseCacheControlHeaders(h http.Header) (CacheControl, error) { 19 | return ParseCacheControl(strings.Join(h["Cache-Control"], ", ")) 20 | } 21 | 22 | func ParseCacheControl(input string) (CacheControl, error) { 23 | cc := make(CacheControl) 24 | length := len(input) 25 | isValue := false 26 | lastKey := "" 27 | 28 | for pos := 0; pos < length; pos++ { 29 | var token string 30 | switch input[pos] { 31 | case '"': 32 | if offset := strings.IndexAny(input[pos+1:], `"`); offset != -1 { 33 | token = input[pos+1 : pos+1+offset] 34 | } else { 35 | token = input[pos+1:] 36 | } 37 | pos += len(token) + 1 38 | case ',', '\n', '\r', ' ', '\t': 39 | continue 40 | case '=': 41 | isValue = true 42 | continue 43 | default: 44 | if offset := strings.IndexAny(input[pos:], "\"\n\t\r ,="); offset != -1 { 45 | token = input[pos : pos+offset] 46 | } else { 47 | token = input[pos:] 48 | } 49 | pos += len(token) - 1 50 | } 51 | if isValue { 52 | cc.Add(lastKey, token) 53 | isValue = false 54 | } else { 55 | cc.Add(token, "") 56 | lastKey = token 57 | } 58 | } 59 | 60 | return cc, nil 61 | } 62 | 63 | func (cc CacheControl) Get(key string) (string, bool) { 64 | v, exists := cc[key] 65 | if exists && len(v) > 0 { 66 | return v[0], true 67 | } 68 | return "", exists 69 | } 70 | 71 | func (cc CacheControl) Add(key, val string) { 72 | if !cc.Has(key) { 73 | cc[key] = []string{} 74 | } 75 | if val != "" { 76 | cc[key] = append(cc[key], val) 77 | } 78 | } 79 | 80 | func (cc CacheControl) Has(key string) bool { 81 | _, exists := cc[key] 82 | return exists 83 | } 84 | 85 | func (cc CacheControl) Duration(key string) (time.Duration, error) { 86 | d, _ := cc.Get(key) 87 | return time.ParseDuration(d + "s") 88 | } 89 | 90 | func (cc CacheControl) String() string { 91 | keys := make([]string, len(cc)) 92 | for k := range cc { 93 | keys = append(keys, k) 94 | } 95 | sort.Strings(keys) 96 | buf := bytes.Buffer{} 97 | 98 | for _, k := range keys { 99 | vals := cc[k] 100 | if len(vals) == 0 { 101 | buf.WriteString(k + ", ") 102 | } 103 | for _, val := range vals { 104 | buf.WriteString(fmt.Sprintf("%s=%q, ", k, val)) 105 | } 106 | } 107 | 108 | return strings.TrimSuffix(buf.String(), ", ") 109 | } 110 | -------------------------------------------------------------------------------- /pkg/stream.v1/stream.go: -------------------------------------------------------------------------------- 1 | // Package stream provides a way to read and write to a synchronous buffered pipe, with multiple reader support. 2 | package stream 3 | 4 | import ( 5 | "errors" 6 | "sync" 7 | ) 8 | 9 | // ErrRemoving is returned when requesting a Reader on a Stream which is being Removed. 10 | var ErrRemoving = errors.New("cannot open a new reader while removing file") 11 | 12 | // Stream is used to concurrently Write and Read from a File. 13 | type Stream struct { 14 | grp sync.WaitGroup 15 | b *broadcaster 16 | file File 17 | fs FileSystem 18 | removing chan struct{} 19 | } 20 | 21 | // New creates a new Stream from the StdFileSystem with Name "name". 22 | func New(name string) (*Stream, error) { 23 | return NewStream(name, StdFileSystem) 24 | } 25 | 26 | // NewStream creates a new Stream with Name "name" in FileSystem fs. 27 | func NewStream(name string, fs FileSystem) (*Stream, error) { 28 | f, err := fs.Create(name) 29 | sf := &Stream{ 30 | file: f, 31 | fs: fs, 32 | b: newBroadcaster(), 33 | removing: make(chan struct{}), 34 | } 35 | sf.inc() 36 | return sf, err 37 | } 38 | 39 | // Name returns the name of the underlying File in the FileSystem. 40 | func (s *Stream) Name() string { return s.file.Name() } 41 | 42 | // Write writes p to the Stream. It's concurrent safe to be called with Stream's other methods. 43 | func (s *Stream) Write(p []byte) (int, error) { 44 | defer s.b.Broadcast() 45 | s.b.Lock() 46 | defer s.b.Unlock() 47 | return s.file.Write(p) 48 | } 49 | 50 | // Close will close the active stream. This will cause Readers to return EOF once they have 51 | // read the entire stream. 52 | func (s *Stream) Close() error { 53 | defer s.dec() 54 | defer s.b.Close() 55 | s.b.Lock() 56 | defer s.b.Unlock() 57 | return s.file.Close() 58 | } 59 | 60 | // Remove will block until the Stream and all its Readers have been Closed, 61 | // at which point it will delete the underlying file. NextReader() will return 62 | // ErrRemoving if called after Remove. 63 | func (s *Stream) Remove() error { 64 | close(s.removing) 65 | s.grp.Wait() 66 | return s.fs.Remove(s.file.Name()) 67 | } 68 | 69 | // NextReader will return a concurrent-safe Reader for this stream. Each Reader will 70 | // see a complete and independent view of the stream, and can Read will the stream 71 | // is written to. 72 | func (s *Stream) NextReader() (*Reader, error) { 73 | s.inc() 74 | 75 | select { 76 | case <-s.removing: 77 | s.dec() 78 | return nil, ErrRemoving 79 | default: 80 | } 81 | 82 | file, err := s.fs.Open(s.file.Name()) 83 | if err != nil { 84 | s.dec() 85 | return nil, err 86 | } 87 | 88 | return &Reader{file: file, s: s}, nil 89 | } 90 | 91 | func (s *Stream) inc() { s.grp.Add(1) } 92 | func (s *Stream) dec() { s.grp.Done() } 93 | -------------------------------------------------------------------------------- /pkg/stream.v1/README.md: -------------------------------------------------------------------------------- 1 | stream 2 | ========== 3 | 4 | [![GoDoc](https://godoc.org/github.com/djherbis/stream?status.svg)](https://godoc.org/github.com/djherbis/stream) 5 | [![Release](https://img.shields.io/github/release/djherbis/stream.svg)](https://github.com/djherbis/stream/releases/latest) 6 | [![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE.txt) 7 | [![Build Status](https://travis-ci.org/djherbis/stream.svg?branch=master)](https://travis-ci.org/djherbis/stream) 8 | [![Coverage Status](https://coveralls.io/repos/djherbis/stream/badge.svg?branch=master)](https://coveralls.io/r/djherbis/stream?branch=master) 9 | 10 | Usage 11 | ------------ 12 | 13 | Write and Read concurrently, and independently. 14 | 15 | To explain further, if you need to write to multiple places you can use io.MultiWriter, 16 | if you need multiple Readers on something you can use io.TeeReader. If you want concurrency you can use io.Pipe(). 17 | 18 | However all of these methods "tie" each Read/Write together, your readers can't read from different places in the stream, each write must be distributed to all readers in sequence. 19 | 20 | This package provides a way for multiple Readers to read off the same Writer, without waiting for the others. This is done by writing to a "File" interface which buffers the input so it can be read at any time from many independent readers. Readers can even be created while writing or after the stream is closed. They will all see a consistent view of the stream and will block until the section of the stream they request is written, all while being unaffected by the actions of the other readers. 21 | 22 | The use case for this stems from my other project djherbis/fscache. I needed a byte caching mechanism which allowed many independent clients to have access to the data while it was being written, rather than re-generating the byte stream for each of them or waiting for a complete copy of the stream which could be stored and then re-used. 23 | 24 | ```go 25 | import( 26 | "io" 27 | "log" 28 | "os" 29 | "time" 30 | 31 | "github.com/djherbis/stream" 32 | ) 33 | 34 | func main(){ 35 | w, err := stream.New("mystream") 36 | if err != nil { 37 | log.Fatal(err) 38 | } 39 | 40 | go func(){ 41 | io.WriteString(w, "Hello World!") 42 | <-time.After(time.Second) 43 | io.WriteString(w, "Streaming updates...") 44 | w.Close() 45 | }() 46 | 47 | waitForReader := make(chan struct{}) 48 | go func(){ 49 | // Read from the stream 50 | r, err := w.NextReader() 51 | if err != nil { 52 | log.Fatal(err) 53 | } 54 | io.Copy(os.Stdout, r) // Hello World! (1 second) Streaming updates... 55 | r.Close() 56 | close(waitForReader) 57 | }() 58 | 59 | // Full copy of the stream! 60 | r, err := w.NextReader() 61 | if err != nil { 62 | log.Fatal(err) 63 | } 64 | io.Copy(os.Stdout, r) // Hello World! (1 second) Streaming updates... 65 | 66 | // r supports io.ReaderAt too. 67 | p := make([]byte, 4) 68 | r.ReadAt(p, 1) // Read "ello" into p 69 | 70 | r.Close() 71 | 72 | <-waitForReader // don't leave main before go-routine finishes 73 | } 74 | ``` 75 | 76 | Installation 77 | ------------ 78 | ```sh 79 | go get github.com/djherbis/stream 80 | ``` 81 | -------------------------------------------------------------------------------- /define/ubuntu.go: -------------------------------------------------------------------------------- 1 | package define 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | const ( 8 | UBUNTU_GEO_MIRROR_API = "http://mirrors.ubuntu.com/mirrors.txt" 9 | UBUNTU_BENCHMAKR_URL = "dists/noble/main/binary-amd64/Release" 10 | ) 11 | 12 | var UBUNTU_HOST_PATTERN = regexp.MustCompile(`/ubuntu/(.+)$`) 13 | 14 | // http://mirrors.ubuntu.com/mirrors.txt 2022.11.19 15 | // Sites that contain protocol headers, restrict access to resources using that protocol 16 | var UBUNTU_OFFICIAL_MIRRORS = []string{ 17 | "mirrors.cn99.com/ubuntu/", 18 | "mirrors.tuna.tsinghua.edu.cn/ubuntu/", 19 | "mirrors.cnnic.cn/ubuntu/", 20 | "mirror.bjtu.edu.cn/ubuntu/", 21 | "mirrors.cqu.edu.cn/ubuntu/", 22 | "http://mirrors.skyshe.cn/ubuntu/", 23 | // duplicate "mirrors.tuna.tsinghua.edu.cn/ubuntu-ports/", 24 | "mirrors.yun-idc.com/ubuntu/", 25 | "http://mirror.dlut.edu.cn/ubuntu/", 26 | "mirrors.xjtu.edu.cn/ubuntu/", 27 | "mirrors.huaweicloud.com/repository/ubuntu/", 28 | "mirrors.bupt.edu.cn/ubuntu/", 29 | "mirrors.hit.edu.cn/ubuntu/", 30 | // duplicate "repo.huaweicloud.com/ubuntu/", 31 | "http://mirrors.sohu.com/ubuntu/", 32 | "mirror.nju.edu.cn/ubuntu/", 33 | "mirrors.bfsu.edu.cn/ubuntu/", 34 | "mirror.lzu.edu.cn/ubuntu/", 35 | "mirrors.aliyun.com/ubuntu/", 36 | "ftp.sjtu.edu.cn/ubuntu/", 37 | "mirrors.njupt.edu.cn/ubuntu/", 38 | "mirrors.cloud.tencent.com/ubuntu/", 39 | "http://mirrors.dgut.edu.cn/ubuntu/", 40 | "mirrors.ustc.edu.cn/ubuntu/", 41 | "mirrors.sdu.edu.cn/ubuntu/", 42 | "http://cn.archive.ubuntu.com/ubuntu/", 43 | } 44 | 45 | var UBUNTU_CUSTOM_MIRRORS = []string{ 46 | "mirrors.163.com/ubuntu/", 47 | } 48 | 49 | var BUILDIN_UBUNTU_MIRRORS = GenerateBuildInList(UBUNTU_OFFICIAL_MIRRORS, UBUNTU_CUSTOM_MIRRORS) 50 | 51 | var UBUNTU_DEFAULT_CACHE_RULES = []Rule{ 52 | {Pattern: regexp.MustCompile(`deb$`), CacheControl: `max-age=100000`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU}, 53 | {Pattern: regexp.MustCompile(`udeb$`), CacheControl: `max-age=100000`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU}, 54 | {Pattern: regexp.MustCompile(`InRelease$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU}, 55 | {Pattern: regexp.MustCompile(`DiffIndex$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU}, 56 | {Pattern: regexp.MustCompile(`PackagesIndex$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU}, 57 | {Pattern: regexp.MustCompile(`Packages\.(bz2|gz|lzma)$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU}, 58 | {Pattern: regexp.MustCompile(`SourcesIndex$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU}, 59 | {Pattern: regexp.MustCompile(`Sources\.(bz2|gz|lzma)$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU}, 60 | {Pattern: regexp.MustCompile(`Release(\.gpg)?$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU}, 61 | {Pattern: regexp.MustCompile(`Translation-(en|fr)\.(gz|bz2|bzip2|lzma)$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU}, 62 | // Add file file hash 63 | {Pattern: regexp.MustCompile(`\/by-hash\/`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU}, 64 | } 65 | -------------------------------------------------------------------------------- /pkg/vfs/vfs.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "io" 5 | "os" 6 | ) 7 | 8 | // Opener is the interface which specifies the methods for 9 | // opening a file. All the VFS implementations implement 10 | // this interface. 11 | type Opener interface { 12 | // Open returns a readable file at the given path. See also 13 | // the shorthand function ReadFile. 14 | Open(path string) (RFile, error) 15 | // OpenFile returns a readable and writable file at the given 16 | // path. Note that, depending on the flags, the file might be 17 | // only readable or only writable. See also the shorthand 18 | // function WriteFile. 19 | OpenFile(path string, flag int, perm os.FileMode) (WFile, error) 20 | } 21 | 22 | // RFile is the interface implemented by the returned value from a VFS 23 | // Open method. It allows reading and seeking, and must be closed after use. 24 | type RFile interface { 25 | io.Reader 26 | io.Seeker 27 | io.Closer 28 | } 29 | 30 | // WFile is the interface implemented by the returned value from a VFS 31 | // OpenFile method. It allows reading, seeking and writing, and must 32 | // be closed after use. Note that, depending on the flags passed to 33 | // OpenFile, the Read or Write methods might always return an error (e.g. 34 | // if the file was opened in read-only or write-only mode). 35 | type WFile interface { 36 | io.Reader 37 | io.Writer 38 | io.Seeker 39 | io.Closer 40 | } 41 | 42 | // VFS is the interface implemented by all the Virtual File Systems. 43 | type VFS interface { 44 | Opener 45 | // Lstat returns the os.FileInfo for the given path, without 46 | // following symlinks. 47 | Lstat(path string) (os.FileInfo, error) 48 | // Stat returns the os.FileInfo for the given path, following 49 | // symlinks. 50 | Stat(path string) (os.FileInfo, error) 51 | // ReadDir returns the contents of the directory at path as an slice 52 | // of os.FileInfo, ordered alphabetically by name. If path is not a 53 | // directory or the permissions don't allow it, an error will be 54 | // returned. 55 | ReadDir(path string) ([]os.FileInfo, error) 56 | // Mkdir creates a directory at the given path. If the directory 57 | // already exists or its parent directory does not exist or 58 | // the permissions don't allow it, an error will be returned. See 59 | // also the shorthand function MkdirAll. 60 | Mkdir(path string, perm os.FileMode) error 61 | // Remove removes the item at the given path. If the path does 62 | // not exists or the permissions don't allow removing it or it's 63 | // a non-empty directory, an error will be returned. See also 64 | // the shorthand function RemoveAll. 65 | Remove(path string) error 66 | // String returns a human-readable description of the VFS. 67 | String() string 68 | } 69 | 70 | // TemporaryVFS represents a temporary on-disk file system which can be removed 71 | // by calling its Close method. 72 | type TemporaryVFS interface { 73 | VFS 74 | // Root returns the root directory for the temporary VFS. 75 | Root() string 76 | // Close removes all the files in temporary VFS. 77 | Close() error 78 | } 79 | 80 | // Container is implemented by some file systems which 81 | // contain another one. 82 | type Container interface { 83 | // VFS returns the underlying VFS. 84 | VFS() VFS 85 | } 86 | -------------------------------------------------------------------------------- /state/global.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "net/url" 5 | "sync" 6 | 7 | define "github.com/soulteary/apt-proxy/define" 8 | mirrors "github.com/soulteary/apt-proxy/internal/mirrors" 9 | ) 10 | 11 | var ( 12 | proxyMode int 13 | modeMutex sync.RWMutex 14 | ) 15 | 16 | func SetProxyMode(mode int) { 17 | modeMutex.Lock() 18 | defer modeMutex.Unlock() 19 | proxyMode = mode 20 | } 21 | 22 | func GetProxyMode() int { 23 | modeMutex.RLock() 24 | defer modeMutex.RUnlock() 25 | return proxyMode 26 | } 27 | 28 | // MirrorState manages mirror URL states 29 | type MirrorState struct { 30 | url *url.URL 31 | distType int 32 | mutex sync.RWMutex 33 | } 34 | 35 | // NewMirrorState creates a new MirrorState instance 36 | func NewMirrorState(distType int) *MirrorState { 37 | return &MirrorState{ 38 | distType: distType, 39 | } 40 | } 41 | 42 | // Set updates the mirror URL 43 | func (m *MirrorState) Set(input string) { 44 | m.mutex.Lock() 45 | defer m.mutex.Unlock() 46 | 47 | if input == "" { 48 | m.url = nil 49 | return 50 | } 51 | 52 | mirror := input 53 | if alias := mirrors.GetMirrorURLByAliases(m.distType, input); alias != "" { 54 | mirror = alias 55 | } 56 | 57 | url, err := url.Parse(mirror) 58 | if err != nil { 59 | m.url = nil 60 | return 61 | } 62 | m.url = url 63 | } 64 | 65 | // Get returns the current mirror URL 66 | func (m *MirrorState) Get() *url.URL { 67 | m.mutex.RLock() 68 | defer m.mutex.RUnlock() 69 | return m.url 70 | } 71 | 72 | // Reset clears the mirror URL 73 | func (m *MirrorState) Reset() { 74 | m.mutex.Lock() 75 | defer m.mutex.Unlock() 76 | m.url = nil 77 | } 78 | 79 | var ( 80 | // Mirror states for different distributions 81 | UbuntuMirror = NewMirrorState(define.TYPE_LINUX_DISTROS_UBUNTU) 82 | UbuntuPortsMirror = NewMirrorState(define.TYPE_LINUX_DISTROS_UBUNTU_PORTS) 83 | DebianMirror = NewMirrorState(define.TYPE_LINUX_DISTROS_DEBIAN) 84 | CentOSMirror = NewMirrorState(define.TYPE_LINUX_DISTROS_CENTOS) 85 | AlpineMirror = NewMirrorState(define.TYPE_LINUX_DISTROS_ALPINE) 86 | ) 87 | 88 | // Convenience functions for backward compatibility 89 | func SetUbuntuMirror(input string) { UbuntuMirror.Set(input) } 90 | func GetUbuntuMirror() *url.URL { return UbuntuMirror.Get() } 91 | func ResetUbuntuMirror() { UbuntuMirror.Reset() } 92 | 93 | func SetUbuntuPortsMirror(input string) { UbuntuPortsMirror.Set(input) } 94 | func GetUbuntuPortsMirror() *url.URL { return UbuntuPortsMirror.Get() } 95 | func ResetUbuntuPortsMirror() { UbuntuPortsMirror.Reset() } 96 | 97 | func SetDebianMirror(input string) { DebianMirror.Set(input) } 98 | func GetDebianMirror() *url.URL { return DebianMirror.Get() } 99 | func ResetDebianMirror() { DebianMirror.Reset() } 100 | 101 | func SetCentOSMirror(input string) { CentOSMirror.Set(input) } 102 | func GetCentOSMirror() *url.URL { return CentOSMirror.Get() } 103 | func ResetCentOSMirror() { CentOSMirror.Reset() } 104 | 105 | func SetAlpineMirror(input string) { AlpineMirror.Set(input) } 106 | func GetAlpineMirror() *url.URL { return AlpineMirror.Get() } 107 | func ResetAlpineMirror() { AlpineMirror.Reset() } 108 | -------------------------------------------------------------------------------- /internal/mirrors/mirrors_test.go: -------------------------------------------------------------------------------- 1 | package mirrors 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | define "github.com/soulteary/apt-proxy/define" 8 | ) 9 | 10 | func TestGetUbuntuMirrorByAliases(t *testing.T) { 11 | alias := GetMirrorURLByAliases(define.TYPE_LINUX_DISTROS_UBUNTU, "cn:tsinghua") 12 | if !strings.Contains(alias, "mirrors.tuna.tsinghua.edu.cn/ubuntu/") { 13 | t.Fatal("Test Get Mirror By Custom Name Failed") 14 | } 15 | 16 | alias = GetMirrorURLByAliases(define.TYPE_LINUX_DISTROS_UBUNTU, "cn:not-found") 17 | if alias != "" { 18 | t.Fatal("Test Get Mirror By Custom Name Failed") 19 | } 20 | } 21 | 22 | func TestGetDebianMirrorByAliases(t *testing.T) { 23 | alias := GetMirrorURLByAliases(define.TYPE_LINUX_DISTROS_DEBIAN, "cn:tsinghua") 24 | if !strings.Contains(alias, "mirrors.tuna.tsinghua.edu.cn/debian/") { 25 | t.Fatal("Test Get Mirror By Custom Name Failed") 26 | } 27 | 28 | alias = GetMirrorURLByAliases(define.TYPE_LINUX_DISTROS_DEBIAN, "cn:not-found") 29 | if alias != "" { 30 | t.Fatal("Test Get Mirror By Custom Name Failed") 31 | } 32 | } 33 | 34 | func TestGetCentOSMirrorByAliases(t *testing.T) { 35 | alias := GetMirrorURLByAliases(define.TYPE_LINUX_DISTROS_CENTOS, "cn:tsinghua") 36 | if !strings.Contains(alias, "mirrors.tuna.tsinghua.edu.cn/centos/") { 37 | t.Fatal("Test Get Mirror By Custom Name Failed") 38 | } 39 | 40 | alias = GetMirrorURLByAliases(define.TYPE_LINUX_DISTROS_CENTOS, "cn:not-found") 41 | if alias != "" { 42 | t.Fatal("Test Get Mirror By Custom Name Failed") 43 | } 44 | } 45 | 46 | func TestGetMirrorUrlsByGeo(t *testing.T) { 47 | mirrors := GetGeoMirrorUrlsByMode(define.TYPE_LINUX_ALL_DISTROS) 48 | if len(mirrors) == 0 { 49 | t.Fatal("No mirrors found") 50 | } 51 | 52 | mirrors = GetGeoMirrorUrlsByMode(define.TYPE_LINUX_DISTROS_DEBIAN) 53 | if len(mirrors) != len(BUILDIN_DEBIAN_MIRRORS) { 54 | t.Fatal("Get mirrors error") 55 | } 56 | 57 | mirrors = GetGeoMirrorUrlsByMode(define.TYPE_LINUX_DISTROS_UBUNTU) 58 | if len(mirrors) == 0 { 59 | t.Fatal("No mirrors found") 60 | } 61 | } 62 | 63 | func TestGetPredefinedConfiguration(t *testing.T) { 64 | res, pattern := GetPredefinedConfiguration(define.TYPE_LINUX_DISTROS_UBUNTU) 65 | if res != define.UBUNTU_BENCHMAKR_URL { 66 | t.Fatal("Failed to get resource link") 67 | } 68 | if !pattern.MatchString("/ubuntu/InRelease") { 69 | t.Fatal("Failed to verify domain name rules") 70 | } 71 | if !pattern.MatchString("/ubuntu/InRelease") { 72 | t.Fatal("Failed to verify domain name rules") 73 | } 74 | 75 | res, pattern = GetPredefinedConfiguration(define.TYPE_LINUX_DISTROS_DEBIAN) 76 | if res != define.DEBIAN_BENCHMAKR_URL { 77 | t.Fatal("Failed to get resource link") 78 | } 79 | if !pattern.MatchString("/debian/InRelease") { 80 | t.Fatal("Failed to verify domain name rules") 81 | } 82 | 83 | res, pattern = GetPredefinedConfiguration(define.TYPE_LINUX_DISTROS_CENTOS) 84 | if res != define.CENTOS_BENCHMAKR_URL { 85 | t.Fatal("Failed to get resource link") 86 | } 87 | if !pattern.MatchString("/centos/test/repomd.xml") { 88 | t.Fatal("Failed to verify domain name rules") 89 | } 90 | 91 | res, pattern = GetPredefinedConfiguration(define.TYPE_LINUX_DISTROS_ALPINE) 92 | if res != define.ALPINE_BENCHMAKR_URL { 93 | t.Fatal("Failed to get resource link") 94 | } 95 | if !pattern.MatchString("/alpine/test/APKINDEX.tar.gz") { 96 | t.Fatal("Failed to verify domain name rules") 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /pkg/httplog/log.go: -------------------------------------------------------------------------------- 1 | package httplog 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "log" 8 | "net/http" 9 | "net/http/httputil" 10 | "os" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | const ( 16 | CacheHeader = "X-Cache" 17 | ) 18 | 19 | type responseWriter struct { 20 | http.ResponseWriter 21 | status int 22 | size int 23 | t time.Time 24 | errorOutput bytes.Buffer 25 | } 26 | 27 | func (l *responseWriter) Header() http.Header { 28 | return l.ResponseWriter.Header() 29 | } 30 | 31 | func (l *responseWriter) Write(b []byte) (int, error) { 32 | if l.status == 0 { 33 | l.status = http.StatusOK 34 | } 35 | if isError(l.status) { 36 | l.errorOutput.Write(b) 37 | } 38 | size, err := l.ResponseWriter.Write(b) 39 | l.size += size 40 | return size, err 41 | } 42 | 43 | func (l *responseWriter) WriteHeader(s int) { 44 | l.ResponseWriter.WriteHeader(s) 45 | l.status = s 46 | } 47 | 48 | func (l *responseWriter) Status() int { 49 | return l.status 50 | } 51 | 52 | func (l *responseWriter) Size() int { 53 | return l.size 54 | } 55 | 56 | func NewResponseLogger(delegate http.Handler) *ResponseLogger { 57 | return &ResponseLogger{Handler: delegate} 58 | } 59 | 60 | type ResponseLogger struct { 61 | http.Handler 62 | DumpRequests, DumpErrors, DumpResponses bool 63 | } 64 | 65 | func (l *ResponseLogger) ServeHTTP(w http.ResponseWriter, req *http.Request) { 66 | if l.DumpRequests { 67 | b, _ := httputil.DumpRequest(req, false) 68 | writePrefixString(strings.TrimSpace(string(b)), ">> ", os.Stderr) 69 | } 70 | 71 | respWr := &responseWriter{ResponseWriter: w, t: time.Now()} 72 | l.Handler.ServeHTTP(respWr, req) 73 | 74 | if l.DumpResponses { 75 | buf := &bytes.Buffer{} 76 | buf.WriteString(fmt.Sprintf("HTTP/1.1 %d %s\r\n", 77 | respWr.status, http.StatusText(respWr.status), 78 | )) 79 | respWr.Header().Write(buf) 80 | writePrefixString(strings.TrimSpace(buf.String()), "<< ", os.Stderr) 81 | } 82 | 83 | if l.DumpErrors && isError(respWr.status) { 84 | writePrefixString(respWr.errorOutput.String(), "<< ", os.Stderr) 85 | } 86 | 87 | l.writeLog(req, respWr) 88 | } 89 | 90 | func (l *ResponseLogger) writeLog(req *http.Request, respWr *responseWriter) { 91 | cacheStatus := respWr.Header().Get(CacheHeader) 92 | 93 | if strings.HasPrefix(cacheStatus, "HIT") { 94 | cacheStatus = "\x1b[32;1mHIT\x1b[0m" 95 | } else if strings.HasPrefix(cacheStatus, "MISS") { 96 | cacheStatus = "\x1b[31;1mMISS\x1b[0m" 97 | } else { 98 | cacheStatus = "\x1b[33;1mSKIP\x1b[0m" 99 | } 100 | 101 | clientIP := req.RemoteAddr 102 | if colon := strings.LastIndex(clientIP, ":"); colon != -1 { 103 | clientIP = clientIP[:colon] 104 | } 105 | 106 | log.Printf( 107 | "%s \"%s %s %s\" (%s) %d %s %s", 108 | clientIP, 109 | req.Method, 110 | req.URL.String(), 111 | req.Proto, 112 | http.StatusText(respWr.status), 113 | respWr.size, 114 | cacheStatus, 115 | time.Since(respWr.t).String(), 116 | ) 117 | } 118 | 119 | func isError(code int) bool { 120 | return code >= 500 121 | } 122 | 123 | func writePrefixString(s, prefix string, w io.Writer) { 124 | os.Stderr.Write([]byte("\n")) 125 | for _, line := range strings.Split(s, "\r\n") { 126 | w.Write([]byte(prefix)) 127 | w.Write([]byte(line)) 128 | w.Write([]byte("\n")) 129 | } 130 | os.Stderr.Write([]byte("\n")) 131 | } 132 | -------------------------------------------------------------------------------- /define/ubuntu-ports.go: -------------------------------------------------------------------------------- 1 | package define 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | const ( 8 | UBUNTU_PORTS_GEO_MIRROR_API = "http://mirrors.ubuntu.com/mirrors.txt" 9 | UBUNTU_PORTS_BENCHMAKR_URL = "dists/noble/InRelease/Release" 10 | ) 11 | 12 | var UBUNTU_PORTS_HOST_PATTERN = regexp.MustCompile(`/ubuntu-ports/(.+)$`) 13 | 14 | // http://mirrors.ubuntu.com/mirrors.txt 2022.11.19 15 | // Sites that contain protocol headers, restrict access to resources using that protocol 16 | var UBUNTU_PORTS_OFFICIAL_MIRRORS = []string{ 17 | "mirrors.cn99.com/ubuntu-ports/", 18 | "mirrors.tuna.tsinghua.edu.cn/ubuntu-ports/", 19 | "mirrors.cnnic.cn/ubuntu-ports/", 20 | "mirror.bjtu.edu.cn/ubuntu-ports/", 21 | "mirrors.cqu.edu.cn/ubuntu-ports/", 22 | "http://mirrors.skyshe.cn/ubuntu-ports/", 23 | // duplicate "mirrors.tuna.tsinghua.edu.cn/ubuntu-ports/", 24 | "mirrors.yun-idc.com/ubuntu-ports/", 25 | "http://mirror.dlut.edu.cn/ubuntu-ports/", 26 | "mirrors.xjtu.edu.cn/ubuntu-ports/", 27 | "mirrors.huaweicloud.com/repository/ubuntu-ports/", 28 | "mirrors.bupt.edu.cn/ubuntu-ports/", 29 | "mirrors.hit.edu.cn/ubuntu-ports/", 30 | // duplicate "repo.huaweicloud.com/ubuntu-ports/", 31 | "http://mirrors.sohu.com/ubuntu-ports/", 32 | "mirror.nju.edu.cn/ubuntu-ports/", 33 | "mirrors.bfsu.edu.cn/ubuntu-ports/", 34 | "mirror.lzu.edu.cn/ubuntu-ports/", 35 | "mirrors.aliyun.com/ubuntu-ports/", 36 | "ftp.sjtu.edu.cn/ubuntu-ports/", 37 | "mirrors.njupt.edu.cn/ubuntu-ports/", 38 | "mirrors.cloud.tencent.com/ubuntu-ports/", 39 | "http://mirrors.dgut.edu.cn/ubuntu-ports/", 40 | "mirrors.ustc.edu.cn/ubuntu-ports/", 41 | "mirrors.sdu.edu.cn/ubuntu-ports/", 42 | "http://cn.archive.ubuntu.com/ubuntu-ports/", 43 | } 44 | 45 | var UBUNTU_PORTS_CUSTOM_MIRRORS = []string{ 46 | "mirrors.163.com/ubuntu-ports/", 47 | } 48 | 49 | var BUILDIN_UBUNTU_PORTS_MIRRORS = GenerateBuildInList(UBUNTU_PORTS_OFFICIAL_MIRRORS, UBUNTU_PORTS_CUSTOM_MIRRORS) 50 | 51 | var UBUNTU_PORTS_DEFAULT_CACHE_RULES = []Rule{ 52 | {Pattern: regexp.MustCompile(`deb$`), CacheControl: `max-age=100000`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU_PORTS}, 53 | {Pattern: regexp.MustCompile(`udeb$`), CacheControl: `max-age=100000`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU_PORTS}, 54 | {Pattern: regexp.MustCompile(`InRelease$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU_PORTS}, 55 | {Pattern: regexp.MustCompile(`DiffIndex$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU_PORTS}, 56 | {Pattern: regexp.MustCompile(`PackagesIndex$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU_PORTS}, 57 | {Pattern: regexp.MustCompile(`Packages\.(bz2|gz|lzma)$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU_PORTS}, 58 | {Pattern: regexp.MustCompile(`SourcesIndex$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU_PORTS}, 59 | {Pattern: regexp.MustCompile(`Sources\.(bz2|gz|lzma)$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU_PORTS}, 60 | {Pattern: regexp.MustCompile(`Release(\.gpg)?$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU_PORTS}, 61 | {Pattern: regexp.MustCompile(`Translation-(en|fr)\.(gz|bz2|bzip2|lzma)$`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU_PORTS}, 62 | // Add file file hash 63 | {Pattern: regexp.MustCompile(`\/by-hash\/`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU_PORTS}, 64 | } 65 | -------------------------------------------------------------------------------- /internal/mirrors/templates.go: -------------------------------------------------------------------------------- 1 | package mirrors 2 | 3 | import ( 4 | "bytes" 5 | "text/template" 6 | ) 7 | 8 | // URLTemplateData holds data for URL template execution 9 | type URLTemplateData struct { 10 | Scheme string 11 | Host string 12 | Path string 13 | Query string 14 | URL string 15 | } 16 | 17 | // ListenAddressTemplateData holds data for listen address template execution 18 | type ListenAddressTemplateData struct { 19 | Host string 20 | Port string 21 | } 22 | 23 | var ( 24 | // httpURLTemplate is a template for constructing HTTP URLs 25 | httpURLTemplate = template.Must(template.New("httpURL").Parse("http://{{.URL}}")) 26 | 27 | // httpsURLTemplate is a template for constructing HTTPS URLs 28 | httpsURLTemplate = template.Must(template.New("httpsURL").Parse("https://{{.URL}}")) 29 | 30 | // fullURLTemplate is a template for constructing full URLs with scheme, host, and path 31 | fullURLTemplate = template.Must(template.New("fullURL").Parse("{{.Scheme}}://{{.Host}}{{.Path}}{{.Query}}")) 32 | 33 | // listenAddressTemplate is a template for constructing listen addresses 34 | listenAddressTemplate = template.Must(template.New("listenAddress").Parse("{{.Host}}:{{.Port}}")) 35 | ) 36 | 37 | // BuildHTTPURL constructs an HTTP URL using templates 38 | func BuildHTTPURL(url string) (string, error) { 39 | var buf bytes.Buffer 40 | data := URLTemplateData{URL: url} 41 | if err := httpURLTemplate.Execute(&buf, data); err != nil { 42 | return "", err 43 | } 44 | return buf.String(), nil 45 | } 46 | 47 | // BuildHTTPSURL constructs an HTTPS URL using templates 48 | func BuildHTTPSURL(url string) (string, error) { 49 | var buf bytes.Buffer 50 | data := URLTemplateData{URL: url} 51 | if err := httpsURLTemplate.Execute(&buf, data); err != nil { 52 | return "", err 53 | } 54 | return buf.String(), nil 55 | } 56 | 57 | // BuildFullURL constructs a full URL with scheme, host, path, and optional query using templates 58 | func BuildFullURL(scheme, host, path, query string) (string, error) { 59 | var buf bytes.Buffer 60 | data := URLTemplateData{ 61 | Scheme: scheme, 62 | Host: host, 63 | Path: path, 64 | Query: query, 65 | } 66 | if err := fullURLTemplate.Execute(&buf, data); err != nil { 67 | return "", err 68 | } 69 | return buf.String(), nil 70 | } 71 | 72 | // BuildListenAddress constructs a listen address using templates 73 | func BuildListenAddress(host, port string) (string, error) { 74 | var buf bytes.Buffer 75 | data := ListenAddressTemplateData{ 76 | Host: host, 77 | Port: port, 78 | } 79 | if err := listenAddressTemplate.Execute(&buf, data); err != nil { 80 | return "", err 81 | } 82 | return buf.String(), nil 83 | } 84 | 85 | // PathTemplateData holds data for path template execution 86 | type PathTemplateData struct { 87 | Path string 88 | Query string 89 | } 90 | 91 | var ( 92 | // pathQueryTemplate is a template for constructing paths with query strings 93 | pathQueryTemplate = template.Must(template.New("pathQuery").Parse("{{.Path}}{{.Query}}")) 94 | ) 95 | 96 | // BuildPathWithQuery constructs a path with query using templates 97 | func BuildPathWithQuery(path, query string) (string, error) { 98 | var buf bytes.Buffer 99 | data := PathTemplateData{ 100 | Path: path, 101 | Query: query, 102 | } 103 | if err := pathQueryTemplate.Execute(&buf, data); err != nil { 104 | return "", err 105 | } 106 | return buf.String(), nil 107 | } 108 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | project_name: apt-proxy 2 | 3 | builds: 4 | - <<: &build_defaults 5 | env: 6 | - CGO_ENABLED=0 7 | ldflags: 8 | - -w -s -X "github.com/soulteary/apt-proxy/cli/cli.Version={{ .Tag }}" 9 | id: macos 10 | goos: [ darwin ] 11 | goarch: [ amd64, arm64 ] 12 | 13 | - <<: *build_defaults 14 | id: linux 15 | goos: [linux] 16 | goarch: ["386", arm, amd64, arm64] 17 | goarm: 18 | - "7" 19 | - "6" 20 | 21 | dockers: 22 | 23 | - image_templates: 24 | - "soulteary/apt-proxy:linux-amd64-{{ .Tag }}" 25 | - "soulteary/apt-proxy:linux-amd64" 26 | dockerfile: docker/Dockerfile.gorelease 27 | use: buildx 28 | goarch: amd64 29 | build_flag_templates: 30 | - "--pull" 31 | - "--platform=linux/amd64" 32 | - "--label=org.opencontainers.image.title={{ .ProjectName }}" 33 | - "--label=org.opencontainers.image.description={{ .ProjectName }}" 34 | - "--label=org.opencontainers.image.url=https://github.com/soulteary/apt-proxy" 35 | - "--label=org.opencontainers.image.source=https://github.com/soulteary/apt-proxy" 36 | - "--label=org.opencontainers.image.version={{ .Version }}" 37 | - "--label=org.opencontainers.image.created={{ .Date }}" 38 | - "--label=org.opencontainers.image.revision={{ .FullCommit }}" 39 | - "--label=org.opencontainers.image.licenses=Apache-v2" 40 | 41 | - image_templates: 42 | - "soulteary/apt-proxy:linux-arm64-{{ .Tag }}" 43 | - "soulteary/apt-proxy:linux-arm64" 44 | dockerfile: docker/Dockerfile.gorelease 45 | use: buildx 46 | goos: linux 47 | goarch: arm64 48 | goarm: '' 49 | build_flag_templates: 50 | - "--pull" 51 | - "--platform=linux/arm64" 52 | - "--label=org.opencontainers.image.title={{ .ProjectName }}" 53 | - "--label=org.opencontainers.image.description={{ .ProjectName }}" 54 | - "--label=org.opencontainers.image.url=https://github.com/soulteary/apt-proxy" 55 | - "--label=org.opencontainers.image.source=https://github.com/soulteary/apt-proxy" 56 | - "--label=org.opencontainers.image.version={{ .Version }}" 57 | - "--label=org.opencontainers.image.created={{ .Date }}" 58 | - "--label=org.opencontainers.image.revision={{ .FullCommit }}" 59 | - "--label=org.opencontainers.image.licenses=Apache-v2" 60 | 61 | - image_templates: 62 | - "soulteary/apt-proxy:linux-armv7-{{ .Tag }}" 63 | - "soulteary/apt-proxy:linux-armv7" 64 | dockerfile: docker/Dockerfile.gorelease 65 | use: buildx 66 | goos: linux 67 | goarch: arm 68 | goarm: "7" 69 | build_flag_templates: 70 | - "--pull" 71 | - "--platform=linux/arm/v7" 72 | - "--label=org.opencontainers.image.title={{ .ProjectName }}" 73 | - "--label=org.opencontainers.image.description={{ .ProjectName }}" 74 | - "--label=org.opencontainers.image.url=https://github.com/soulteary/apt-proxy" 75 | - "--label=org.opencontainers.image.source=https://github.com/soulteary/apt-proxy" 76 | - "--label=org.opencontainers.image.version={{ .Version }}" 77 | - "--label=org.opencontainers.image.created={{ .Date }}" 78 | - "--label=org.opencontainers.image.revision={{ .FullCommit }}" 79 | - "--label=org.opencontainers.image.licenses=Apache-v2" 80 | 81 | docker_manifests: 82 | - name_template: "soulteary/apt-proxy:{{ .Tag }}" 83 | image_templates: 84 | - "soulteary/apt-proxy:linux-amd64-{{ .Tag }}" 85 | - "soulteary/apt-proxy:linux-arm64-{{ .Tag }}" 86 | - "soulteary/apt-proxy:linux-armv7-{{ .Tag }}" 87 | skip_push: "false" 88 | 89 | - name_template: "soulteary/apt-proxy:latest" 90 | image_templates: 91 | - "soulteary/apt-proxy:linux-amd64-{{ .Tag }}" 92 | - "soulteary/apt-proxy:linux-arm64-{{ .Tag }}" 93 | - "soulteary/apt-proxy:linux-armv7-{{ .Tag }}" 94 | skip_push: "false" 95 | -------------------------------------------------------------------------------- /define/mirror_test.go: -------------------------------------------------------------------------------- 1 | package define_test 2 | 3 | // func TestPrintUbuntuPingScript(t *testing.T) { 4 | // for _, url := range Define.UBUNTU_OFFICIAL_MIRRORS { 5 | // if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) { 6 | // fmt.Println(`echo "` + url + `"`) 7 | // http := "curl --connect-timeout 2 -I http://" + url + Define.UBUNTU_BENCHMAKR_URL 8 | // fmt.Println(http) 9 | // https := "curl --connect-timeout 2 -I https://" + url + Define.UBUNTU_BENCHMAKR_URL 10 | // fmt.Println(https) 11 | // } 12 | // } 13 | // for _, url := range Define.UBUNTU_CUSTOM_MIRRORS { 14 | // if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) { 15 | // fmt.Println(`echo "` + url + `"`) 16 | // http := "curl --connect-timeout 2 -I http://" + url + Define.UBUNTU_BENCHMAKR_URL 17 | // fmt.Println(http) 18 | // https := "curl --connect-timeout 2 -I https://" + url + Define.UBUNTU_BENCHMAKR_URL 19 | // fmt.Println(https) 20 | // } 21 | // } 22 | // } 23 | 24 | // func TestPrintDebianPingScript(t *testing.T) { 25 | // for _, url := range Define.DEBIAN_OFFICIAL_MIRRORS { 26 | // if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) { 27 | // fmt.Println(`echo "` + url + `"`) 28 | // http := "curl --connect-timeout 2 -I http://" + url + Define.DEBIAN_BENCHMAKR_URL 29 | // fmt.Println(http) 30 | // https := "curl --connect-timeout 2 -I https://" + url + Define.DEBIAN_BENCHMAKR_URL 31 | // fmt.Println(https) 32 | // } 33 | // } 34 | // for _, url := range Define.DEBIAN_CUSTOM_MIRRORS { 35 | // if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) { 36 | // fmt.Println(`echo "` + url + `"`) 37 | // http := "curl --connect-timeout 2 -I http://" + url + Define.DEBIAN_BENCHMAKR_URL 38 | // fmt.Println(http) 39 | // https := "curl --connect-timeout 2 -I https://" + url + Define.DEBIAN_BENCHMAKR_URL 40 | // fmt.Println(https) 41 | // } 42 | // } 43 | // } 44 | 45 | // func TestPrintCentosPingScript(t *testing.T) { 46 | // for _, url := range Define.CENTOS_OFFICIAL_MIRRORS { 47 | // if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) { 48 | // fmt.Println(`echo "` + url + `"`) 49 | // http := "curl --connect-timeout 2 -I http://" + url + Define.CENTOS_BENCHMAKR_URL 50 | // fmt.Println(http) 51 | // https := "curl --connect-timeout 2 -I https://" + url + Define.CENTOS_BENCHMAKR_URL 52 | // fmt.Println(https) 53 | // } 54 | // } 55 | // for _, url := range Define.CENTOS_CUSTOM_MIRRORS { 56 | // if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) { 57 | // fmt.Println(`echo "` + url + `"`) 58 | // http := "curl --connect-timeout 2 -I http://" + url + Define.CENTOS_BENCHMAKR_URL 59 | // fmt.Println(http) 60 | // https := "curl --connect-timeout 2 -I https://" + url + Define.CENTOS_BENCHMAKR_URL 61 | // fmt.Println(https) 62 | // } 63 | // } 64 | // } 65 | 66 | // func TestPrintCentosPingScript(t *testing.T) { 67 | // for _, url := range Define.ALPINE_OFFICIAL_MIRRORS { 68 | // if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) { 69 | // fmt.Println(`echo "` + url + `"`) 70 | // http := "curl --connect-timeout 2 -I http://" + url + Define.ALPINE_BENCHMAKR_URL 71 | // fmt.Println(http) 72 | // https := "curl --connect-timeout 2 -I https://" + url + Define.ALPINE_BENCHMAKR_URL 73 | // fmt.Println(https) 74 | // } 75 | // } 76 | // for _, url := range Define.CENTOS_CUSTOM_MIRRORS { 77 | // if !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) { 78 | // fmt.Println(`echo "` + url + `"`) 79 | // http := "curl --connect-timeout 2 -I http://" + url + Define.ALPINE_BENCHMAKR_URL 80 | // fmt.Println(http) 81 | // https := "curl --connect-timeout 2 -I https://" + url + Define.ALPINE_BENCHMAKR_URL 82 | // fmt.Println(https) 83 | // } 84 | // } 85 | // } 86 | -------------------------------------------------------------------------------- /pkg/vfs/open.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "archive/tar" 5 | "archive/zip" 6 | "bytes" 7 | "compress/bzip2" 8 | "compress/gzip" 9 | "fmt" 10 | "io" 11 | "log" 12 | "os" 13 | "path/filepath" 14 | "strings" 15 | ) 16 | 17 | // Zip returns an in-memory VFS initialized with the 18 | // contents of the .zip file read from the given io.Reader. 19 | // Since archive/zip requires an io.ReaderAt rather than an 20 | // io.Reader, and a known size, Zip will read the whole file 21 | // into memory and provide its own buffering if r does not 22 | // implement io.ReaderAt or size is <= 0. 23 | func Zip(r io.Reader, size int64) (VFS, error) { 24 | rat, _ := r.(io.ReaderAt) 25 | if rat == nil || size <= 0 { 26 | data, err := io.ReadAll(r) 27 | if err != nil { 28 | return nil, err 29 | } 30 | rat = bytes.NewReader(data) 31 | size = int64(len(data)) 32 | } 33 | zr, err := zip.NewReader(rat, size) 34 | if err != nil { 35 | return nil, err 36 | } 37 | files := make(map[string]*File) 38 | for _, file := range zr.File { 39 | if file.Mode().IsDir() { 40 | continue 41 | } 42 | f, err := file.Open() 43 | if err != nil { 44 | return nil, err 45 | } 46 | data, err := io.ReadAll(f) 47 | errClose := f.Close() 48 | if errClose != nil { 49 | return nil, errClose 50 | } 51 | if err != nil { 52 | return nil, err 53 | } 54 | files[file.Name] = &File{ 55 | Data: data, 56 | Mode: file.Mode(), 57 | ModTime: file.ModTime(), 58 | } 59 | } 60 | return Map(files) 61 | } 62 | 63 | // Tar returns an in-memory VFS initialized with the 64 | // contents of the .tar file read from the given io.Reader. 65 | func Tar(r io.Reader) (VFS, error) { 66 | files := make(map[string]*File) 67 | tr := tar.NewReader(r) 68 | for { 69 | hdr, err := tr.Next() 70 | if err != nil { 71 | if err == io.EOF { 72 | break 73 | } 74 | return nil, err 75 | } 76 | if hdr.FileInfo().IsDir() { 77 | continue 78 | } 79 | data, err := io.ReadAll(tr) 80 | if err != nil { 81 | return nil, err 82 | } 83 | files[hdr.Name] = &File{ 84 | Data: data, 85 | Mode: hdr.FileInfo().Mode(), 86 | ModTime: hdr.ModTime, 87 | } 88 | } 89 | return Map(files) 90 | } 91 | 92 | // TarGzip returns an in-memory VFS initialized with the 93 | // contents of the .tar.gz file read from the given io.Reader. 94 | func TarGzip(r io.Reader) (VFS, error) { 95 | zr, err := gzip.NewReader(r) 96 | if err != nil { 97 | return nil, err 98 | } 99 | defer zr.Close() 100 | return Tar(zr) 101 | } 102 | 103 | // TarBzip2 returns an in-memory VFS initialized with the 104 | // contents of then .tar.bz2 file read from the given io.Reader. 105 | func TarBzip2(r io.Reader) (VFS, error) { 106 | bzr := bzip2.NewReader(r) 107 | return Tar(bzr) 108 | } 109 | 110 | // Open returns an in-memory VFS initialized with the contents 111 | // of the given filename, which must have one of the following 112 | // extensions: 113 | // 114 | // - .zip 115 | // - .tar 116 | // - .tar.gz 117 | // - .tar.bz2 118 | func Open(filename string) (VFS, error) { 119 | file, err := os.Open(filepath.Clean(filename)) 120 | if err != nil { 121 | return nil, err 122 | } 123 | 124 | defer func() { 125 | if err := file.Close(); err != nil { 126 | log.Fatal(err) 127 | } 128 | }() 129 | 130 | base := filepath.Base(filename) 131 | ext := strings.ToLower(filepath.Ext(base)) 132 | nonExt := filename[:len(filename)-len(ext)] 133 | if strings.ToLower(filepath.Ext(nonExt)) == ".tar" { 134 | ext = ".tar" + ext 135 | } 136 | switch ext { 137 | case ".zip": 138 | st, err := file.Stat() 139 | if err != nil { 140 | return nil, err 141 | } 142 | return Zip(file, st.Size()) 143 | case ".tar": 144 | return Tar(file) 145 | case ".tar.gz": 146 | return TarGzip(file) 147 | case ".tar.bz2": 148 | return TarBzip2(file) 149 | } 150 | return nil, fmt.Errorf("can't open a VFS from a %s file", ext) 151 | } 152 | -------------------------------------------------------------------------------- /pkg/vfs/file_util.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "bytes" 5 | "compress/zlib" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "log" 10 | "runtime" 11 | "time" 12 | ) 13 | 14 | var ( 15 | errFileClosed = errors.New("file is closed") 16 | ) 17 | 18 | // NewRFile returns a RFile from a *File. 19 | func NewRFile(f *File) (RFile, error) { 20 | data, err := fileData(f) 21 | if err != nil { 22 | return nil, err 23 | } 24 | return &file{f: f, data: data, readable: true}, nil 25 | } 26 | 27 | // NewWFile returns a WFile from a *File. 28 | func NewWFile(f *File, read bool, write bool) (WFile, error) { 29 | data, err := fileData(f) 30 | if err != nil { 31 | return nil, err 32 | } 33 | w := &file{f: f, data: data, readable: read, writable: write} 34 | runtime.SetFinalizer(w, closeFile) 35 | return w, nil 36 | } 37 | 38 | func closeFile(f *file) { 39 | err := f.Close() 40 | if err != nil { 41 | log.Println(err) 42 | } 43 | } 44 | 45 | func fileData(f *File) ([]byte, error) { 46 | if len(f.Data) == 0 || f.Mode&ModeCompress == 0 { 47 | return f.Data, nil 48 | } 49 | zr, err := zlib.NewReader(bytes.NewReader(f.Data)) 50 | if err != nil { 51 | return nil, err 52 | } 53 | defer zr.Close() 54 | var out bytes.Buffer 55 | for { 56 | _, err := io.CopyN(&out, zr, 1024) 57 | if err != nil { 58 | if err == io.EOF { 59 | break 60 | } 61 | return nil, err 62 | } 63 | } 64 | return out.Bytes(), nil 65 | } 66 | 67 | type file struct { 68 | f *File 69 | data []byte 70 | offset int 71 | readable bool 72 | writable bool 73 | closed bool 74 | } 75 | 76 | func (f *file) Read(p []byte) (int, error) { 77 | if !f.readable { 78 | return 0, ErrWriteOnly 79 | } 80 | f.f.RLock() 81 | defer f.f.RUnlock() 82 | if f.closed { 83 | return 0, errFileClosed 84 | } 85 | if f.offset > len(f.data) { 86 | return 0, io.EOF 87 | } 88 | n := copy(p, f.data[f.offset:]) 89 | f.offset += n 90 | if n < len(p) { 91 | return n, io.EOF 92 | } 93 | return n, nil 94 | } 95 | 96 | func (f *file) Seek(offset int64, whence int) (int64, error) { 97 | f.f.Lock() 98 | defer f.f.Unlock() 99 | if f.closed { 100 | return 0, errFileClosed 101 | } 102 | switch whence { 103 | case io.SeekStart: 104 | f.offset = int(offset) 105 | case io.SeekCurrent: 106 | f.offset += int(offset) 107 | case io.SeekEnd: 108 | f.offset = len(f.data) + int(offset) 109 | default: 110 | panic(fmt.Errorf("Seek: invalid whence %d", whence)) 111 | } 112 | if f.offset > len(f.data) { 113 | f.offset = len(f.data) 114 | } else if f.offset < 0 { 115 | f.offset = 0 116 | } 117 | return int64(f.offset), nil 118 | } 119 | 120 | func (f *file) Write(p []byte) (int, error) { 121 | if !f.writable { 122 | return 0, ErrReadOnly 123 | } 124 | f.f.Lock() 125 | defer f.f.Unlock() 126 | if f.closed { 127 | return 0, errFileClosed 128 | } 129 | count := len(p) 130 | n := copy(f.data[f.offset:], p) 131 | if n < count { 132 | f.data = append(f.data, p[n:]...) 133 | } 134 | f.offset += count 135 | f.f.ModTime = time.Now() 136 | return count, nil 137 | } 138 | 139 | func (f *file) Close() error { 140 | if !f.closed { 141 | f.f.Lock() 142 | defer f.f.Unlock() 143 | if !f.closed { 144 | if f.f.Mode&ModeCompress != 0 { 145 | var buf bytes.Buffer 146 | zw := zlib.NewWriter(&buf) 147 | if _, err := zw.Write(f.data); err != nil { 148 | return err 149 | } 150 | if err := zw.Close(); err != nil { 151 | return err 152 | } 153 | if buf.Len() < len(f.data) { 154 | f.f.Data = buf.Bytes() 155 | } else { 156 | f.f.Mode &= ^ModeCompress 157 | f.f.Data = f.data 158 | } 159 | } else { 160 | f.f.Data = f.data 161 | } 162 | f.closed = true 163 | } 164 | } 165 | return nil 166 | } 167 | 168 | func (f *file) IsCompressed() bool { 169 | return f.f.Mode&ModeCompress != 0 170 | } 171 | 172 | func (f *file) SetCompressed(c bool) { 173 | f.f.Lock() 174 | defer f.f.Unlock() 175 | if c { 176 | f.f.Mode |= ModeCompress 177 | } else { 178 | f.f.Mode &= ^ModeCompress 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /internal/benchmarks/benchmark.go: -------------------------------------------------------------------------------- 1 | package benchmarks 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io" 7 | "log" 8 | "net/http" 9 | "sort" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | const ( 15 | // Configuration constants 16 | BenchmarkMaxTimeout = 150 * time.Second // detect resource timeout 17 | BenchmarkMaxTries = 3 // maximum number of attempts 18 | BenchmarkDetectTimeout = 30 * time.Second // for select fast mirror 19 | ) 20 | 21 | // Result stores benchmark results for a URL 22 | type Result struct { 23 | URL string 24 | Duration time.Duration 25 | } 26 | 27 | // Results implements sort.Interface for []Result based on Duration 28 | type Results []Result 29 | 30 | func (r Results) Len() int { return len(r) } 31 | func (r Results) Less(i, j int) bool { return r[i].Duration < r[j].Duration } 32 | func (r Results) Swap(i, j int) { r[i], r[j] = r[j], r[i] } 33 | 34 | // Benchmark performs HTTP GET requests and measures response time 35 | func Benchmark(ctx context.Context, base, query string, times int) (time.Duration, error) { 36 | client := &http.Client{ 37 | Timeout: BenchmarkMaxTimeout, 38 | Transport: &http.Transport{ 39 | MaxIdleConns: 100, 40 | IdleConnTimeout: 90 * time.Second, 41 | DisableCompression: true, 42 | }, 43 | } 44 | 45 | var totalDuration time.Duration 46 | for i := 0; i < times; i++ { 47 | select { 48 | case <-ctx.Done(): 49 | return 0, ctx.Err() 50 | default: 51 | duration, err := singleBenchmark(ctx, client, base+query) 52 | if err != nil { 53 | return 0, err 54 | } 55 | totalDuration += duration 56 | } 57 | } 58 | 59 | return totalDuration / time.Duration(times), nil 60 | } 61 | 62 | func singleBenchmark(ctx context.Context, client *http.Client, url string) (time.Duration, error) { 63 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 64 | if err != nil { 65 | return 0, err 66 | } 67 | 68 | start := time.Now() 69 | resp, err := client.Do(req) 70 | if err != nil { 71 | return 0, err 72 | } 73 | defer resp.Body.Close() 74 | 75 | // Discard body but handle potential errors 76 | _, err = io.Copy(io.Discard, resp.Body) 77 | if err != nil { 78 | return 0, err 79 | } 80 | 81 | if resp.StatusCode != http.StatusOK { 82 | return 0, errors.New("non-200 status code received") 83 | } 84 | 85 | return time.Since(start), nil 86 | } 87 | 88 | // GetTheFastestMirror finds the fastest responding mirror from the provided list 89 | func GetTheFastestMirror(mirrors []string, testURL string) (string, error) { 90 | ctx, cancel := context.WithTimeout(context.Background(), BenchmarkDetectTimeout) 91 | defer cancel() 92 | 93 | maxResults := min(len(mirrors), 3) 94 | results := make(chan Result, len(mirrors)) 95 | var wg sync.WaitGroup 96 | 97 | // Create error channel to collect errors 98 | errs := make(chan error, len(mirrors)) 99 | 100 | log.Printf("Starting benchmark for %d mirrors", len(mirrors)) 101 | 102 | // Launch benchmarks in parallel 103 | for _, url := range mirrors { 104 | wg.Add(1) 105 | go func(u string) { 106 | defer wg.Done() 107 | duration, err := Benchmark(ctx, u, testURL, BenchmarkMaxTries) 108 | if err != nil { 109 | errs <- err 110 | return 111 | } 112 | results <- Result{URL: u, Duration: duration} 113 | }(url) 114 | } 115 | 116 | // Close results channel when all goroutines complete 117 | go func() { 118 | wg.Wait() 119 | close(results) 120 | close(errs) 121 | }() 122 | 123 | // Collect and sort results 124 | var collectedResults Results 125 | for result := range results { 126 | collectedResults = append(collectedResults, result) 127 | if len(collectedResults) >= maxResults { 128 | break 129 | } 130 | } 131 | 132 | if len(collectedResults) == 0 { 133 | // Collect errors if no results 134 | var errMsgs []error 135 | for err := range errs { 136 | errMsgs = append(errMsgs, err) 137 | } 138 | if len(errMsgs) > 0 { 139 | return "", errors.Join(errMsgs...) 140 | } 141 | return "", errors.New("no valid results found") 142 | } 143 | 144 | sort.Sort(collectedResults) 145 | log.Printf("Completed benchmark. Found %d valid results", len(collectedResults)) 146 | 147 | return collectedResults[0].URL, nil 148 | } 149 | 150 | func min(a, b int) int { 151 | if a < b { 152 | return a 153 | } 154 | return b 155 | } 156 | -------------------------------------------------------------------------------- /pkg/stream.v1/stream_test.go: -------------------------------------------------------------------------------- 1 | package stream 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io" 7 | "os" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | var ( 13 | testdata = []byte("hello\nworld\n") 14 | errFail = errors.New("fail") 15 | ) 16 | 17 | type badFs struct { 18 | readers []File 19 | } 20 | type badFile struct{ name string } 21 | 22 | func (r badFile) Name() string { return r.name } 23 | func (r badFile) Read(p []byte) (int, error) { return 0, errFail } 24 | func (r badFile) ReadAt(p []byte, off int64) (int, error) { return 0, errFail } 25 | func (r badFile) Write(p []byte) (int, error) { return 0, errFail } 26 | func (r badFile) Close() error { return errFail } 27 | 28 | func (fs badFs) Create(name string) (File, error) { return os.Create(name) } 29 | func (fs badFs) Open(name string) (File, error) { 30 | if len(fs.readers) > 0 { 31 | f := fs.readers[len(fs.readers)-1] 32 | fs.readers = fs.readers[:len(fs.readers)-1] 33 | return f, nil 34 | } 35 | return nil, errFail 36 | } 37 | func (fs badFs) Remove(name string) error { return os.Remove(name) } 38 | 39 | func TestMemFs(t *testing.T) { 40 | fs := NewMemFS() 41 | if _, err := fs.Open("not found"); err != ErrNotFoundInMem { 42 | t.Error(err) 43 | t.FailNow() 44 | } 45 | } 46 | 47 | func TestBadFile(t *testing.T) { 48 | fs := badFs{readers: make([]File, 0, 1)} 49 | fs.readers = append(fs.readers, badFile{name: "test"}) 50 | f, err := NewStream("test", fs) 51 | if err != nil { 52 | t.Error(err) 53 | t.FailNow() 54 | } 55 | defer f.Remove() 56 | defer f.Close() 57 | 58 | r, err := f.NextReader() 59 | if err != nil { 60 | t.Error(err) 61 | t.FailNow() 62 | } 63 | defer r.Close() 64 | if r.Name() != "test" { 65 | t.Errorf("expected name to to be 'test' got %s", r.Name()) 66 | t.FailNow() 67 | } 68 | if _, err := r.ReadAt(nil, 0); err == nil { 69 | t.Error("expected ReadAt error") 70 | t.FailNow() 71 | } 72 | if _, err := r.Read(nil); err == nil { 73 | t.Error("expected Read error") 74 | t.FailNow() 75 | } 76 | } 77 | 78 | func TestBadFs(t *testing.T) { 79 | f, err := NewStream("test", badFs{}) 80 | if err != nil { 81 | t.Error(err) 82 | t.FailNow() 83 | } 84 | defer f.Remove() 85 | defer f.Close() 86 | 87 | r, err := f.NextReader() 88 | if err == nil { 89 | t.Error("expected open error") 90 | t.FailNow() 91 | } else { 92 | return 93 | } 94 | r.Close() 95 | } 96 | 97 | func TestStd(t *testing.T) { 98 | f, err := New("test.txt") 99 | if err != nil { 100 | t.Error(err) 101 | t.FailNow() 102 | } 103 | if f.Name() != "test.txt" { 104 | t.Errorf("expected name to be test.txt: %s", f.Name()) 105 | } 106 | testFile(f, t) 107 | } 108 | 109 | func TestMem(t *testing.T) { 110 | f, err := NewStream("test.txt", NewMemFS()) 111 | if err != nil { 112 | t.Error(err) 113 | t.FailNow() 114 | } 115 | f.Write(nil) 116 | testFile(f, t) 117 | } 118 | 119 | func TestRemove(t *testing.T) { 120 | f, err := NewStream("test.txt", NewMemFS()) 121 | if err != nil { 122 | t.Error(err) 123 | t.FailNow() 124 | } 125 | defer f.Close() 126 | go f.Remove() 127 | <-time.After(100 * time.Millisecond) 128 | r, err := f.NextReader() 129 | switch err { 130 | case ErrRemoving: 131 | case nil: 132 | t.Error("expected error on NextReader()") 133 | r.Close() 134 | default: 135 | t.Error("expected diff error on NextReader()", err) 136 | } 137 | 138 | } 139 | 140 | func testFile(f *Stream, t *testing.T) { 141 | 142 | for i := 0; i < 10; i++ { 143 | go testReader(f, t) 144 | } 145 | 146 | for i := 0; i < 10; i++ { 147 | f.Write(testdata) 148 | <-time.After(10 * time.Millisecond) 149 | } 150 | 151 | f.Close() 152 | testReader(f, t) 153 | f.Remove() 154 | } 155 | 156 | func testReader(f *Stream, t *testing.T) { 157 | r, err := f.NextReader() 158 | if err != nil { 159 | t.Error(err) 160 | t.FailNow() 161 | } 162 | defer r.Close() 163 | 164 | buf := bytes.NewBuffer(nil) 165 | sr := io.NewSectionReader(r, 1+int64(len(testdata)*5), 5) 166 | io.Copy(buf, sr) 167 | if !bytes.Equal(buf.Bytes(), testdata[1:6]) { 168 | t.Errorf("unequal %s", buf.Bytes()) 169 | return 170 | } 171 | 172 | buf.Reset() 173 | io.Copy(buf, r) 174 | if !bytes.Equal(buf.Bytes(), bytes.Repeat(testdata, 10)) { 175 | t.Errorf("unequal %s", buf.Bytes()) 176 | return 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /pkg/vfs/mounter.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | "strings" 8 | ) 9 | 10 | const ( 11 | separator = "/" 12 | ) 13 | 14 | func hasSubdir(root, dir string) (string, bool) { 15 | root = path.Clean(root) 16 | if !strings.HasSuffix(root, separator) { 17 | root += separator 18 | } 19 | dir = path.Clean(dir) 20 | if !strings.HasPrefix(dir, root) { 21 | return "", false 22 | } 23 | return dir[len(root):], true 24 | } 25 | 26 | type mountPoint struct { 27 | point string 28 | fs VFS 29 | } 30 | 31 | func (m *mountPoint) String() string { 32 | return fmt.Sprintf("%s at %s", m.fs, m.point) 33 | } 34 | 35 | // Mounter implements the VFS interface and allows mounting different virtual 36 | // file systems at arbitraty points, working much like a UNIX filesystem. 37 | // Note that the first mounted filesystem must be always at "/". 38 | type Mounter struct { 39 | points []*mountPoint 40 | } 41 | 42 | func (m *Mounter) fs(p string) (VFS, string, error) { 43 | for ii := len(m.points) - 1; ii >= 0; ii-- { 44 | if rel, ok := hasSubdir(m.points[ii].point, p); ok { 45 | return m.points[ii].fs, rel, nil 46 | } 47 | } 48 | return nil, "", os.ErrNotExist 49 | } 50 | 51 | // Mount mounts the given filesystem at the given mount point. Unless the 52 | // mount point is /, it must be an already existing directory. 53 | func (m *Mounter) Mount(fs VFS, point string) error { 54 | point = path.Clean(point) 55 | if point == "." || point == "" { 56 | point = "/" 57 | } 58 | if point == "/" { 59 | if len(m.points) > 0 { 60 | return fmt.Errorf("%s is already mounted at /", m.points[0]) 61 | } 62 | m.points = append(m.points, &mountPoint{point, fs}) 63 | return nil 64 | } 65 | stat, err := m.Stat(point) 66 | if err != nil { 67 | return err 68 | } 69 | if !stat.IsDir() { 70 | return fmt.Errorf("%s is not a directory", point) 71 | } 72 | m.points = append(m.points, &mountPoint{point, fs}) 73 | return nil 74 | } 75 | 76 | // Umount umounts the filesystem from the given mount point. If there are other filesystems 77 | // mounted below it or there's no filesystem mounted at that point, an error is returned. 78 | func (m *Mounter) Umount(point string) error { 79 | point = path.Clean(point) 80 | for ii, v := range m.points { 81 | if v.point == point { 82 | // Check if we have mount points below this one 83 | for _, vv := range m.points[ii:] { 84 | if _, ok := hasSubdir(v.point, vv.point); ok { 85 | return fmt.Errorf("can't umount %s because %s is mounted below it", point, vv) 86 | } 87 | } 88 | m.points = append(m.points[:ii], m.points[ii+1:]...) 89 | return nil 90 | } 91 | } 92 | return fmt.Errorf("no filesystem mounted at %s", point) 93 | } 94 | 95 | func (m *Mounter) Open(path string) (RFile, error) { 96 | fs, p, err := m.fs(path) 97 | if err != nil { 98 | return nil, err 99 | } 100 | return fs.Open(p) 101 | } 102 | 103 | func (m *Mounter) OpenFile(path string, flag int, perm os.FileMode) (WFile, error) { 104 | fs, p, err := m.fs(path) 105 | if err != nil { 106 | return nil, err 107 | } 108 | return fs.OpenFile(p, flag, perm) 109 | } 110 | 111 | func (m *Mounter) Lstat(path string) (os.FileInfo, error) { 112 | fs, p, err := m.fs(path) 113 | if err != nil { 114 | return nil, err 115 | } 116 | return fs.Lstat(p) 117 | } 118 | 119 | func (m *Mounter) Stat(path string) (os.FileInfo, error) { 120 | fs, p, err := m.fs(path) 121 | if err != nil { 122 | return nil, err 123 | } 124 | return fs.Stat(p) 125 | } 126 | 127 | func (m *Mounter) ReadDir(path string) ([]os.FileInfo, error) { 128 | fs, p, err := m.fs(path) 129 | if err != nil { 130 | return nil, err 131 | } 132 | return fs.ReadDir(p) 133 | } 134 | 135 | func (m *Mounter) Mkdir(path string, perm os.FileMode) error { 136 | fs, p, err := m.fs(path) 137 | if err != nil { 138 | return err 139 | } 140 | return fs.Mkdir(p, perm) 141 | } 142 | 143 | func (m *Mounter) Remove(path string) error { 144 | // TODO: Don't allow removing an empty directory 145 | // with a mount below it. 146 | fs, p, err := m.fs(path) 147 | if err != nil { 148 | return err 149 | } 150 | return fs.Remove(p) 151 | } 152 | 153 | func (m *Mounter) String() string { 154 | s := make([]string, len(m.points)) 155 | for ii, v := range m.points { 156 | s[ii] = v.String() 157 | } 158 | return fmt.Sprintf("Mounter: %s", strings.Join(s, ", ")) 159 | } 160 | 161 | func mounterCompileTimeCheck() VFS { 162 | return &Mounter{} 163 | } 164 | -------------------------------------------------------------------------------- /pkg/vfs/fs.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | ) 10 | 11 | // IMPORTANT: Note about wrapping os. functions: os.Open, os.OpenFile etc... will return a non-nil 12 | // interface pointing to a nil instance in case of error (whoever decided this disctintion in Go 13 | // was a good idea deservers to be hung by his thumbs). This is highly undesirable, since users 14 | // can't rely on checking f != nil to know if a correct handle was returned. That's why the 15 | // methods in fileSystem do the error checking themselves and return a true nil in case of error. 16 | 17 | type fileSystem struct { 18 | root string 19 | temporary bool 20 | } 21 | 22 | func (fs *fileSystem) path(name string) string { 23 | name = filepath.Clean("/" + name) 24 | return filepath.Join(fs.root, filepath.FromSlash(name)) 25 | } 26 | 27 | // Root returns the root directory of the fileSystem, as an 28 | // absolute path native to the current operating system. 29 | func (fs *fileSystem) Root() string { 30 | return fs.root 31 | } 32 | 33 | // IsTemporary returns wheter the fileSystem is temporary. 34 | func (fs *fileSystem) IsTemporary() bool { 35 | return fs.temporary 36 | } 37 | 38 | func (fs *fileSystem) Open(path string) (RFile, error) { 39 | f, err := os.Open(filepath.Join(fs.root, filepath.Clean(path))) 40 | if err != nil { 41 | return nil, err 42 | } 43 | return f, nil 44 | } 45 | 46 | var ErrInvalidPath = errors.New("invalid path: attempt to access parent directory") 47 | 48 | func containsDotDot(path string) bool { 49 | return strings.Contains(path, "..") 50 | } 51 | 52 | func isUnderRoot(root, path string) bool { 53 | rel, err := filepath.Rel(root, path) 54 | if err != nil { 55 | return false 56 | } 57 | return !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && rel != ".." 58 | } 59 | 60 | func (fs *fileSystem) OpenFile(path string, flag int, mode os.FileMode) (WFile, error) { 61 | cleanPath := filepath.Clean(path) 62 | 63 | if containsDotDot(cleanPath) { 64 | return nil, ErrInvalidPath 65 | } 66 | 67 | fullPath := filepath.Join(fs.root, cleanPath) 68 | 69 | // Resolve to absolute path to prevent path traversal 70 | absPath, err := filepath.Abs(fullPath) 71 | if err != nil { 72 | return nil, ErrInvalidPath 73 | } 74 | 75 | // Ensure the resolved path is within the root directory 76 | if !isUnderRoot(fs.root, absPath) { 77 | return nil, ErrInvalidPath 78 | } 79 | 80 | f, err := os.OpenFile(absPath, flag, mode) 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | return f, nil 86 | } 87 | 88 | func (fs *fileSystem) Lstat(path string) (os.FileInfo, error) { 89 | info, err := os.Lstat(fs.path(path)) 90 | if err != nil { 91 | return nil, err 92 | } 93 | return info, nil 94 | } 95 | 96 | func (fs *fileSystem) Stat(path string) (os.FileInfo, error) { 97 | info, err := os.Stat(fs.path(path)) 98 | if err != nil { 99 | return nil, err 100 | } 101 | return info, nil 102 | } 103 | 104 | func (fs *fileSystem) ReadDir(path string) ([]os.FileInfo, error) { 105 | entries, err := os.ReadDir(fs.path(path)) 106 | if err != nil { 107 | return nil, err 108 | } 109 | 110 | files := make([]os.FileInfo, 0, len(entries)) 111 | for _, entry := range entries { 112 | info, err := entry.Info() 113 | if err != nil { 114 | return nil, err 115 | } 116 | files = append(files, info) 117 | } 118 | 119 | return files, nil 120 | } 121 | 122 | func (fs *fileSystem) Mkdir(path string, perm os.FileMode) error { 123 | return os.Mkdir(fs.path(path), perm) 124 | } 125 | 126 | func (fs *fileSystem) Remove(path string) error { 127 | return os.Remove(fs.path(path)) 128 | } 129 | 130 | func (fs *fileSystem) String() string { 131 | return fmt.Sprintf("fileSystem: %s", fs.root) 132 | } 133 | 134 | // Close is a no-op on non-temporary filesystems. On temporary 135 | // ones (as returned by TmpFS), it removes all the temporary files. 136 | func (f *fileSystem) Close() error { 137 | if f.temporary { 138 | return os.RemoveAll(f.root) 139 | } 140 | return nil 141 | } 142 | 143 | func newFS(root string) (*fileSystem, error) { 144 | abs, err := filepath.Abs(root) 145 | if err != nil { 146 | return nil, err 147 | } 148 | return &fileSystem{root: abs}, nil 149 | } 150 | 151 | // FS returns a VFS at the given path, which must be provided 152 | // as native path of the current operating system. The path might be 153 | // either absolute or relative, but the fileSystem will be anchored 154 | // at the absolute path represented by root at the time of the function 155 | // call. 156 | func FS(root string) (VFS, error) { 157 | return newFS(root) 158 | } 159 | 160 | // TmpFS returns a temporary file system with the given prefix and its root 161 | // directory name, which might be empty. The temporary file system is created 162 | // in the default temporary directory for the operating system. Once you're 163 | // done with the temporary filesystem, you might can all its files by calling 164 | // its Close method. 165 | func TmpFS(prefix string) (TemporaryVFS, error) { 166 | dir, err := os.MkdirTemp("", prefix) 167 | if err != nil { 168 | return nil, err 169 | } 170 | fs, err := newFS(dir) 171 | if err != nil { 172 | return nil, err 173 | } 174 | fs.temporary = true 175 | return fs, nil 176 | } 177 | -------------------------------------------------------------------------------- /pkg/httpcache/util_test.go: -------------------------------------------------------------------------------- 1 | package httpcache_test 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "net/http/httptest" 9 | "strconv" 10 | "strings" 11 | "time" 12 | 13 | "github.com/soulteary/apt-proxy/pkg/httpcache" 14 | ) 15 | 16 | func newRequest(method, url string, h ...string) *http.Request { 17 | req, err := http.NewRequest(method, url, strings.NewReader("")) 18 | if err != nil { 19 | panic(err) 20 | } 21 | req.Header = parseHeaders(h) 22 | req.RemoteAddr = "test.local" 23 | return req 24 | } 25 | 26 | func newResponse(status int, body []byte, h ...string) *http.Response { 27 | return &http.Response{ 28 | Status: fmt.Sprintf("%d %s", status, http.StatusText(status)), 29 | StatusCode: status, 30 | Proto: "HTTP/1.1", 31 | ProtoMajor: 1, 32 | ProtoMinor: 1, 33 | ContentLength: int64(len(body)), 34 | Body: io.NopCloser(bytes.NewReader(body)), 35 | Header: parseHeaders(h), 36 | Close: true, 37 | } 38 | } 39 | 40 | func parseHeaders(input []string) http.Header { 41 | headers := http.Header{} 42 | for _, header := range input { 43 | if idx := strings.Index(header, ": "); idx != -1 { 44 | headers.Add(header[0:idx], strings.TrimSpace(header[idx+1:])) 45 | } 46 | } 47 | return headers 48 | } 49 | 50 | type client struct { 51 | handler http.Handler 52 | cacheHandler *httpcache.Handler 53 | } 54 | 55 | func (c *client) do(r *http.Request) *clientResponse { 56 | rec := httptest.NewRecorder() 57 | c.handler.ServeHTTP(rec, r) 58 | rec.Flush() 59 | 60 | var age int 61 | var err error 62 | 63 | if ageHeader := rec.Header().Get("Age"); ageHeader != "" { 64 | age, err = strconv.Atoi(ageHeader) 65 | if err != nil { 66 | panic("Can't parse age header") 67 | } 68 | } 69 | 70 | // wait for writes to finish 71 | httpcache.Writes.Wait() 72 | 73 | return &clientResponse{ 74 | ResponseRecorder: rec, 75 | cacheStatus: rec.Header().Get(httpcache.CacheHeader), 76 | statusCode: rec.Code, 77 | age: time.Second * time.Duration(age), 78 | body: rec.Body.Bytes(), 79 | header: rec.Header(), 80 | } 81 | } 82 | 83 | func (c *client) get(path string, headers ...string) *clientResponse { 84 | return c.do(newRequest("GET", "http://example.org"+path, headers...)) 85 | } 86 | 87 | func (c *client) head(path string, headers ...string) *clientResponse { 88 | return c.do(newRequest("HEAD", "http://example.org"+path, headers...)) 89 | } 90 | 91 | // func (c *client) put(path string, headers ...string) *clientResponse { 92 | // return c.do(newRequest("PUT", "http://example.org"+path, headers...)) 93 | // } 94 | 95 | // func (c *client) post(path string, headers ...string) *clientResponse { 96 | // return c.do(newRequest("POST", "http://example.org"+path, headers...)) 97 | // } 98 | 99 | type clientResponse struct { 100 | *httptest.ResponseRecorder 101 | cacheStatus string 102 | statusCode int 103 | age time.Duration 104 | body []byte 105 | header http.Header 106 | } 107 | 108 | type upstreamServer struct { 109 | Now time.Time 110 | Body []byte 111 | Filename string 112 | CacheControl string 113 | Etag, Vary string 114 | LastModified time.Time 115 | ResponseDuration time.Duration 116 | StatusCode int 117 | Header http.Header 118 | asserts []func(r *http.Request) 119 | requests int 120 | } 121 | 122 | func (u *upstreamServer) timeTravel(d time.Duration) { 123 | u.Now = u.Now.Add(d) 124 | } 125 | 126 | // func (u *upstreamServer) assert(f func(r *http.Request)) { 127 | // u.asserts = append(u.asserts, f) 128 | // } 129 | 130 | func (u *upstreamServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { 131 | u.requests = u.requests + 1 132 | 133 | for _, assertf := range u.asserts { 134 | assertf(req) 135 | } 136 | 137 | if !u.Now.IsZero() { 138 | rw.Header().Set("Date", u.Now.Format(http.TimeFormat)) 139 | } 140 | 141 | if u.CacheControl != "" { 142 | rw.Header().Set("Cache-Control", u.CacheControl) 143 | } 144 | 145 | if u.Etag != "" { 146 | rw.Header().Set("Etag", u.Etag) 147 | } 148 | 149 | if u.Vary != "" { 150 | rw.Header().Set("Vary", u.Vary) 151 | } 152 | 153 | if u.Header != nil { 154 | for key, headers := range u.Header { 155 | for _, header := range headers { 156 | rw.Header().Add(key, header) 157 | } 158 | } 159 | } 160 | 161 | u.timeTravel(u.ResponseDuration) 162 | 163 | if u.StatusCode != 0 && u.StatusCode != 200 { 164 | rw.WriteHeader(u.StatusCode) 165 | io.Copy(rw, bytes.NewReader(u.Body)) 166 | } else { 167 | http.ServeContent(rw, req, u.Filename, u.LastModified, bytes.NewReader(u.Body)) 168 | } 169 | } 170 | 171 | func (u *upstreamServer) RoundTrip(req *http.Request) (*http.Response, error) { 172 | rec := httptest.NewRecorder() 173 | u.ServeHTTP(rec, req) 174 | rec.Flush() 175 | 176 | resp := newResponse(rec.Code, rec.Body.Bytes()) 177 | resp.Header = rec.Header() 178 | return resp, nil 179 | } 180 | 181 | // func cc(cc string) string { 182 | // return fmt.Sprintf("Cache-Control: %s", cc) 183 | // } 184 | 185 | func readAll(r io.Reader) []byte { 186 | b, err := io.ReadAll(r) 187 | if err != nil { 188 | panic(err) 189 | } 190 | return b 191 | } 192 | 193 | func readAllString(r io.Reader) string { 194 | return string(readAll(r)) 195 | } 196 | -------------------------------------------------------------------------------- /define/define.go: -------------------------------------------------------------------------------- 1 | package define 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "regexp" 7 | "strings" 8 | "text/template" 9 | ) 10 | 11 | const ( 12 | LINUX_ALL_DISTROS string = "all" 13 | LINUX_DISTROS_UBUNTU string = "ubuntu" 14 | LINUX_DISTROS_UBUNTU_PORTS string = "ubuntu-ports" 15 | LINUX_DISTROS_DEBIAN string = "debian" 16 | LINUX_DISTROS_CENTOS string = "centos" 17 | LINUX_DISTROS_ALPINE string = "alpine" 18 | ) 19 | 20 | const ( 21 | TYPE_LINUX_ALL_DISTROS int = 0 22 | TYPE_LINUX_DISTROS_UBUNTU int = 1 23 | TYPE_LINUX_DISTROS_UBUNTU_PORTS int = 2 24 | TYPE_LINUX_DISTROS_DEBIAN int = 3 25 | TYPE_LINUX_DISTROS_CENTOS int = 4 26 | TYPE_LINUX_DISTROS_ALPINE int = 5 27 | ) 28 | 29 | type Rule struct { 30 | OS int 31 | Pattern *regexp.Regexp 32 | CacheControl string 33 | Rewrite bool 34 | } 35 | 36 | func (r *Rule) String() string { 37 | return fmt.Sprintf("%s Cache-Control=%s Rewrite=%#v", 38 | r.Pattern.String(), r.CacheControl, r.Rewrite) 39 | } 40 | 41 | type UrlWithAlias struct { 42 | URL string 43 | Alias string 44 | Http bool 45 | Https bool 46 | Official bool 47 | Bandwidth int64 48 | } 49 | 50 | func GenerateAliasFromURL(url string) string { 51 | pureHost := regexp.MustCompile(`^https?://|\/.*`).ReplaceAllString(url, "") 52 | tldRemoved := regexp.MustCompile(`\.edu\.cn$|.cn$|\.com$|\.net$|\.net.cn$|\.org$|\.org\.cn$`).ReplaceAllString(pureHost, "") 53 | group := strings.Split(tldRemoved, ".") 54 | alias := group[len(group)-1] 55 | 56 | // Use templates for alias construction 57 | var buf bytes.Buffer 58 | data := AliasTemplateData{Alias: alias} 59 | if err := aliasTemplate.Execute(&buf, data); err != nil { 60 | // Fallback to concatenation if template fails 61 | return "cn:" + alias 62 | } 63 | return buf.String() 64 | } 65 | 66 | func GenerateBuildInMirorItem(url string, official bool) UrlWithAlias { 67 | var mirror UrlWithAlias 68 | mirror.Official = official 69 | mirror.Alias = GenerateAliasFromURL(url) 70 | 71 | if strings.HasPrefix(url, "http://") { 72 | mirror.Http = true 73 | mirror.Https = false 74 | } else if strings.HasPrefix(url, "https://") { 75 | mirror.Http = false 76 | mirror.Https = true 77 | } 78 | mirror.URL = url 79 | // TODO 80 | mirror.Bandwidth = 0 81 | return mirror 82 | } 83 | 84 | var ( 85 | // httpURLTemplate is a template for constructing HTTP URLs 86 | httpURLTemplate = template.Must(template.New("httpURL").Parse("http://{{.URL}}")) 87 | 88 | // httpsURLTemplate is a template for constructing HTTPS URLs 89 | httpsURLTemplate = template.Must(template.New("httpsURL").Parse("https://{{.URL}}")) 90 | 91 | // aliasTemplate is a template for constructing aliases 92 | aliasTemplate = template.Must(template.New("alias").Parse("cn:{{.Alias}}")) 93 | ) 94 | 95 | // URLTemplateData holds data for URL template execution 96 | type URLTemplateData struct { 97 | URL string 98 | } 99 | 100 | // AliasTemplateData holds data for alias template execution 101 | type AliasTemplateData struct { 102 | Alias string 103 | } 104 | 105 | // buildHTTPURL constructs an HTTP URL using templates 106 | func buildHTTPURL(url string) (string, error) { 107 | var buf bytes.Buffer 108 | data := URLTemplateData{URL: url} 109 | if err := httpURLTemplate.Execute(&buf, data); err != nil { 110 | return "", err 111 | } 112 | return buf.String(), nil 113 | } 114 | 115 | // buildHTTPSURL constructs an HTTPS URL using templates 116 | func buildHTTPSURL(url string) (string, error) { 117 | var buf bytes.Buffer 118 | data := URLTemplateData{URL: url} 119 | if err := httpsURLTemplate.Execute(&buf, data); err != nil { 120 | return "", err 121 | } 122 | return buf.String(), nil 123 | } 124 | 125 | func GenerateBuildInList(officialList []string, customList []string) (mirrors []UrlWithAlias) { 126 | for _, url := range officialList { 127 | if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { 128 | httpURL, err := buildHTTPURL(url) 129 | if err != nil { 130 | // Fallback to concatenation if template fails 131 | httpURL = "http://" + url 132 | } 133 | mirror := GenerateBuildInMirorItem(httpURL, true) 134 | mirrors = append(mirrors, mirror) 135 | 136 | httpsURL, err := buildHTTPSURL(url) 137 | if err != nil { 138 | // Fallback to concatenation if template fails 139 | httpsURL = "https://" + url 140 | } 141 | mirror = GenerateBuildInMirorItem(httpsURL, true) 142 | mirrors = append(mirrors, mirror) 143 | } else { 144 | mirror := GenerateBuildInMirorItem(url, true) 145 | mirrors = append(mirrors, mirror) 146 | } 147 | } 148 | 149 | for _, url := range customList { 150 | if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { 151 | httpURL, err := buildHTTPURL(url) 152 | if err != nil { 153 | // Fallback to concatenation if template fails 154 | httpURL = "http://" + url 155 | } 156 | mirror := GenerateBuildInMirorItem(httpURL, false) 157 | mirrors = append(mirrors, mirror) 158 | 159 | httpsURL, err := buildHTTPSURL(url) 160 | if err != nil { 161 | // Fallback to concatenation if template fails 162 | httpsURL = "https://" + url 163 | } 164 | mirror = GenerateBuildInMirorItem(httpsURL, false) 165 | mirrors = append(mirrors, mirror) 166 | } else { 167 | mirror := GenerateBuildInMirorItem(url, false) 168 | mirrors = append(mirrors, mirror) 169 | } 170 | } 171 | 172 | return mirrors 173 | } 174 | -------------------------------------------------------------------------------- /pkg/vfs/file.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "os" 5 | "path" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | // EntryType indicates the type of the entry. 11 | type EntryType uint8 12 | 13 | const ( 14 | // EntryTypeFile indicates the entry is a file. 15 | EntryTypeFile EntryType = iota + 1 16 | // EntryTypeDir indicates the entry is a directory. 17 | EntryTypeDir 18 | ) 19 | 20 | const ( 21 | ModeCompress os.FileMode = 1 << 16 22 | ) 23 | 24 | // Entry is the interface implemented by the in-memory representations 25 | // of files and directories. 26 | type Entry interface { 27 | // Type returns the entry type, either EntryTypeFile or 28 | // EntryTypeDir. 29 | Type() EntryType 30 | // Size returns the file size. For directories, it's always zero. 31 | Size() int64 32 | // FileMode returns the file mode as an os.FileMode. 33 | FileMode() os.FileMode 34 | // ModificationTime returns the last time the file or the directory 35 | // was modified. 36 | ModificationTime() time.Time 37 | } 38 | 39 | // Type File represents an in-memory file. Most in-memory VFS implementations 40 | // should use this structure to represent their files, in order to save work. 41 | type File struct { 42 | sync.RWMutex 43 | // Data contains the file data. 44 | Data []byte 45 | // Mode is the file or directory mode. Note that some filesystems 46 | // might ignore the permission bits. 47 | Mode os.FileMode 48 | // ModTime represents the last modification time to the file. 49 | ModTime time.Time 50 | } 51 | 52 | func (f *File) Type() EntryType { 53 | return EntryTypeFile 54 | } 55 | 56 | func (f *File) Size() int64 { 57 | f.RLock() 58 | defer f.RUnlock() 59 | return int64(len(f.Data)) 60 | } 61 | 62 | func (f *File) FileMode() os.FileMode { 63 | return f.Mode 64 | } 65 | 66 | func (f *File) ModificationTime() time.Time { 67 | f.RLock() 68 | defer f.RUnlock() 69 | return f.ModTime 70 | } 71 | 72 | // Type Dir represents an in-memory directory. Most in-memory VFS 73 | // implementations should use this structure to represent their 74 | // directories, in order to save work. 75 | type Dir struct { 76 | sync.RWMutex 77 | // Mode is the file or directory mode. Note that some filesystems 78 | // might ignore the permission bits. 79 | Mode os.FileMode 80 | // ModTime represents the last modification time to directory. 81 | ModTime time.Time 82 | // Entry names in this directory, in order. 83 | EntryNames []string 84 | // Entries in the same order as EntryNames. 85 | Entries []Entry 86 | } 87 | 88 | func (d *Dir) Type() EntryType { 89 | return EntryTypeDir 90 | } 91 | 92 | func (d *Dir) Size() int64 { 93 | return 0 94 | } 95 | 96 | func (d *Dir) FileMode() os.FileMode { 97 | return d.Mode 98 | } 99 | 100 | func (d *Dir) ModificationTime() time.Time { 101 | d.RLock() 102 | defer d.RUnlock() 103 | return d.ModTime 104 | } 105 | 106 | // Add ads a new entry to the directory. If there's already an 107 | // entry ith the same name, an error is returned. 108 | func (d *Dir) Add(name string, entry Entry) error { 109 | // TODO: Binary search 110 | for ii, v := range d.EntryNames { 111 | if v > name { 112 | names := make([]string, len(d.EntryNames)+1) 113 | copy(names, d.EntryNames[:ii]) 114 | names[ii] = name 115 | copy(names[ii+1:], d.EntryNames[ii:]) 116 | d.EntryNames = names 117 | 118 | entries := make([]Entry, len(d.Entries)+1) 119 | copy(entries, d.Entries[:ii]) 120 | entries[ii] = entry 121 | copy(entries[ii+1:], d.Entries[ii:]) 122 | 123 | d.Entries = entries 124 | return nil 125 | } 126 | if v == name { 127 | return os.ErrExist 128 | } 129 | } 130 | // Not added yet, put at the end 131 | d.EntryNames = append(d.EntryNames, name) 132 | d.Entries = append(d.Entries, entry) 133 | return nil 134 | } 135 | 136 | // Find returns the entry with the given name and its index, 137 | // or an error if an entry with that name does not exist in 138 | // the directory. 139 | func (d *Dir) Find(name string) (Entry, int, error) { 140 | for ii, v := range d.EntryNames { 141 | if v == name { 142 | return d.Entries[ii], ii, nil 143 | } 144 | } 145 | return nil, -1, os.ErrNotExist 146 | } 147 | 148 | // EntryInfo implements the os.FileInfo interface wrapping 149 | // a given File and its Path in its VFS. 150 | type EntryInfo struct { 151 | // Path is the full path to the entry in its VFS. 152 | Path string 153 | // Entry is the instance used by the VFS to represent 154 | // the in-memory entry. 155 | Entry Entry 156 | } 157 | 158 | func (info *EntryInfo) Name() string { 159 | return path.Base(info.Path) 160 | } 161 | 162 | func (info *EntryInfo) Size() int64 { 163 | return info.Entry.Size() 164 | } 165 | 166 | func (info *EntryInfo) Mode() os.FileMode { 167 | return info.Entry.FileMode() 168 | } 169 | 170 | func (info *EntryInfo) ModTime() time.Time { 171 | return info.Entry.ModificationTime() 172 | } 173 | 174 | func (info *EntryInfo) IsDir() bool { 175 | return info.Entry.Type() == EntryTypeDir 176 | } 177 | 178 | // Sys returns the underlying Entry. 179 | func (info *EntryInfo) Sys() interface{} { 180 | return info.Entry 181 | } 182 | 183 | // FileInfos represents an slice of os.FileInfo which 184 | // implements the sort.Interface. This type is only 185 | // exported for users who want to implement their own 186 | // filesystems, since VFS.ReadDir requires the returned 187 | // []os.FileInfo to be sorted by name. 188 | type FileInfos []os.FileInfo 189 | 190 | func (f FileInfos) Len() int { 191 | return len(f) 192 | } 193 | 194 | func (f FileInfos) Less(i, j int) bool { 195 | return f[i].Name() < f[j].Name() 196 | } 197 | 198 | func (f FileInfos) Swap(i, j int) { 199 | f[i], f[j] = f[j], f[i] 200 | } 201 | -------------------------------------------------------------------------------- /internal/mirrors/mirrors.go: -------------------------------------------------------------------------------- 1 | package mirrors 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | 7 | define "github.com/soulteary/apt-proxy/define" 8 | ) 9 | 10 | func GenerateMirrorListByPredefined(osType int) (mirrors []string) { 11 | var src []define.UrlWithAlias 12 | switch osType { 13 | case define.TYPE_LINUX_ALL_DISTROS: 14 | src = append(src, define.BUILDIN_UBUNTU_MIRRORS...) 15 | src = append(src, define.BUILDIN_UBUNTU_PORTS_MIRRORS...) 16 | src = append(src, define.BUILDIN_DEBIAN_MIRRORS...) 17 | src = append(src, define.BUILDIN_CENTOS_MIRRORS...) 18 | src = append(src, define.BUILDIN_ALPINE_MIRRORS...) 19 | case define.TYPE_LINUX_DISTROS_UBUNTU: 20 | src = define.BUILDIN_UBUNTU_MIRRORS 21 | case define.TYPE_LINUX_DISTROS_UBUNTU_PORTS: 22 | src = define.BUILDIN_UBUNTU_PORTS_MIRRORS 23 | case define.TYPE_LINUX_DISTROS_DEBIAN: 24 | src = define.BUILDIN_DEBIAN_MIRRORS 25 | case define.TYPE_LINUX_DISTROS_CENTOS: 26 | src = define.BUILDIN_CENTOS_MIRRORS 27 | case define.TYPE_LINUX_DISTROS_ALPINE: 28 | src = define.BUILDIN_ALPINE_MIRRORS 29 | } 30 | 31 | for _, mirror := range src { 32 | mirrors = append(mirrors, mirror.URL) 33 | } 34 | return mirrors 35 | } 36 | 37 | var BUILDIN_UBUNTU_MIRRORS = GenerateMirrorListByPredefined(define.TYPE_LINUX_DISTROS_UBUNTU) 38 | var BUILDIN_UBUNTU_PORTS_MIRRORS = GenerateMirrorListByPredefined(define.TYPE_LINUX_DISTROS_UBUNTU_PORTS) 39 | var BUILDIN_DEBIAN_MIRRORS = GenerateMirrorListByPredefined(define.TYPE_LINUX_DISTROS_DEBIAN) 40 | var BUILDIN_CENTOS_MIRRORS = GenerateMirrorListByPredefined(define.TYPE_LINUX_DISTROS_CENTOS) 41 | var BUILDIN_ALPINE_MIRRORS = GenerateMirrorListByPredefined(define.TYPE_LINUX_DISTROS_ALPINE) 42 | 43 | func GetGeoMirrorUrlsByMode(mode int) (mirrors []string) { 44 | if mode == define.TYPE_LINUX_DISTROS_UBUNTU { 45 | ubuntuMirrorsOnline, err := GetUbuntuMirrorUrlsByGeo() 46 | if err != nil { 47 | return BUILDIN_UBUNTU_MIRRORS 48 | } 49 | return ubuntuMirrorsOnline 50 | } 51 | 52 | if mode == define.TYPE_LINUX_DISTROS_UBUNTU_PORTS { 53 | ubuntuPortsMirrorsOnline, err := GetUbuntuMirrorUrlsByGeo() 54 | if err != nil { 55 | return BUILDIN_UBUNTU_PORTS_MIRRORS 56 | } 57 | 58 | results := make([]string, 0, len(ubuntuPortsMirrorsOnline)) 59 | for _, mirror := range ubuntuPortsMirrorsOnline { 60 | results = append(results, strings.ReplaceAll(mirror, "/ubuntu/", "/ubuntu-ports/")) 61 | } 62 | return results 63 | } 64 | 65 | if mode == define.TYPE_LINUX_DISTROS_DEBIAN { 66 | return BUILDIN_DEBIAN_MIRRORS 67 | } 68 | 69 | if mode == define.TYPE_LINUX_DISTROS_CENTOS { 70 | return BUILDIN_CENTOS_MIRRORS 71 | } 72 | 73 | if mode == define.TYPE_LINUX_DISTROS_ALPINE { 74 | return BUILDIN_ALPINE_MIRRORS 75 | } 76 | 77 | mirrors = append(mirrors, BUILDIN_UBUNTU_MIRRORS...) 78 | mirrors = append(mirrors, BUILDIN_UBUNTU_PORTS_MIRRORS...) 79 | mirrors = append(mirrors, BUILDIN_DEBIAN_MIRRORS...) 80 | mirrors = append(mirrors, BUILDIN_CENTOS_MIRRORS...) 81 | mirrors = append(mirrors, BUILDIN_ALPINE_MIRRORS...) 82 | return mirrors 83 | } 84 | 85 | func GetFullMirrorURL(mirror define.UrlWithAlias) string { 86 | if mirror.Http { 87 | if strings.HasPrefix(mirror.URL, "http://") { 88 | return mirror.URL 89 | } 90 | url, err := BuildHTTPURL(mirror.URL) 91 | if err != nil { 92 | // Fallback to concatenation if template fails 93 | return "http://" + mirror.URL 94 | } 95 | return url 96 | } 97 | if mirror.Https { 98 | if strings.HasPrefix(mirror.URL, "https://") { 99 | return mirror.URL 100 | } 101 | url, err := BuildHTTPSURL(mirror.URL) 102 | if err != nil { 103 | // Fallback to concatenation if template fails 104 | return "https://" + mirror.URL 105 | } 106 | return url 107 | } 108 | url, err := BuildHTTPSURL(mirror.URL) 109 | if err != nil { 110 | // Fallback to concatenation if template fails 111 | return "https://" + mirror.URL 112 | } 113 | return url 114 | } 115 | 116 | func GetMirrorURLByAliases(osType int, alias string) string { 117 | switch osType { 118 | case define.TYPE_LINUX_DISTROS_UBUNTU: 119 | for _, mirror := range define.BUILDIN_UBUNTU_MIRRORS { 120 | if mirror.Alias == alias { 121 | return GetFullMirrorURL(mirror) 122 | } 123 | } 124 | case define.TYPE_LINUX_DISTROS_UBUNTU_PORTS: 125 | for _, mirror := range define.BUILDIN_UBUNTU_PORTS_MIRRORS { 126 | if mirror.Alias == alias { 127 | return GetFullMirrorURL(mirror) 128 | } 129 | } 130 | case define.TYPE_LINUX_DISTROS_DEBIAN: 131 | for _, mirror := range define.BUILDIN_DEBIAN_MIRRORS { 132 | if mirror.Alias == alias { 133 | return GetFullMirrorURL(mirror) 134 | } 135 | } 136 | case define.TYPE_LINUX_DISTROS_CENTOS: 137 | for _, mirror := range define.BUILDIN_CENTOS_MIRRORS { 138 | if mirror.Alias == alias { 139 | return GetFullMirrorURL(mirror) 140 | } 141 | } 142 | case define.TYPE_LINUX_DISTROS_ALPINE: 143 | for _, mirror := range define.BUILDIN_ALPINE_MIRRORS { 144 | if mirror.Alias == alias { 145 | return GetFullMirrorURL(mirror) 146 | } 147 | } 148 | } 149 | return "" 150 | } 151 | 152 | func GetPredefinedConfiguration(proxyMode int) (string, *regexp.Regexp) { 153 | switch proxyMode { 154 | case define.TYPE_LINUX_DISTROS_UBUNTU: 155 | return define.UBUNTU_BENCHMAKR_URL, define.UBUNTU_HOST_PATTERN 156 | case define.TYPE_LINUX_DISTROS_UBUNTU_PORTS: 157 | return define.UBUNTU_PORTS_BENCHMAKR_URL, define.UBUNTU_PORTS_HOST_PATTERN 158 | case define.TYPE_LINUX_DISTROS_DEBIAN: 159 | return define.DEBIAN_BENCHMAKR_URL, define.DEBIAN_HOST_PATTERN 160 | case define.TYPE_LINUX_DISTROS_CENTOS: 161 | return define.CENTOS_BENCHMAKR_URL, define.CENTOS_HOST_PATTERN 162 | case define.TYPE_LINUX_DISTROS_ALPINE: 163 | return define.ALPINE_BENCHMAKR_URL, define.ALPINE_HOST_PATTERN 164 | } 165 | return "", nil 166 | } 167 | -------------------------------------------------------------------------------- /cli/cli.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/soulteary/apt-proxy/define" 9 | "github.com/soulteary/apt-proxy/internal/mirrors" 10 | "github.com/soulteary/apt-proxy/state" 11 | ) 12 | 13 | // defaults holds all default configuration values 14 | type defaults struct { 15 | Host string 16 | Port string 17 | CacheDir string 18 | UbuntuMirror string 19 | UbuntuPortsMirror string 20 | DebianMirror string 21 | CentOSMirror string 22 | AlpineMirror string 23 | ModeName string 24 | Debug bool 25 | } 26 | 27 | var ( 28 | // Version is set during build time 29 | Version string 30 | 31 | // defaultConfig holds default configuration values 32 | defaultConfig = defaults{ 33 | Host: "0.0.0.0", 34 | Port: "3142", 35 | CacheDir: "./.aptcache", 36 | UbuntuMirror: "", // "https://mirrors.tuna.tsinghua.edu.cn/ubuntu/" 37 | UbuntuPortsMirror: "", // "https://mirrors.tuna.tsinghua.edu.cn/ubuntu-ports/" 38 | DebianMirror: "", // "https://mirrors.tuna.tsinghua.edu.cn/debian/" 39 | CentOSMirror: "", // "https://mirrors.tuna.tsinghua.edu.cn/centos/" 40 | AlpineMirror: "", // "https://mirrors.tuna.tsinghua.edu.cn/alpine/" 41 | ModeName: define.LINUX_ALL_DISTROS, 42 | Debug: false, 43 | } 44 | 45 | // validModes maps mode strings to their corresponding integer values 46 | validModes = map[string]int{ 47 | define.LINUX_DISTROS_UBUNTU: define.TYPE_LINUX_DISTROS_UBUNTU, 48 | define.LINUX_DISTROS_UBUNTU_PORTS: define.TYPE_LINUX_DISTROS_UBUNTU_PORTS, 49 | define.LINUX_DISTROS_DEBIAN: define.TYPE_LINUX_DISTROS_DEBIAN, 50 | define.LINUX_DISTROS_CENTOS: define.TYPE_LINUX_DISTROS_CENTOS, 51 | define.LINUX_DISTROS_ALPINE: define.TYPE_LINUX_DISTROS_ALPINE, 52 | define.LINUX_ALL_DISTROS: define.TYPE_LINUX_ALL_DISTROS, 53 | } 54 | ) 55 | 56 | // getProxyMode converts a mode string (e.g., "ubuntu", "debian", "all") to its 57 | // corresponding integer constant. Returns an error if the mode is invalid. 58 | func getProxyMode(mode string) (int, error) { 59 | if modeValue, exists := validModes[mode]; exists { 60 | return modeValue, nil 61 | } 62 | return 0, fmt.Errorf("invalid mode: %s", mode) 63 | } 64 | 65 | // ParseFlags parses command-line flags and returns a Config struct with all 66 | // application settings. It validates the mode parameter and sets up global state. 67 | // Returns an error if flag parsing fails or if an invalid mode is specified. 68 | func ParseFlags() (*Config, error) { 69 | flags := flag.NewFlagSet("apt-proxy", flag.ContinueOnError) 70 | 71 | var ( 72 | host string 73 | port string 74 | userMode string 75 | config Config 76 | ) 77 | 78 | // Define flags 79 | flags.StringVar(&host, "host", defaultConfig.Host, "the host to bind to") 80 | flags.StringVar(&port, "port", defaultConfig.Port, "the port to bind to") 81 | flags.StringVar(&userMode, "mode", defaultConfig.ModeName, 82 | "select the mode of system to cache: all / ubuntu / ubuntu-ports / debian / centos / alpine") 83 | flags.BoolVar(&config.Debug, "debug", defaultConfig.Debug, "whether to output debugging logging") 84 | flags.StringVar(&config.CacheDir, "cachedir", defaultConfig.CacheDir, "the dir to store cache data in") 85 | flags.StringVar(&config.Mirrors.Ubuntu, "ubuntu", defaultConfig.UbuntuMirror, "the ubuntu mirror for fetching packages") 86 | flags.StringVar(&config.Mirrors.UbuntuPorts, "ubuntu-ports", defaultConfig.UbuntuPortsMirror, "the ubuntu ports mirror for fetching packages") 87 | flags.StringVar(&config.Mirrors.Debian, "debian", defaultConfig.DebianMirror, "the debian mirror for fetching packages") 88 | flags.StringVar(&config.Mirrors.CentOS, "centos", defaultConfig.CentOSMirror, "the centos mirror for fetching packages") 89 | flags.StringVar(&config.Mirrors.Alpine, "alpine", defaultConfig.AlpineMirror, "the alpine mirror for fetching packages") 90 | 91 | if err := flags.Parse(os.Args[1:]); err != nil { 92 | return nil, fmt.Errorf("parsing flags: %w", err) 93 | } 94 | 95 | // Validate and set mode 96 | mode, err := getProxyMode(userMode) 97 | if err != nil { 98 | return nil, err 99 | } 100 | config.Mode = mode 101 | 102 | // Set listen address using templates 103 | listenAddr, err := mirrors.BuildListenAddress(host, port) 104 | if err != nil { 105 | // Fallback to fmt.Sprintf if template fails 106 | config.Listen = fmt.Sprintf("%s:%s", host, port) 107 | } else { 108 | config.Listen = listenAddr 109 | } 110 | config.Version = Version 111 | 112 | // Update global state 113 | if err := updateGlobalState(&config); err != nil { 114 | return nil, fmt.Errorf("updating global state: %w", err) 115 | } 116 | 117 | return &config, nil 118 | } 119 | 120 | // updateGlobalState updates the global state with the current configuration, 121 | // including proxy mode and mirror URLs for all supported distributions. 122 | // This enables components throughout the application to access configuration. 123 | func updateGlobalState(config *Config) error { 124 | state.SetProxyMode(config.Mode) 125 | 126 | state.SetUbuntuMirror(config.Mirrors.Ubuntu) 127 | state.SetUbuntuPortsMirror(config.Mirrors.UbuntuPorts) 128 | state.SetDebianMirror(config.Mirrors.Debian) 129 | state.SetCentOSMirror(config.Mirrors.CentOS) 130 | state.SetAlpineMirror(config.Mirrors.Alpine) 131 | 132 | return nil 133 | } 134 | 135 | // ValidateConfig performs validation on the configuration to ensure all required 136 | // fields are set and valid. Returns an error if validation fails. 137 | func ValidateConfig(config *Config) error { 138 | if config == nil { 139 | return fmt.Errorf("configuration cannot be nil") 140 | } 141 | 142 | if config.CacheDir == "" { 143 | return fmt.Errorf("cache directory must be specified") 144 | } 145 | 146 | if config.Listen == "" { 147 | return fmt.Errorf("listen address must be specified") 148 | } 149 | 150 | return nil 151 | } 152 | -------------------------------------------------------------------------------- /pkg/httpcache/resource.go: -------------------------------------------------------------------------------- 1 | package httpcache 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | const ( 14 | lastModDivisor = 10 15 | viaPseudonym = "httpcache" 16 | ) 17 | 18 | var Clock = func() time.Time { 19 | return time.Now().UTC() 20 | } 21 | 22 | type ReadSeekCloser interface { 23 | io.Reader 24 | io.Seeker 25 | io.Closer 26 | } 27 | 28 | type byteReadSeekCloser struct { 29 | *bytes.Reader 30 | } 31 | 32 | func (brsc *byteReadSeekCloser) Close() error { return nil } 33 | 34 | type Resource struct { 35 | ReadSeekCloser 36 | RequestTime, ResponseTime time.Time 37 | header http.Header 38 | statusCode int 39 | cc CacheControl 40 | stale bool 41 | } 42 | 43 | func NewResource(statusCode int, body ReadSeekCloser, hdrs http.Header) *Resource { 44 | return &Resource{ 45 | header: hdrs, 46 | ReadSeekCloser: body, 47 | statusCode: statusCode, 48 | } 49 | } 50 | 51 | func NewResourceBytes(statusCode int, b []byte, hdrs http.Header) *Resource { 52 | return &Resource{ 53 | header: hdrs, 54 | statusCode: statusCode, 55 | ReadSeekCloser: &byteReadSeekCloser{bytes.NewReader(b)}, 56 | } 57 | } 58 | 59 | func (r *Resource) IsNonErrorStatus() bool { 60 | return r.statusCode >= 200 && r.statusCode < 400 61 | } 62 | 63 | func (r *Resource) Status() int { 64 | return r.statusCode 65 | } 66 | 67 | func (r *Resource) Header() http.Header { 68 | return r.header 69 | } 70 | 71 | func (r *Resource) IsStale() bool { 72 | return r.stale 73 | } 74 | 75 | func (r *Resource) MarkStale() { 76 | r.stale = true 77 | } 78 | 79 | func (r *Resource) cacheControl() (CacheControl, error) { 80 | if r.cc != nil { 81 | return r.cc, nil 82 | } 83 | 84 | cc, err := ParseCacheControlHeaders(r.header) 85 | if err != nil { 86 | return cc, err 87 | } 88 | 89 | r.cc = cc 90 | return cc, nil 91 | } 92 | 93 | func (r *Resource) LastModified() time.Time { 94 | var modTime time.Time 95 | 96 | if lastModHeader := r.header.Get("Last-Modified"); lastModHeader != "" { 97 | if t, err := http.ParseTime(lastModHeader); err == nil { 98 | modTime = t 99 | } 100 | } 101 | 102 | return modTime 103 | } 104 | 105 | func (r *Resource) Expires() (time.Time, error) { 106 | if expires := r.header.Get("Expires"); expires != "" { 107 | return http.ParseTime(expires) 108 | } 109 | 110 | return time.Time{}, nil 111 | } 112 | 113 | func (r *Resource) MustValidate(shared bool) bool { 114 | cc, err := r.cacheControl() 115 | if err != nil { 116 | debugf("Error parsing Cache-Control: %v", err.Error()) 117 | return true 118 | } 119 | 120 | // The s-maxage directive also implies the semantics of proxy-revalidate 121 | if cc.Has("s-maxage") && shared { 122 | return true 123 | } 124 | 125 | if cc.Has("must-revalidate") || (cc.Has("proxy-revalidate") && shared) { 126 | return true 127 | } 128 | 129 | return false 130 | } 131 | 132 | func (r *Resource) DateAfter(d time.Time) bool { 133 | if dateHeader := r.header.Get("Date"); dateHeader != "" { 134 | if t, err := http.ParseTime(dateHeader); err != nil { 135 | return false 136 | } else { 137 | return t.After(d) 138 | } 139 | } 140 | return false 141 | } 142 | 143 | // Calculate the age of the resource 144 | func (r *Resource) Age() (time.Duration, error) { 145 | var age time.Duration 146 | 147 | if ageInt, err := intHeader("Age", r.header); err == nil { 148 | age = time.Second * time.Duration(ageInt) 149 | } 150 | 151 | if proxyDate, err := timeHeader(ProxyDateHeader, r.header); err == nil { 152 | return Clock().Sub(proxyDate) + age, nil 153 | } 154 | 155 | if date, err := timeHeader("Date", r.header); err == nil { 156 | return Clock().Sub(date) + age, nil 157 | } 158 | 159 | return time.Duration(0), errors.New("unable to calculate age") 160 | } 161 | 162 | func (r *Resource) MaxAge(shared bool) (time.Duration, error) { 163 | cc, err := r.cacheControl() 164 | if err != nil { 165 | return time.Duration(0), err 166 | } 167 | 168 | if cc.Has("s-maxage") && shared { 169 | if maxAge, err := cc.Duration("s-maxage"); err != nil { 170 | return time.Duration(0), err 171 | } else if maxAge > 0 { 172 | return maxAge, nil 173 | } 174 | } 175 | 176 | if cc.Has("max-age") { 177 | if maxAge, err := cc.Duration("max-age"); err != nil { 178 | return time.Duration(0), err 179 | } else if maxAge > 0 { 180 | return maxAge, nil 181 | } 182 | } 183 | 184 | if expiresVal := r.header.Get("Expires"); expiresVal != "" { 185 | expires, err := http.ParseTime(expiresVal) 186 | if err != nil { 187 | return time.Duration(0), err 188 | } 189 | return expires.Sub(Clock()), nil 190 | } 191 | 192 | return time.Duration(0), nil 193 | } 194 | 195 | func (r *Resource) RemovePrivateHeaders() { 196 | cc, err := r.cacheControl() 197 | if err != nil { 198 | debugf("Error parsing Cache-Control: %s", err.Error()) 199 | } 200 | 201 | for _, p := range cc["private"] { 202 | debugf("removing private header %q", p) 203 | r.header.Del(p) 204 | } 205 | } 206 | 207 | func (r *Resource) HasValidators() bool { 208 | if r.header.Get("Last-Modified") != "" || r.header.Get("Etag") != "" { 209 | return true 210 | } 211 | 212 | return false 213 | } 214 | 215 | func (r *Resource) HasExplicitExpiration() bool { 216 | cc, err := r.cacheControl() 217 | if err != nil { 218 | debugf("Error parsing Cache-Control: %s", err.Error()) 219 | return false 220 | } 221 | 222 | if d, _ := cc.Duration("max-age"); d > time.Duration(0) { 223 | return true 224 | } 225 | 226 | if d, _ := cc.Duration("s-maxage"); d > time.Duration(0) { 227 | return true 228 | } 229 | 230 | if exp, _ := r.Expires(); !exp.IsZero() { 231 | return true 232 | } 233 | 234 | return false 235 | } 236 | 237 | func (r *Resource) HeuristicFreshness() time.Duration { 238 | if !r.HasExplicitExpiration() && r.header.Get("Last-Modified") != "" { 239 | return Clock().Sub(r.LastModified()) / time.Duration(lastModDivisor) 240 | } 241 | 242 | return time.Duration(0) 243 | } 244 | 245 | func (r *Resource) Via() string { 246 | via := []string{} 247 | via = append(via, fmt.Sprintf("1.1 %s", viaPseudonym)) 248 | return strings.Join(via, ",") 249 | } 250 | -------------------------------------------------------------------------------- /cli/daemon.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "os/signal" 9 | "syscall" 10 | "time" 11 | 12 | "github.com/soulteary/apt-proxy/internal/server" 13 | "github.com/soulteary/apt-proxy/pkg/httpcache" 14 | "github.com/soulteary/apt-proxy/pkg/httplog" 15 | ) 16 | 17 | // Config holds all application configuration 18 | type Config struct { 19 | Debug bool 20 | Version string 21 | CacheDir string 22 | Mode int 23 | Listen string 24 | Mirrors MirrorConfig 25 | } 26 | 27 | // MirrorConfig holds mirror-specific configuration 28 | type MirrorConfig struct { 29 | Ubuntu string 30 | UbuntuPorts string 31 | Debian string 32 | CentOS string 33 | Alpine string 34 | } 35 | 36 | // Server represents the main application server that handles HTTP requests, 37 | // manages caching, and coordinates all server components. 38 | type Server struct { 39 | config *Config // Application configuration 40 | cache httpcache.Cache // HTTP cache implementation 41 | proxy *server.PackageStruct // Main proxy router 42 | logger *httplog.ResponseLogger // Request/response logger 43 | router http.Handler // HTTP router with orthodox routing 44 | server *http.Server // HTTP server instance 45 | } 46 | 47 | // NewServer creates and initializes a new Server instance with the provided 48 | // configuration. It sets up caching, proxy routing, logging, and HTTP server. 49 | // Returns an error if initialization fails. 50 | func NewServer(cfg *Config) (*Server, error) { 51 | if cfg == nil { 52 | return nil, fmt.Errorf("configuration cannot be nil") 53 | } 54 | 55 | s := &Server{ 56 | config: cfg, 57 | } 58 | 59 | if err := s.initialize(); err != nil { 60 | return nil, fmt.Errorf("failed to initialize server: %w", err) 61 | } 62 | 63 | return s, nil 64 | } 65 | 66 | // initialize sets up all server components including cache, proxy router, 67 | // logging, and HTTP server configuration. This method is called automatically 68 | // by NewServer and should not be called directly. 69 | func (s *Server) initialize() error { 70 | // Initialize cache 71 | cache, err := httpcache.NewDiskCache(s.config.CacheDir) 72 | if err != nil { 73 | return fmt.Errorf("failed to initialize cache: %w", err) 74 | } 75 | s.cache = cache 76 | 77 | // Initialize proxy 78 | s.proxy = server.CreatePackageStructRouter(s.config.CacheDir) 79 | 80 | // Wrap proxy with cache 81 | cachedHandler := httpcache.NewHandler(s.cache, s.proxy.Handler) 82 | s.proxy.Handler = cachedHandler 83 | 84 | // Initialize logger 85 | if s.config.Debug { 86 | log.Printf("debug mode enabled") 87 | httpcache.DebugLogging = true 88 | } 89 | s.logger = httplog.NewResponseLogger(cachedHandler) 90 | s.logger.DumpRequests = s.config.Debug 91 | s.logger.DumpResponses = s.config.Debug 92 | s.logger.DumpErrors = s.config.Debug 93 | 94 | // Create router with orthodox routing (home and ping handlers) 95 | s.router = server.CreateRouter(s.logger, s.config.CacheDir) 96 | 97 | // Initialize HTTP server 98 | s.server = &http.Server{ 99 | Addr: s.config.Listen, 100 | Handler: s.router, 101 | ReadHeaderTimeout: 50 * time.Second, 102 | ReadTimeout: 50 * time.Second, 103 | WriteTimeout: 100 * time.Second, 104 | IdleTimeout: 120 * time.Second, 105 | } 106 | 107 | return nil 108 | } 109 | 110 | // Start begins serving HTTP requests and handles graceful shutdown on SIGINT or SIGTERM. 111 | // The server runs in a goroutine while the main goroutine waits for shutdown signals. 112 | // Returns an error if the server fails to start or encounters a fatal error. 113 | func (s *Server) Start() error { 114 | log.Printf("starting apt-proxy %s", s.config.Version) 115 | log.Printf("listening on %s", s.config.Listen) 116 | 117 | // Setup graceful shutdown 118 | ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) 119 | defer stop() 120 | 121 | // Start server in goroutine 122 | serverErr := make(chan error, 1) 123 | go func() { 124 | if err := s.server.ListenAndServe(); err != nil && err != http.ErrServerClosed { 125 | serverErr <- err 126 | } 127 | }() 128 | 129 | log.Println("server started successfully 🚀") 130 | 131 | // Wait for shutdown signal or server error 132 | select { 133 | case err := <-serverErr: 134 | return fmt.Errorf("server error: %w", err) 135 | case <-ctx.Done(): 136 | return s.shutdown() 137 | } 138 | } 139 | 140 | // shutdown performs a graceful server shutdown with a 5-second timeout. 141 | // It allows in-flight requests to complete before closing the server. 142 | // Returns an error if shutdown fails or times out. 143 | func (s *Server) shutdown() error { 144 | log.Println("shutting down server...") 145 | 146 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 147 | defer cancel() 148 | 149 | if err := s.server.Shutdown(ctx); err != nil { 150 | return fmt.Errorf("failed to shutdown server gracefully: %w", err) 151 | } 152 | 153 | log.Println("server shutdown complete") 154 | return nil 155 | } 156 | 157 | // Daemon is the main entry point for starting the application daemon. 158 | // It validates the configuration, creates and starts the server, and handles 159 | // any startup errors. This function blocks until the server shuts down. 160 | func Daemon(flags *Config) { 161 | if flags == nil { 162 | log.Fatalf("configuration cannot be nil") 163 | } 164 | 165 | if err := ValidateConfig(flags); err != nil { 166 | log.Fatalf("invalid configuration: %v", err) 167 | } 168 | 169 | cfg := &Config{ 170 | Debug: flags.Debug, 171 | Version: flags.Version, 172 | CacheDir: flags.CacheDir, 173 | Mode: flags.Mode, 174 | Listen: flags.Listen, 175 | Mirrors: MirrorConfig{ 176 | Ubuntu: flags.Mirrors.Ubuntu, 177 | UbuntuPorts: flags.Mirrors.UbuntuPorts, 178 | Debian: flags.Mirrors.Debian, 179 | CentOS: flags.Mirrors.CentOS, 180 | Alpine: flags.Mirrors.Alpine, 181 | }, 182 | } 183 | 184 | server, err := NewServer(cfg) 185 | if err != nil { 186 | log.Fatalf("failed to create server: %v", err) 187 | } 188 | 189 | if err := server.Start(); err != nil { 190 | log.Fatalf("server error: %v", err) 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /README_CN.md: -------------------------------------------------------------------------------- 1 | # APT Proxy / 轻量 APT 加速工具 2 | 3 | [![Security Scan](https://github.com/soulteary/apt-proxy/actions/workflows/scan.yml/badge.svg)](https://github.com/soulteary/apt-proxy/actions/workflows/scan.yml) [![Release](https://github.com/soulteary/apt-proxy/actions/workflows/release.yaml/badge.svg)](https://github.com/soulteary/apt-proxy/actions/workflows/release.yaml) [![goreportcard](https://img.shields.io/badge/go%20report-A+-brightgreen.svg?style=flat)](https://goreportcard.com/report/github.com/soulteary/apt-proxy) [![Docker Image](https://img.shields.io/docker/pulls/soulteary/apt-proxy.svg)](https://hub.docker.com/r/soulteary/apt-proxy) 4 | 5 | 6 |

7 | ENGLISH | 中文文档 8 |

9 | 10 | 11 | 12 | > 一个轻量级的 APT 缓存代理 - 仅仅只有 2MB 大小！ 13 | 14 | 15 | 16 | APT Proxy 是一个轻量级的缓存工具，用于 APT、YUM 和 APK 包（支持 Ubuntu、Debian、CentOS 和 Alpine Linux）。它可以无缝地与传统系统安装和 Docker 环境配合使用。 17 | 18 | 你也可以将它作为古老的 [apt-cacher-ng](https://www.unix-ag.uni-kl.de/~bloch/acng/) 安全可靠的替代品。 19 | 20 | ## 支持的平台 21 | 22 | - Linux：x86_64 / x86_32 / Ubuntu ARM64v8 23 | - ARM：ARM64v8 / ARM32v6 / ARM32v7 24 | - macOS：x86_64 / Apple Silicon (ARM64v8) 25 | 26 | ## 快速开始 27 | 28 | 直接运行二进制文件： 29 | 30 | ```bash 31 | ./apt-proxy 32 | 33 | 2022/06/12 16:15:40 running apt-proxy 34 | 2022/06/12 16:15:41 Start benchmarking mirrors 35 | 2022/06/12 16:15:41 Finished benchmarking mirrors 36 | 2022/06/12 16:15:41 using fastest mirror https://mirrors.company.ltd/ubuntu/ 37 | 2022/06/12 16:15:41 proxy listening on 0.0.0.0:3142 38 | ``` 39 | 40 | 当你看到类似上面的日志时，一个带有缓存功能的 APT 代理服务就启动完毕了。 41 | 42 | ## Ubuntu / Debian 支持 43 | 44 | 要在 `apt-get` 命令中使用代理，请在命令前添加代理设置： 45 | 46 | ```bash 47 | # 使用 apt-proxy 更新包列表 48 | http_proxy=http://your-domain-or-ip-address:3142 apt-get -o pkgProblemResolver=true -o Acquire::http=true update 49 | # 使用 apt-proxy 安装包 50 | http_proxy=http://your-domain-or-ip-address:3142 apt-get -o pkgProblemResolver=true -o Acquire::http=true install vim -y 51 | ``` 52 | 53 | 由于包被本地缓存，后续的包操作将会显著加快。 54 | 55 | ## CentOS 支持 56 | 57 | 对于 CentOS 7： 58 | 59 | ```bash 60 | cat /etc/yum.repos.d/CentOS-Base.repo | sed -e s/mirrorlist.*$// | sed -e s/#baseurl/baseurl/ | sed -e s#http://mirror.centos.org#http://your-domain-or-ip-address:3142# | tee /etc/yum.repos.d/CentOS-Base.repo 61 | ``` 62 | 63 | 对于 CentOS 8： 64 | 65 | ```bash 66 | sed -i -e"s#mirror.centos.org#http://your-domain-or-ip-address:3142#g" /etc/yum.repos.d/CentOS-* 67 | sed -i -e"s/#baseurl/baseurl/" /etc/yum.repos.d/CentOS-* 68 | sed -i -e"s#\$releasever/#8-stream/#" /etc/yum.repos.d/CentOS-* 69 | ``` 70 | 71 | 运行 `yum update` 验证配置。 72 | 73 | ## Alpine Linux 支持 74 | 75 | APT Proxy 也可以加速 Alpine Linux 的包下载： 76 | 77 | ```bash 78 | cat /etc/apk/repositories | sed -e s#https://.*.alpinelinux.org#http://your-domain-or-ip-address:3142# | tee /etc/apk/repositories 79 | ``` 80 | 81 | 运行 `apk update` 验证配置。 82 | 83 | ## 镜像配置 84 | 85 | 你可以通过两种方式指定镜像： 86 | 87 | 使用完整 URL： 88 | 89 | ```bash 90 | # 同时缓存 Ubuntu 和 Debian 包 91 | ./apt-proxy --ubuntu=https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ --debian=https://mirrors.tuna.tsinghua.edu.cn/debian/ 92 | # 仅缓存 Ubuntu 包 93 | ./apt-proxy --mode=ubuntu --ubuntu=https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ 94 | # 仅缓存 Debian 包 95 | ./apt-proxy --mode=debian --debian=https://mirrors.tuna.tsinghua.edu.cn/debian/ 96 | ``` 97 | 98 | 使用快捷方式： 99 | 100 | ```bash 101 | go run apt-proxy.go --ubuntu=cn:tsinghua --debian=cn:163 102 | 2022/06/15 10:55:26 running apt-proxy 103 | 2022/06/15 10:55:26 using specify debian mirror https://mirrors.163.com/debian/ 104 | 2022/06/15 10:55:26 using specify ubuntu mirror https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ 105 | 2022/06/15 10:55:26 proxy listening on 0.0.0.0:3142 106 | ``` 107 | 108 | 可用的快捷方式： 109 | 110 | - cn:tsinghua 111 | - cn:ustc 112 | - cn:163 113 | - cn:aliyun 114 | - cn:huaweicloud 115 | - cn:tencent 等等... 116 | 117 | ## Docker 集成 118 | 119 | 要加速 Docker 容器中的包安装： 120 | 121 | ```bash 122 | # 启动容器（Ubuntu 或 Debian） 123 | docker run --rm -it ubuntu 124 | # 或 125 | docker run --rm -it debian 126 | 127 | # 使用代理安装包 128 | http_proxy=http://host.docker.internal:3142 apt-get -o Debug::pkgProblemResolver=true -o Debug::Acquire::http=true update && \ 129 | http_proxy=http://host.docker.internal:3142 apt-get -o Debug::pkgProblemResolver=true -o Debug::Acquire::http=true install vim -y 130 | ``` 131 | 132 | ## Docker 部署 133 | 134 | 135 | 136 | 使用单个命令部署： 137 | 138 | ```bash 139 | docker run -d --name=apt-proxy -p 3142:3142 soulteary/apt-proxy 140 | ``` 141 | 142 | ## 配置选项 143 | 144 | 我们可以通过使用 `-h` 参数来查看程序支持的所有参数： 145 | 146 | ```bash 147 | ./apt-proxy -h 148 | 149 | 用法说明： 150 | -alpine string 151 | 用于获取包的 alpine 镜像 152 | -cachedir string 153 | 存储缓存数据的目录 (默认 "./.aptcache") 154 | -centos string 155 | 用于获取包的 centos 镜像 156 | -debian string 157 | 用于获取包的 debian 镜像 158 | -debug 159 | 是否输出调试日志 160 | -host string 161 | 绑定的主机地址 (默认 "0.0.0.0") 162 | -mode all 163 | 选择要缓存的系统模式：all / `ubuntu` / `ubuntu-ports` / `debian` / `centos` / `alpine` (默认 "all") 164 | -port string 165 | 绑定的端口 (默认 "3142") 166 | -ubuntu string 167 | 用于获取包的 ubuntu 镜像 168 | ``` 169 | 170 | ## 开发 171 | 172 | 运行测试： 173 | 174 | ```bash 175 | # 运行带覆盖率报告的测试 176 | go test -cover ./... 177 | 178 | # 生成并查看详细的覆盖率报告 179 | go test -coverprofile=coverage.out ./... 180 | go tool cover -html=coverage.out 181 | ``` 182 | 183 | 184 | ## 调试包操作 185 | 186 | 对于 Ubuntu/Debian： 187 | 188 | ```bash 189 | http_proxy=http://192.168.33.1:3142 apt-get -o Debug::pkgProblemResolver=true -o Debug::Acquire::http=true update 190 | http_proxy=http://192.168.33.1:3142 apt-get -o Debug::pkgProblemResolver=true -o Debug::Acquire::http=true install apache2 191 | ``` 192 | 193 | ## 开源协议 194 | 195 | 这个项目基于 [Apache License 2.0](https://github.com/soulteary/apt-proxy/blob/master/LICENSE)。 196 | 197 | ## 依赖组件 198 | 199 | - 未指定协议 200 | - [lox/apt-proxy](https://github.com/lox/apt-proxy#readme) 201 | - MIT License 202 | - [lox/httpcache](https://github.com/lox/httpcache/blob/master/LICENSE) 203 | - [djherbis/stream](https://github.com/djherbis/stream/blob/master/LICENSE) 204 | - Mozilla Public License 2.0 205 | - [rainycape/vfs](https://github.com/rainycape/vfs/blob/master/LICENSE) 206 | -------------------------------------------------------------------------------- /pkg/vfs/mem.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | pathpkg "path" 8 | "strings" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | var ( 14 | errNoEmptyNameFile = errors.New("can't create file with empty name") 15 | errNoEmptyNameDir = errors.New("can't create directory with empty name") 16 | ) 17 | 18 | type memoryFileSystem struct { 19 | mu sync.RWMutex 20 | root *Dir 21 | } 22 | 23 | // entry must always be called with the lock held 24 | func (fs *memoryFileSystem) entry(path string) (Entry, *Dir, int, error) { 25 | path = cleanPath(path) 26 | if path == "" || path == "/" || path == "." { 27 | return fs.root, nil, 0, nil 28 | } 29 | if path[0] == '/' { 30 | path = path[1:] 31 | } 32 | dir := fs.root 33 | for { 34 | p := strings.IndexByte(path, '/') 35 | name := path 36 | if p > 0 { 37 | name = path[:p] 38 | path = path[p+1:] 39 | } else { 40 | path = "" 41 | } 42 | dir.RLock() 43 | entry, pos, err := dir.Find(name) 44 | dir.RUnlock() 45 | if err != nil { 46 | return nil, nil, 0, err 47 | } 48 | if len(path) == 0 { 49 | return entry, dir, pos, nil 50 | } 51 | if entry.Type() != EntryTypeDir { 52 | break 53 | } 54 | dir = entry.(*Dir) 55 | } 56 | return nil, nil, 0, os.ErrNotExist 57 | } 58 | 59 | func (fs *memoryFileSystem) dirEntry(path string) (*Dir, error) { 60 | entry, _, _, err := fs.entry(path) 61 | if err != nil { 62 | return nil, err 63 | } 64 | if entry.Type() != EntryTypeDir { 65 | return nil, fmt.Errorf("%s it's not a directory", path) 66 | } 67 | return entry.(*Dir), nil 68 | } 69 | 70 | func (fs *memoryFileSystem) Open(path string) (RFile, error) { 71 | entry, _, _, err := fs.entry(path) 72 | if err != nil { 73 | return nil, err 74 | } 75 | if entry.Type() != EntryTypeFile { 76 | return nil, fmt.Errorf("%s is not a file", path) 77 | } 78 | return NewRFile(entry.(*File)) 79 | } 80 | 81 | func (fs *memoryFileSystem) OpenFile(path string, flag int, mode os.FileMode) (WFile, error) { 82 | if mode&os.ModeType != 0 { 83 | return nil, fmt.Errorf("%T does not support special files", fs) 84 | } 85 | path = cleanPath(path) 86 | dir, base := pathpkg.Split(path) 87 | if base == "" { 88 | return nil, errNoEmptyNameFile 89 | } 90 | fs.mu.RLock() 91 | d, err := fs.dirEntry(dir) 92 | fs.mu.RUnlock() 93 | if err != nil { 94 | return nil, err 95 | } 96 | 97 | d.Lock() 98 | defer d.Unlock() 99 | f, _, _ := d.Find(base) 100 | if f == nil && flag&os.O_CREATE == 0 { 101 | return nil, os.ErrNotExist 102 | } 103 | // Read only file? 104 | if flag&os.O_WRONLY == 0 && flag&os.O_RDWR == 0 { 105 | if f == nil { 106 | return nil, os.ErrNotExist 107 | } 108 | return NewWFile(f.(*File), true, false) 109 | } 110 | // Write file, either f != nil or flag&os.O_CREATE 111 | if f != nil { 112 | if f.Type() != EntryTypeFile { 113 | return nil, fmt.Errorf("%s is not a file", path) 114 | } 115 | if flag&os.O_EXCL != 0 { 116 | return nil, os.ErrExist 117 | } 118 | // Check if we should truncate 119 | if flag&os.O_TRUNC != 0 { 120 | file := f.(*File) 121 | file.Lock() 122 | file.ModTime = time.Now() 123 | file.Data = nil 124 | file.Unlock() 125 | } 126 | } else { 127 | f = &File{ModTime: time.Now()} 128 | err = d.Add(base, f) 129 | if err != nil { 130 | return nil, os.ErrExist 131 | } 132 | } 133 | return NewWFile(f.(*File), flag&os.O_RDWR != 0, true) 134 | } 135 | 136 | func (fs *memoryFileSystem) Lstat(path string) (os.FileInfo, error) { 137 | return fs.Stat(path) 138 | } 139 | 140 | func (fs *memoryFileSystem) Stat(path string) (os.FileInfo, error) { 141 | entry, _, _, err := fs.entry(path) 142 | if err != nil { 143 | return nil, err 144 | } 145 | return &EntryInfo{Path: path, Entry: entry}, nil 146 | } 147 | 148 | func (fs *memoryFileSystem) ReadDir(path string) ([]os.FileInfo, error) { 149 | fs.mu.RLock() 150 | defer fs.mu.RUnlock() 151 | return fs.readDir(path) 152 | } 153 | 154 | func (fs *memoryFileSystem) readDir(path string) ([]os.FileInfo, error) { 155 | entry, _, _, err := fs.entry(path) 156 | if err != nil { 157 | return nil, err 158 | } 159 | if entry.Type() != EntryTypeDir { 160 | return nil, fmt.Errorf("%s is not a directory", path) 161 | } 162 | dir := entry.(*Dir) 163 | dir.RLock() 164 | infos := make([]os.FileInfo, len(dir.Entries)) 165 | for ii, v := range dir.EntryNames { 166 | infos[ii] = &EntryInfo{ 167 | Path: pathpkg.Join(path, v), 168 | Entry: dir.Entries[ii], 169 | } 170 | } 171 | dir.RUnlock() 172 | return infos, nil 173 | } 174 | 175 | func (fs *memoryFileSystem) Mkdir(path string, perm os.FileMode) error { 176 | path = cleanPath(path) 177 | dir, base := pathpkg.Split(path) 178 | if base == "" { 179 | if dir == "/" || dir == "" { 180 | return os.ErrExist 181 | } 182 | return errNoEmptyNameDir 183 | } 184 | fs.mu.RLock() 185 | d, err := fs.dirEntry(dir) 186 | fs.mu.RUnlock() 187 | if err != nil { 188 | return err 189 | } 190 | d.Lock() 191 | defer d.Unlock() 192 | if _, p, _ := d.Find(base); p >= 0 { 193 | return os.ErrExist 194 | } 195 | err = d.Add(base, &Dir{ 196 | Mode: os.ModeDir | perm, 197 | ModTime: time.Now(), 198 | }) 199 | if err != nil { 200 | return os.ErrExist 201 | } 202 | return nil 203 | } 204 | 205 | func (fs *memoryFileSystem) Remove(path string) error { 206 | entry, dir, _, err := fs.entry(path) 207 | if err != nil { 208 | return err 209 | } 210 | if entry.Type() == EntryTypeDir && len(entry.(*Dir).Entries) > 0 { 211 | return fmt.Errorf("directory %s not empty", path) 212 | } 213 | // Lock again, the position might have changed 214 | dir.Lock() 215 | _, pos, err := dir.Find(pathpkg.Base(path)) 216 | if err == nil { 217 | dir.EntryNames = append(dir.EntryNames[:pos], dir.EntryNames[pos+1:]...) 218 | dir.Entries = append(dir.Entries[:pos], dir.Entries[pos+1:]...) 219 | } 220 | dir.Unlock() 221 | return err 222 | } 223 | 224 | func (fs *memoryFileSystem) String() string { 225 | return "MemoryFileSystem" 226 | } 227 | 228 | func newMemory() *memoryFileSystem { 229 | fs := &memoryFileSystem{ 230 | root: &Dir{ 231 | Mode: os.ModeDir | 0755, 232 | ModTime: time.Now(), 233 | }, 234 | } 235 | return fs 236 | } 237 | 238 | // Memory returns an empty in memory VFS. 239 | func Memory() VFS { 240 | return newMemory() 241 | } 242 | 243 | func cleanPath(path string) string { 244 | return strings.Trim(pathpkg.Clean("/"+path), "/") 245 | } 246 | -------------------------------------------------------------------------------- /pkg/httpcache/cache.go: -------------------------------------------------------------------------------- 1 | package httpcache 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "errors" 7 | "fmt" 8 | "hash/fnv" 9 | "io" 10 | "log" 11 | "net/http" 12 | "net/textproto" 13 | "os" 14 | pathutil "path" 15 | "strconv" 16 | "strings" 17 | "time" 18 | 19 | "github.com/soulteary/apt-proxy/pkg/vfs" 20 | ) 21 | 22 | const ( 23 | headerPrefix = "header/" 24 | bodyPrefix = "body/" 25 | formatPrefix = "v1/" 26 | ) 27 | 28 | // Returned when a resource doesn't exist 29 | var ErrNotFoundInCache = errors.New("not found in cache") 30 | 31 | type Cache interface { 32 | Header(key string) (Header, error) 33 | Store(res *Resource, keys ...string) error 34 | Retrieve(key string) (*Resource, error) 35 | Invalidate(keys ...string) 36 | Freshen(res *Resource, keys ...string) error 37 | } 38 | 39 | // cache provides a storage mechanism for cached Resources 40 | type cache struct { 41 | fs vfs.VFS 42 | stale map[string]time.Time 43 | } 44 | 45 | var _ Cache = (*cache)(nil) 46 | 47 | type Header struct { 48 | http.Header 49 | StatusCode int 50 | } 51 | 52 | // NewCache returns a cache backend off the provided VFS 53 | func NewVFSCache(fs vfs.VFS) Cache { 54 | return &cache{fs: fs, stale: map[string]time.Time{}} 55 | } 56 | 57 | // NewMemoryCache returns an ephemeral cache in memory 58 | func NewMemoryCache() Cache { 59 | return NewVFSCache(vfs.Memory()) 60 | } 61 | 62 | // NewDiskCache returns a disk-backed cache 63 | func NewDiskCache(dir string) (Cache, error) { 64 | if err := os.MkdirAll(dir, 0750); err != nil { 65 | return nil, err 66 | } 67 | fs, err := vfs.FS(dir) 68 | if err != nil { 69 | return nil, err 70 | } 71 | chfs, err := vfs.Chroot("/", fs) 72 | if err != nil { 73 | return nil, err 74 | } 75 | return NewVFSCache(chfs), nil 76 | } 77 | 78 | func (c *cache) vfsWrite(path string, r io.Reader) error { 79 | if err := vfs.MkdirAll(c.fs, pathutil.Dir(path), 0700); err != nil { 80 | return err 81 | } 82 | f, err := c.fs.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) 83 | if err != nil { 84 | return err 85 | } 86 | defer f.Close() 87 | if _, err := io.Copy(f, r); err != nil { 88 | return err 89 | } 90 | return nil 91 | } 92 | 93 | // Retrieve the Status and Headers for a given key path 94 | func (c *cache) Header(key string) (Header, error) { 95 | path := headerPrefix + formatPrefix + hashKey(key) 96 | f, err := c.fs.Open(path) 97 | if err != nil { 98 | if vfs.IsNotExist(err) { 99 | return Header{}, ErrNotFoundInCache 100 | } 101 | return Header{}, err 102 | } 103 | 104 | return readHeaders(bufio.NewReader(f)) 105 | } 106 | 107 | // Store a resource against a number of keys 108 | func (c *cache) Store(res *Resource, keys ...string) error { 109 | var buf = &bytes.Buffer{} 110 | 111 | if length, err := strconv.ParseInt(res.Header().Get("Content-Length"), 10, 64); err == nil { 112 | if _, err = io.CopyN(buf, res, length); err != nil { 113 | return err 114 | } 115 | } else if _, err = io.Copy(buf, res); err != nil { 116 | return err 117 | } 118 | 119 | for _, key := range keys { 120 | delete(c.stale, key) 121 | 122 | if err := c.storeBody(buf, key); err != nil { 123 | return err 124 | } 125 | 126 | if err := c.storeHeader(res.Status(), res.Header(), key); err != nil { 127 | return err 128 | } 129 | } 130 | 131 | return nil 132 | } 133 | 134 | func (c *cache) storeBody(r io.Reader, key string) error { 135 | if err := c.vfsWrite(bodyPrefix+formatPrefix+hashKey(key), r); err != nil { 136 | return err 137 | } 138 | return nil 139 | } 140 | 141 | func (c *cache) storeHeader(code int, h http.Header, key string) error { 142 | hb := &bytes.Buffer{} 143 | hb.Write([]byte(fmt.Sprintf("HTTP/1.1 %d %s\r\n", code, http.StatusText(code)))) 144 | if err := headersToWriter(h, hb); err != nil { 145 | return err 146 | } 147 | if err := c.vfsWrite(headerPrefix+formatPrefix+hashKey(key), bytes.NewReader(hb.Bytes())); err != nil { 148 | return err 149 | } 150 | return nil 151 | } 152 | 153 | // Retrieve returns a cached Resource for the given key 154 | func (c *cache) Retrieve(key string) (*Resource, error) { 155 | f, err := c.fs.Open(bodyPrefix + formatPrefix + hashKey(key)) 156 | if err != nil { 157 | if vfs.IsNotExist(err) { 158 | return nil, ErrNotFoundInCache 159 | } 160 | return nil, err 161 | } 162 | h, err := c.Header(key) 163 | if err != nil { 164 | if vfs.IsNotExist(err) { 165 | return nil, ErrNotFoundInCache 166 | } 167 | return nil, err 168 | } 169 | res := NewResource(h.StatusCode, f, h.Header) 170 | if staleTime, exists := c.stale[key]; exists { 171 | if !res.DateAfter(staleTime) { 172 | log.Printf("stale marker of %s found", staleTime) 173 | res.MarkStale() 174 | } 175 | } 176 | return res, nil 177 | } 178 | 179 | func (c *cache) Invalidate(keys ...string) { 180 | log.Printf("invalidating %q", keys) 181 | for _, key := range keys { 182 | c.stale[key] = Clock() 183 | } 184 | } 185 | 186 | func (c *cache) Freshen(res *Resource, keys ...string) error { 187 | for _, key := range keys { 188 | if h, err := c.Header(key); err == nil { 189 | if h.StatusCode == res.Status() && headersEqual(h.Header, res.Header()) { 190 | debugf("freshening key %s", key) 191 | if err := c.storeHeader(h.StatusCode, res.Header(), key); err != nil { 192 | return err 193 | } 194 | } else { 195 | debugf("freshen failed, invalidating %s", key) 196 | c.Invalidate(key) 197 | } 198 | } 199 | } 200 | return nil 201 | } 202 | 203 | func hashKey(key string) string { 204 | h := fnv.New64a() 205 | _, err := h.Write([]byte(key)) 206 | if err != nil { 207 | return "unable-to-calculate" 208 | } 209 | return fmt.Sprintf("%x", h.Sum(nil)) 210 | } 211 | 212 | func readHeaders(r *bufio.Reader) (Header, error) { 213 | tp := textproto.NewReader(r) 214 | line, err := tp.ReadLine() 215 | if err != nil { 216 | return Header{}, err 217 | } 218 | 219 | f := strings.SplitN(line, " ", 3) 220 | if len(f) < 2 { 221 | return Header{}, fmt.Errorf("malformed HTTP response: %s", line) 222 | } 223 | statusCode, err := strconv.Atoi(f[1]) 224 | if err != nil { 225 | return Header{}, fmt.Errorf("malformed HTTP status code: %s", f[1]) 226 | } 227 | 228 | mimeHeader, err := tp.ReadMIMEHeader() 229 | if err != nil { 230 | return Header{}, err 231 | } 232 | return Header{StatusCode: statusCode, Header: http.Header(mimeHeader)}, nil 233 | } 234 | 235 | func headersToWriter(h http.Header, w io.Writer) error { 236 | if err := h.Write(w); err != nil { 237 | return err 238 | } 239 | // ReadMIMEHeader expects a trailing newline 240 | _, err := w.Write([]byte("\r\n")) 241 | return err 242 | } 243 | -------------------------------------------------------------------------------- /example/assets/logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkg/vfs/util.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "os" 8 | pathpkg "path" 9 | "strings" 10 | ) 11 | 12 | var ( 13 | // SkipDir is used by a WalkFunc to signal Walk that 14 | // it wans to skip the given directory. 15 | SkipDir = errors.New("skip this directory") 16 | // ErrReadOnly is returned from Write() on a read-only file. 17 | ErrReadOnly = errors.New("can't write to read only file") 18 | // ErrWriteOnly is returned from Read() on a write-only file. 19 | ErrWriteOnly = errors.New("can't read from write only file") 20 | ) 21 | 22 | // WalkFunc is the function type used by Walk to iterate over a VFS. 23 | type WalkFunc func(fs VFS, path string, info os.FileInfo, err error) error 24 | 25 | func walk(fs VFS, p string, info os.FileInfo, fn WalkFunc) error { 26 | err := fn(fs, p, info, nil) 27 | if err != nil { 28 | if info.IsDir() && err == SkipDir { 29 | err = nil 30 | } 31 | return err 32 | } 33 | if !info.IsDir() { 34 | return nil 35 | } 36 | infos, err := fs.ReadDir(p) 37 | if err != nil { 38 | return fn(fs, p, info, err) 39 | } 40 | for _, v := range infos { 41 | name := pathpkg.Join(p, v.Name()) 42 | fileInfo, err := fs.Lstat(name) 43 | if err != nil { 44 | if err := fn(fs, name, fileInfo, err); err != nil && err != SkipDir { 45 | return err 46 | } 47 | continue 48 | } 49 | if err := walk(fs, name, fileInfo, fn); err != nil && (!fileInfo.IsDir() || err != SkipDir) { 50 | return err 51 | } 52 | } 53 | return nil 54 | } 55 | 56 | // Walk iterates over all the files in the VFS which descend from the given 57 | // root (including root itself), descending into any subdirectories. In each 58 | // directory, files are visited in alphabetical order. The given function might 59 | // chose to skip a directory by returning SkipDir. 60 | func Walk(fs VFS, root string, fn WalkFunc) error { 61 | info, err := fs.Lstat(root) 62 | if err != nil { 63 | return fn(fs, root, nil, err) 64 | } 65 | return walk(fs, root, info, fn) 66 | } 67 | 68 | func makeDir(fs VFS, path string, perm os.FileMode) error { 69 | stat, err := fs.Lstat(path) 70 | if err == nil { 71 | if !stat.IsDir() { 72 | return fmt.Errorf("%s exists and is not a directory", path) 73 | } 74 | } else { 75 | if err := fs.Mkdir(path, perm); err != nil { 76 | return err 77 | } 78 | } 79 | return nil 80 | } 81 | 82 | // MkdirAll makes all directories pointed by the given path, using the same 83 | // permissions for all of them. Note that MkdirAll skips directories which 84 | // already exists rather than returning an error. 85 | func MkdirAll(fs VFS, path string, perm os.FileMode) error { 86 | cur := "/" 87 | if err := makeDir(fs, cur, perm); err != nil { 88 | return err 89 | } 90 | parts := strings.Split(path, "/") 91 | for _, v := range parts { 92 | cur += v 93 | if err := makeDir(fs, cur, perm); err != nil { 94 | return err 95 | } 96 | cur += "/" 97 | } 98 | return nil 99 | } 100 | 101 | // RemoveAll removes all files from the given fs and path, including 102 | // directories (by removing its contents first). 103 | func RemoveAll(fs VFS, path string) error { 104 | stat, err := fs.Lstat(path) 105 | if err != nil { 106 | if err == os.ErrNotExist { 107 | return nil 108 | } 109 | return err 110 | } 111 | if stat.IsDir() { 112 | files, err := fs.ReadDir(path) 113 | if err != nil { 114 | return err 115 | } 116 | for _, v := range files { 117 | filePath := pathpkg.Join(path, v.Name()) 118 | if err := RemoveAll(fs, filePath); err != nil { 119 | return err 120 | } 121 | } 122 | } 123 | return fs.Remove(path) 124 | } 125 | 126 | // ReadFile reads the file at the given path from the given fs, returning 127 | // either its contents or an error if the file couldn't be read. 128 | func ReadFile(fs VFS, path string) ([]byte, error) { 129 | f, err := fs.Open(path) 130 | if err != nil { 131 | return nil, err 132 | } 133 | defer f.Close() 134 | return io.ReadAll(f) 135 | } 136 | 137 | // WriteFile writes a file at the given path and fs with the given data and 138 | // permissions. If the file already exists, WriteFile truncates it before 139 | // writing. If the file can't be created, an error will be returned. 140 | func WriteFile(fs VFS, path string, data []byte, perm os.FileMode) error { 141 | f, err := fs.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm) 142 | if err != nil { 143 | return err 144 | } 145 | _, err = f.Write(data) 146 | if err != nil { 147 | errClose := f.Close() 148 | if errClose != nil { 149 | return err 150 | } 151 | return err 152 | } 153 | return f.Close() 154 | } 155 | 156 | // Clone copies all the files from the src VFS to dst. Note that files or directories with 157 | // all permissions set to 0 will be set to 0755 for directories and 0644 for files. If you 158 | // need more granularity, use Walk directly to clone the file systems. 159 | func Clone(dst VFS, src VFS) error { 160 | err := Walk(src, "/", func(fs VFS, path string, info os.FileInfo, err error) error { 161 | if err != nil { 162 | return err 163 | } 164 | if info.IsDir() { 165 | perm := info.Mode() & os.ModePerm 166 | if perm == 0 { 167 | perm = 0755 168 | } 169 | err := dst.Mkdir(path, info.Mode()|perm) 170 | if err != nil && !IsExist(err) { 171 | return err 172 | } 173 | return nil 174 | } 175 | data, err := ReadFile(fs, path) 176 | if err != nil { 177 | return err 178 | } 179 | perm := info.Mode() & os.ModePerm 180 | if perm == 0 { 181 | perm = 0644 182 | } 183 | if err := WriteFile(dst, path, data, info.Mode()|perm); err != nil { 184 | return err 185 | } 186 | return nil 187 | }) 188 | return err 189 | } 190 | 191 | // IsExist returns wheter the error indicates that the file or directory 192 | // already exists. 193 | func IsExist(err error) bool { 194 | return os.IsExist(err) 195 | } 196 | 197 | // IsExist returns wheter the error indicates that the file or directory 198 | // does not exist. 199 | func IsNotExist(err error) bool { 200 | return os.IsNotExist(err) 201 | } 202 | 203 | // Compressor is the interface implemented by VFS files which can be 204 | // transparently compressed and decompressed. Currently, this is only 205 | // supported by the in-memory filesystems. 206 | type Compressor interface { 207 | IsCompressed() bool 208 | SetCompressed(c bool) 209 | } 210 | 211 | // Compress is a shorthand method for compressing all the files in a VFS. 212 | // Note that not all file systems support transparent compression/decompression. 213 | func Compress(fs VFS) error { 214 | return Walk(fs, "/", func(fs VFS, p string, info os.FileInfo, err error) error { 215 | if err != nil { 216 | return err 217 | } 218 | mode := info.Mode() 219 | if mode.IsDir() || mode&ModeCompress != 0 { 220 | return nil 221 | } 222 | f, err := fs.Open(p) 223 | if err != nil { 224 | return err 225 | } 226 | if c, ok := f.(Compressor); ok { 227 | c.SetCompressed(true) 228 | } 229 | return f.Close() 230 | }) 231 | } 232 | -------------------------------------------------------------------------------- /internal/rewriter/rewriter.go: -------------------------------------------------------------------------------- 1 | package rewriter 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "net/url" 7 | "regexp" 8 | "strings" 9 | "sync" 10 | 11 | "github.com/soulteary/apt-proxy/define" 12 | "github.com/soulteary/apt-proxy/internal/benchmarks" 13 | "github.com/soulteary/apt-proxy/internal/mirrors" 14 | "github.com/soulteary/apt-proxy/state" 15 | ) 16 | 17 | // URLRewriter holds the mirror and pattern for URL rewriting 18 | type URLRewriter struct { 19 | mirror *url.URL 20 | pattern *regexp.Regexp 21 | } 22 | 23 | // URLRewriters manages rewriters for different distributions 24 | type URLRewriters struct { 25 | Ubuntu *URLRewriter 26 | UbuntuPorts *URLRewriter 27 | Debian *URLRewriter 28 | Centos *URLRewriter 29 | Alpine *URLRewriter 30 | Mu sync.RWMutex 31 | } 32 | 33 | // getRewriterConfig returns configuration for a specific distribution 34 | func getRewriterConfig(mode int) (getMirror func() *url.URL, name string) { 35 | switch mode { 36 | case define.TYPE_LINUX_DISTROS_UBUNTU: 37 | return state.GetUbuntuMirror, "Ubuntu" 38 | case define.TYPE_LINUX_DISTROS_UBUNTU_PORTS: 39 | return state.GetUbuntuPortsMirror, "Ubuntu Ports" 40 | case define.TYPE_LINUX_DISTROS_DEBIAN: 41 | return state.GetDebianMirror, "Debian" 42 | case define.TYPE_LINUX_DISTROS_CENTOS: 43 | return state.GetCentOSMirror, "CentOS" 44 | case define.TYPE_LINUX_DISTROS_ALPINE: 45 | return state.GetAlpineMirror, "Alpine" 46 | default: 47 | return nil, "" 48 | } 49 | } 50 | 51 | // createRewriter creates a new URLRewriter for a specific distribution 52 | func createRewriter(mode int) *URLRewriter { 53 | getMirror, name := getRewriterConfig(mode) 54 | if getMirror == nil { 55 | return nil 56 | } 57 | 58 | benchmarkURL, pattern := mirrors.GetPredefinedConfiguration(mode) 59 | rewriter := &URLRewriter{pattern: pattern} 60 | mirror := getMirror() 61 | 62 | if mirror != nil { 63 | log.Printf("using specified [%s] mirror [%s]", name, mirror) 64 | rewriter.mirror = mirror 65 | return rewriter 66 | } 67 | 68 | mirrorURLs := mirrors.GetGeoMirrorUrlsByMode(mode) 69 | fastest, err := benchmarks.GetTheFastestMirror(mirrorURLs, benchmarkURL) 70 | if err != nil { 71 | log.Printf("Error finding fastest [%s] mirror: %v", name, err) 72 | return rewriter 73 | } 74 | 75 | if mirror, err := url.Parse(fastest); err == nil { 76 | log.Printf("using fastest [%s] mirror [%s]", name, fastest) 77 | rewriter.mirror = mirror 78 | } 79 | 80 | return rewriter 81 | } 82 | 83 | // CreateNewRewriters initializes rewriters based on mode 84 | func CreateNewRewriters(mode int) *URLRewriters { 85 | rewriters := &URLRewriters{} 86 | 87 | switch mode { 88 | case define.TYPE_LINUX_DISTROS_UBUNTU: 89 | rewriters.Ubuntu = createRewriter(mode) 90 | case define.TYPE_LINUX_DISTROS_UBUNTU_PORTS: 91 | rewriters.UbuntuPorts = createRewriter(mode) 92 | case define.TYPE_LINUX_DISTROS_DEBIAN: 93 | rewriters.Debian = createRewriter(mode) 94 | case define.TYPE_LINUX_DISTROS_CENTOS: 95 | rewriters.Centos = createRewriter(mode) 96 | case define.TYPE_LINUX_DISTROS_ALPINE: 97 | rewriters.Alpine = createRewriter(mode) 98 | default: 99 | rewriters.Ubuntu = createRewriter(define.TYPE_LINUX_DISTROS_UBUNTU) 100 | rewriters.UbuntuPorts = createRewriter(define.TYPE_LINUX_DISTROS_UBUNTU_PORTS) 101 | rewriters.Debian = createRewriter(define.TYPE_LINUX_DISTROS_DEBIAN) 102 | rewriters.Centos = createRewriter(define.TYPE_LINUX_DISTROS_CENTOS) 103 | rewriters.Alpine = createRewriter(define.TYPE_LINUX_DISTROS_ALPINE) 104 | } 105 | 106 | return rewriters 107 | } 108 | 109 | // GetRewriteRulesByMode returns caching rules for a specific mode 110 | func GetRewriteRulesByMode(mode int) []define.Rule { 111 | switch mode { 112 | case define.TYPE_LINUX_DISTROS_UBUNTU: 113 | return define.UBUNTU_DEFAULT_CACHE_RULES 114 | case define.TYPE_LINUX_DISTROS_UBUNTU_PORTS: 115 | return define.UBUNTU_PORTS_DEFAULT_CACHE_RULES 116 | case define.TYPE_LINUX_DISTROS_DEBIAN: 117 | return define.DEBIAN_DEFAULT_CACHE_RULES 118 | case define.TYPE_LINUX_DISTROS_CENTOS: 119 | return define.CENTOS_DEFAULT_CACHE_RULES 120 | case define.TYPE_LINUX_DISTROS_ALPINE: 121 | return define.ALPINE_DEFAULT_CACHE_RULES 122 | default: 123 | rules := make([]define.Rule, 0) 124 | rules = append(rules, define.UBUNTU_DEFAULT_CACHE_RULES...) 125 | rules = append(rules, define.UBUNTU_PORTS_DEFAULT_CACHE_RULES...) 126 | rules = append(rules, define.DEBIAN_DEFAULT_CACHE_RULES...) 127 | rules = append(rules, define.CENTOS_DEFAULT_CACHE_RULES...) 128 | rules = append(rules, define.ALPINE_DEFAULT_CACHE_RULES...) 129 | return rules 130 | } 131 | } 132 | 133 | // RewriteRequestByMode rewrites the request URL to point to the configured mirror 134 | // for the specified distribution mode. It matches the request path against 135 | // distribution-specific patterns and replaces the URL scheme, host, and path 136 | // with the mirror's configuration. If rewriters is nil, the function returns early. 137 | func RewriteRequestByMode(r *http.Request, rewriters *URLRewriters, mode int) { 138 | if rewriters == nil { 139 | return 140 | } 141 | rewriters.Mu.RLock() 142 | defer rewriters.Mu.RUnlock() 143 | 144 | rewriter := &URLRewriter{} 145 | switch mode { 146 | case define.TYPE_LINUX_DISTROS_UBUNTU: 147 | rewriter = rewriters.Ubuntu 148 | case define.TYPE_LINUX_DISTROS_UBUNTU_PORTS: 149 | rewriter = rewriters.UbuntuPorts 150 | case define.TYPE_LINUX_DISTROS_DEBIAN: 151 | rewriter = rewriters.Debian 152 | case define.TYPE_LINUX_DISTROS_CENTOS: 153 | rewriter = rewriters.Centos 154 | case define.TYPE_LINUX_DISTROS_ALPINE: 155 | rewriter = rewriters.Alpine 156 | } 157 | 158 | if rewriter == nil || rewriter.mirror == nil || rewriter.pattern == nil { 159 | return 160 | } 161 | 162 | uri := r.URL.String() 163 | if !rewriter.pattern.MatchString(uri) { 164 | return 165 | } 166 | 167 | r.Header.Add("Content-Location", uri) 168 | matches := rewriter.pattern.FindStringSubmatch(uri) 169 | if len(matches) == 0 { 170 | return 171 | } 172 | 173 | queryRaw := matches[len(matches)-1] 174 | unescapedQuery, err := url.PathUnescape(queryRaw) 175 | if err != nil { 176 | unescapedQuery = queryRaw 177 | } 178 | 179 | r.URL.Scheme = rewriter.mirror.Scheme 180 | r.URL.Host = rewriter.mirror.Host 181 | if mode == define.TYPE_LINUX_DISTROS_DEBIAN { 182 | slugs_query := strings.Split(r.URL.Path, "/") 183 | slugs_mirror := strings.Split(rewriter.mirror.Path, "/") 184 | slugs_mirror[0] = slugs_query[0] 185 | r.URL.Path = strings.Join(slugs_query, "/") 186 | return 187 | } 188 | // Use templates for path construction 189 | path, err := mirrors.BuildPathWithQuery(rewriter.mirror.Path, unescapedQuery) 190 | if err != nil { 191 | // Fallback to concatenation if template fails 192 | r.URL.Path = rewriter.mirror.Path + unescapedQuery 193 | } else { 194 | r.URL.Path = path 195 | } 196 | } 197 | 198 | // MatchingRule finds a matching rule for the given path 199 | func MatchingRule(path string, rules []define.Rule) (*define.Rule, bool) { 200 | for _, rule := range rules { 201 | if rule.Pattern.MatchString(path) { 202 | return &rule, true 203 | } 204 | } 205 | return nil, false 206 | } 207 | -------------------------------------------------------------------------------- /internal/server/proxy.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "net/http" 7 | "net/http/httputil" 8 | "regexp" 9 | "time" 10 | 11 | define "github.com/soulteary/apt-proxy/define" 12 | rewriter "github.com/soulteary/apt-proxy/internal/rewriter" 13 | state "github.com/soulteary/apt-proxy/state" 14 | ) 15 | 16 | var hostPatternMap = map[*regexp.Regexp][]define.Rule{ 17 | define.UBUNTU_HOST_PATTERN: define.UBUNTU_DEFAULT_CACHE_RULES, 18 | define.UBUNTU_PORTS_HOST_PATTERN: define.UBUNTU_PORTS_DEFAULT_CACHE_RULES, 19 | define.DEBIAN_HOST_PATTERN: define.DEBIAN_DEFAULT_CACHE_RULES, 20 | define.CENTOS_HOST_PATTERN: define.CENTOS_DEFAULT_CACHE_RULES, 21 | define.ALPINE_HOST_PATTERN: define.ALPINE_DEFAULT_CACHE_RULES, 22 | } 23 | 24 | var ( 25 | rewriters *rewriter.URLRewriters 26 | defaultTransport = &http.Transport{ 27 | Proxy: http.ProxyFromEnvironment, 28 | ResponseHeaderTimeout: 45 * time.Second, 29 | DisableKeepAlives: true, 30 | MaxIdleConns: 100, 31 | IdleConnTimeout: 90 * time.Second, 32 | DisableCompression: false, 33 | } 34 | ) 35 | 36 | // PackageStruct is the main HTTP handler that routes requests to appropriate 37 | // distribution-specific handlers and applies caching rules. 38 | type PackageStruct struct { 39 | Handler http.Handler // The underlying HTTP handler (typically a reverse proxy) 40 | Rules []define.Rule // Caching rules for different package types 41 | CacheDir string // Cache directory path for statistics 42 | } 43 | 44 | // responseWriter wraps http.ResponseWriter to inject cache control headers 45 | // based on the matched caching rule. 46 | type responseWriter struct { 47 | http.ResponseWriter 48 | rule *define.Rule // The matched caching rule for this request 49 | } 50 | 51 | // CreatePackageStructRouter initializes and returns a new PackageStruct instance 52 | // configured for the current proxy mode. It sets up URL rewriters and caching rules. 53 | func CreatePackageStructRouter(cacheDir string) *PackageStruct { 54 | mode := state.GetProxyMode() 55 | rewriters = rewriter.CreateNewRewriters(mode) 56 | 57 | return &PackageStruct{ 58 | Rules: rewriter.GetRewriteRulesByMode(mode), 59 | CacheDir: cacheDir, 60 | Handler: &httputil.ReverseProxy{ 61 | Director: func(r *http.Request) {}, 62 | Transport: defaultTransport, 63 | }, 64 | } 65 | } 66 | 67 | // CreateRouter creates an http.Handler with orthodox routing using http.ServeMux. 68 | // It registers home and ping handlers, and uses the provided handler for package requests. 69 | func CreateRouter(handler http.Handler, cacheDir string) http.Handler { 70 | // Use http.ServeMux for orthodox routing 71 | mux := http.NewServeMux() 72 | 73 | // Register ping endpoint handler first (most specific route) 74 | mux.HandleFunc("/_/ping/", func(rw http.ResponseWriter, r *http.Request) { 75 | handlePing(rw, r) 76 | }) 77 | 78 | // Register home page handler for exact "/" path 79 | mux.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) { 80 | // Only handle exact "/" path, delegate everything else to package proxy 81 | if r.URL.Path != "/" { 82 | handler.ServeHTTP(rw, r) 83 | return 84 | } 85 | handleHomePage(rw, r, cacheDir) 86 | }) 87 | 88 | return mux 89 | } 90 | 91 | // handleHomePage serves the home page with statistics 92 | func handleHomePage(rw http.ResponseWriter, r *http.Request, cacheDir string) { 93 | tpl, status := RenderInternalUrls("/", cacheDir) 94 | rw.WriteHeader(status) 95 | rw.Header().Set("Content-Type", "text/html; charset=utf-8") 96 | if _, err := io.WriteString(rw, tpl); err != nil { 97 | log.Printf("Error rendering home page: %v", err) 98 | } 99 | } 100 | 101 | // handlePing serves the ping endpoint 102 | func handlePing(rw http.ResponseWriter, r *http.Request) { 103 | rw.WriteHeader(http.StatusOK) 104 | rw.Header().Set("Content-Type", "text/plain; charset=utf-8") 105 | if _, err := io.WriteString(rw, "pong"); err != nil { 106 | log.Printf("Error writing ping response: %v", err) 107 | } 108 | } 109 | 110 | // ServeHTTP implements http.Handler interface. It processes incoming requests, 111 | // matches them against caching rules, and routes them to the appropriate handler. 112 | // If a matching rule is found, the request is processed with cache control headers. 113 | func (ap *PackageStruct) ServeHTTP(rw http.ResponseWriter, r *http.Request) { 114 | rule := ap.handleExternalURLs(r) 115 | if rule != nil { 116 | if ap.Handler != nil { 117 | ap.Handler.ServeHTTP(&responseWriter{rw, rule}, r) 118 | } else { 119 | http.Error(rw, "Internal Server Error: handler not initialized", http.StatusInternalServerError) 120 | } 121 | } else { 122 | http.NotFound(rw, r) 123 | } 124 | } 125 | 126 | // handleExternalURLs processes requests for external package repositories. 127 | // It matches the request path against known distribution patterns and returns 128 | // the appropriate caching rule if a match is found. 129 | func (ap *PackageStruct) handleExternalURLs(r *http.Request) *define.Rule { 130 | path := r.URL.Path 131 | for pattern, rules := range hostPatternMap { 132 | if pattern.MatchString(path) { 133 | return ap.processMatchingRule(r, rules) 134 | } 135 | } 136 | return nil 137 | } 138 | 139 | // processMatchingRule processes a request that matches a distribution pattern. 140 | // It finds the specific caching rule, removes client cache control headers, 141 | // and rewrites the URL if necessary. 142 | func (ap *PackageStruct) processMatchingRule(r *http.Request, rules []define.Rule) *define.Rule { 143 | rule, match := rewriter.MatchingRule(r.URL.Path, rules) 144 | if !match { 145 | return nil 146 | } 147 | 148 | r.Header.Del("Cache-Control") 149 | if rule.Rewrite { 150 | ap.rewriteRequest(r, rule) 151 | } 152 | return rule 153 | } 154 | 155 | // rewriteRequest rewrites the request URL to point to the configured mirror 156 | // for the distribution. This enables transparent proxying to different mirrors 157 | // while maintaining the original request path structure. 158 | func (ap *PackageStruct) rewriteRequest(r *http.Request, rule *define.Rule) { 159 | if r.URL == nil { 160 | log.Printf("Error: request URL is nil, cannot rewrite") 161 | return 162 | } 163 | before := r.URL.String() 164 | rewriter.RewriteRequestByMode(r, rewriters, rule.OS) 165 | 166 | if r.URL != nil { 167 | r.Host = r.URL.Host 168 | log.Printf("Rewrote %q to %q", before, r.URL.String()) 169 | } 170 | } 171 | 172 | // WriteHeader implements http.ResponseWriter interface. It injects cache control 173 | // headers based on the matched rule before writing the status code. 174 | func (rw *responseWriter) WriteHeader(status int) { 175 | if rw.shouldSetCacheControl(status) { 176 | rw.Header().Set("Cache-Control", rw.rule.CacheControl) 177 | } 178 | rw.ResponseWriter.WriteHeader(status) 179 | } 180 | 181 | // shouldSetCacheControl determines whether cache control headers should be set 182 | // for the given HTTP status code. Only certain status codes are cacheable. 183 | func (rw *responseWriter) shouldSetCacheControl(status int) bool { 184 | return rw.rule != nil && 185 | rw.rule.CacheControl != "" && 186 | (status == http.StatusOK || status == http.StatusNotFound) 187 | } 188 | -------------------------------------------------------------------------------- /pkg/vfs/vfs_test.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "crypto/sha1" 5 | "encoding/hex" 6 | "fmt" 7 | "io" 8 | "os" 9 | "path/filepath" 10 | "reflect" 11 | "testing" 12 | ) 13 | 14 | const ( 15 | goTestFile = "go1.3.src.tar.gz" 16 | ) 17 | 18 | type errNoTestFile string 19 | 20 | func (e errNoTestFile) Error() string { 21 | return fmt.Sprintf("%s test file not found, use testdata/download-data.sh to fetch it", filepath.Base(string(e))) 22 | } 23 | 24 | func openOptionalTestFile(t testing.TB, name string) *os.File { 25 | filename := filepath.Join("testdata", name) 26 | f, err := os.Open(filename) 27 | if err != nil { 28 | t.Skip(errNoTestFile(filename)) 29 | } 30 | return f 31 | } 32 | 33 | func testVFS(t *testing.T, fs VFS) { 34 | if err := WriteFile(fs, "a", []byte("A"), 0644); err != nil { 35 | t.Fatal(err) 36 | } 37 | data, err := ReadFile(fs, "a") 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | if string(data) != "A" { 42 | t.Errorf("expecting file a to contain \"A\" got %q instead", string(data)) 43 | } 44 | if err := WriteFile(fs, "b", []byte("B"), 0755); err != nil { 45 | t.Fatal(err) 46 | } 47 | if _, err := fs.OpenFile("b", os.O_CREATE|os.O_TRUNC|os.O_EXCL|os.O_WRONLY, 0755); err == nil || !IsExist(err) { 48 | t.Errorf("error should be ErrExist, it's %v", err) 49 | } 50 | fb, err := fs.OpenFile("b", os.O_TRUNC|os.O_WRONLY, 0755) 51 | if err != nil { 52 | t.Fatalf("error opening b: %s", err) 53 | } 54 | if _, err := fb.Write([]byte("BB")); err != nil { 55 | t.Errorf("error writing to b: %s", err) 56 | } 57 | if _, err := fb.Seek(0, io.SeekStart); err != nil { 58 | t.Errorf("error seeking b: %s", err) 59 | } 60 | if _, err := fb.Read(make([]byte, 2)); err == nil { 61 | t.Error("allowed reading WRONLY file b") 62 | } 63 | if err := fb.Close(); err != nil { 64 | t.Errorf("error closing b: %s", err) 65 | } 66 | files, err := fs.ReadDir("/") 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | if len(files) != 2 { 71 | t.Errorf("expecting 2 files, got %d", len(files)) 72 | } 73 | if n := files[0].Name(); n != "a" { 74 | t.Errorf("expecting first file named \"a\", got %q", n) 75 | } 76 | if n := files[1].Name(); n != "b" { 77 | t.Errorf("expecting first file named \"b\", got %q", n) 78 | } 79 | for ii, v := range files { 80 | es := int64(ii + 1) 81 | if s := v.Size(); es != s { 82 | t.Errorf("expecting file %s to have size %d, has %d", v.Name(), es, s) 83 | } 84 | } 85 | if err := MkdirAll(fs, "a/b/c/d", 0); err == nil { 86 | t.Error("should not allow dir over file") 87 | } 88 | if err := MkdirAll(fs, "c/d", 0755); err != nil { 89 | t.Fatal(err) 90 | } 91 | // Idempotent 92 | if err := MkdirAll(fs, "c/d", 0755); err != nil { 93 | t.Fatal(err) 94 | } 95 | if err := fs.Mkdir("c", 0755); err == nil || !IsExist(err) { 96 | t.Errorf("err should be ErrExist, it's %v", err) 97 | } 98 | // Should fail to remove, c is not empty 99 | if err := fs.Remove("c"); err == nil { 100 | t.Fatalf("removed non-empty directory") 101 | } 102 | var walked []os.FileInfo 103 | var walkedNames []string 104 | err = Walk(fs, "c", func(fs VFS, path string, info os.FileInfo, err error) error { 105 | if err != nil { 106 | return err 107 | } 108 | walked = append(walked, info) 109 | walkedNames = append(walkedNames, path) 110 | return nil 111 | }) 112 | if err != nil { 113 | t.Fatal(err) 114 | } 115 | if exp := []string{"c", "c/d"}; !reflect.DeepEqual(exp, walkedNames) { 116 | t.Error(fmt.Printf("expecting walked names %v, got %v", exp, walkedNames)) 117 | } 118 | for _, v := range walked { 119 | if !v.IsDir() { 120 | t.Errorf("%s should be a dir", v.Name()) 121 | } 122 | } 123 | if err := RemoveAll(fs, "c"); err != nil { 124 | t.Fatal(err) 125 | } 126 | err = Walk(fs, "c", func(fs VFS, path string, info os.FileInfo, err error) error { 127 | return err 128 | }) 129 | if err == nil || !IsNotExist(err) { 130 | t.Errorf("error should be ErrNotExist, it's %v", err) 131 | } 132 | } 133 | 134 | func TestMapFS(t *testing.T) { 135 | fs, err := Map(nil) 136 | if err != nil { 137 | t.Fatal(err) 138 | } 139 | testVFS(t, fs) 140 | } 141 | 142 | func TestPopulatedMap(t *testing.T) { 143 | files := map[string]*File{ 144 | "a/1": {}, 145 | "a/2": {}, 146 | } 147 | fs, err := Map(files) 148 | if err != nil { 149 | t.Fatal(err) 150 | } 151 | infos, err := fs.ReadDir("a") 152 | if err != nil { 153 | t.Fatal(err) 154 | } 155 | if c := len(infos); c != 2 { 156 | t.Fatalf("expecting 2 files in a, got %d", c) 157 | } 158 | if infos[0].Name() != "1" || infos[1].Name() != "2" { 159 | t.Errorf("expecting names 1, 2 got %q, %q", infos[0].Name(), infos[1].Name()) 160 | } 161 | } 162 | 163 | func TestBadPopulatedMap(t *testing.T) { 164 | // 1 can't be file and directory 165 | files := map[string]*File{ 166 | "a/1": {}, 167 | "a/1/2": {}, 168 | } 169 | _, err := Map(files) 170 | if err == nil { 171 | t.Fatal("Map should not work with a path as both file and directory") 172 | } 173 | } 174 | 175 | func TestTmpFS(t *testing.T) { 176 | fs, err := TmpFS("vfs-test") 177 | if err != nil { 178 | t.Fatal(err) 179 | } 180 | defer fs.Close() 181 | testVFS(t, fs) 182 | } 183 | 184 | const ( 185 | go13FileCount = 4157 186 | // +1 because of the root, the real count is 407 187 | go13DirCount = 407 + 1 188 | ) 189 | 190 | func countFileSystem(fs VFS) (int, int, error) { 191 | files, dirs := 0, 0 192 | err := Walk(fs, "/", func(fs VFS, _ string, info os.FileInfo, err error) error { 193 | if err != nil { 194 | return err 195 | } 196 | if info.IsDir() { 197 | dirs++ 198 | } else { 199 | files++ 200 | } 201 | return nil 202 | }) 203 | return files, dirs, err 204 | } 205 | 206 | func testGoFileCount(t *testing.T, fs VFS) { 207 | files, dirs, err := countFileSystem(fs) 208 | if err != nil { 209 | t.Fatal(err) 210 | } 211 | if files != go13FileCount { 212 | t.Errorf("expecting %d files in go1.3, got %d instead", go13FileCount, files) 213 | } 214 | if dirs != go13DirCount { 215 | t.Errorf("expecting %d directories in go1.3, got %d instead", go13DirCount, dirs) 216 | } 217 | } 218 | 219 | func TestGo13Files(t *testing.T) { 220 | f := openOptionalTestFile(t, goTestFile) 221 | defer f.Close() 222 | fs, err := TarGzip(f) 223 | if err != nil { 224 | t.Fatal(err) 225 | } 226 | testGoFileCount(t, fs) 227 | } 228 | 229 | func TestMounter(t *testing.T) { 230 | m := &Mounter{} 231 | f := openOptionalTestFile(t, goTestFile) 232 | defer f.Close() 233 | fs, err := TarGzip(f) 234 | if err != nil { 235 | t.Fatal(err) 236 | } 237 | m.Mount(fs, "/") 238 | testGoFileCount(t, m) 239 | } 240 | 241 | func TestClone(t *testing.T) { 242 | fs, err := Open(filepath.Join("testdata", "fs.zip")) 243 | if err != nil { 244 | t.Fatal(err) 245 | } 246 | infos1, err := fs.ReadDir("/") 247 | if err != nil { 248 | t.Fatal(err) 249 | } 250 | mem1 := Memory() 251 | if err := Clone(mem1, fs); err != nil { 252 | t.Fatal(err) 253 | } 254 | infos2, err := mem1.ReadDir("/") 255 | if err != nil { 256 | t.Fatal(err) 257 | } 258 | if len(infos2) != len(infos1) { 259 | t.Fatalf("cloned fs has %d entries in / rather than %d", len(infos2), len(infos1)) 260 | } 261 | mem2 := Memory() 262 | if err := Clone(mem2, mem1); err != nil { 263 | t.Fatal(err) 264 | } 265 | infos3, err := mem2.ReadDir("/") 266 | if err != nil { 267 | t.Fatal(err) 268 | } 269 | if len(infos3) != len(infos2) { 270 | t.Fatalf("cloned fs has %d entries in / rather than %d", len(infos3), len(infos2)) 271 | } 272 | } 273 | 274 | func measureVFSMemorySize(t testing.TB, fs VFS) int { 275 | mem, ok := fs.(*memoryFileSystem) 276 | if !ok { 277 | t.Fatalf("%T is not a memory filesystem", fs) 278 | } 279 | var total int 280 | var f func(d *Dir) 281 | f = func(d *Dir) { 282 | for _, v := range d.Entries { 283 | total += int(v.Size()) 284 | if sd, ok := v.(*Dir); ok { 285 | f(sd) 286 | } 287 | } 288 | } 289 | f(mem.root) 290 | return total 291 | } 292 | 293 | func hashVFS(t testing.TB, fs VFS) string { 294 | sha := sha1.New() 295 | err := Walk(fs, "/", func(fs VFS, p string, info os.FileInfo, err error) error { 296 | if err != nil || info.IsDir() { 297 | return err 298 | } 299 | f, err := fs.Open(p) 300 | if err != nil { 301 | return err 302 | } 303 | defer f.Close() 304 | if _, err := io.Copy(sha, f); err != nil { 305 | return err 306 | } 307 | return nil 308 | }) 309 | if err != nil { 310 | t.Fatal(err) 311 | } 312 | return hex.EncodeToString(sha.Sum(nil)) 313 | } 314 | 315 | func TestCompress(t *testing.T) { 316 | f := openOptionalTestFile(t, goTestFile) 317 | defer f.Close() 318 | fs, err := TarGzip(f) 319 | if err != nil { 320 | t.Fatal(err) 321 | } 322 | size1 := measureVFSMemorySize(t, fs) 323 | hash1 := hashVFS(t, fs) 324 | if err := Compress(fs); err != nil { 325 | t.Fatalf("can't compress fs: %s", err) 326 | } 327 | testGoFileCount(t, fs) 328 | size2 := measureVFSMemorySize(t, fs) 329 | hash2 := hashVFS(t, fs) 330 | if size2 >= size1 { 331 | t.Fatalf("compressed fs takes more memory %d than bare fs %d", size2, size1) 332 | } 333 | if hash1 != hash2 { 334 | t.Fatalf("compressing fs changed hash from %s to %s", hash1, hash2) 335 | } 336 | } 337 | --------------------------------------------------------------------------------