├── .dockerignore ├── Dockerfile ├── etc ├── prerm.sh ├── postinst.sh ├── yukid.service └── daemon.example.toml ├── .gitignore ├── pkg ├── server │ ├── validator.go │ ├── config_test.go │ ├── middlewares.go │ ├── meta_handlers_test.go │ ├── meta_handlers.go │ ├── config.go │ ├── main_test.go │ ├── utils_test.go │ ├── repo_handlers_test.go │ ├── main.go │ ├── repo_handlers.go │ └── utils.go ├── api │ ├── const.go │ └── types.go ├── yukictl │ ├── cmd │ │ ├── util.go │ │ ├── meta │ │ │ ├── meta.go │ │ │ └── ls.go │ │ ├── repo │ │ │ ├── repo.go │ │ │ ├── rm.go │ │ │ └── ls.go │ │ ├── completion.go │ │ ├── reload.go │ │ └── sync.go │ ├── factory │ │ ├── factory.go │ │ └── impl.go │ └── register.go ├── fs │ ├── utils.go │ └── fs.go ├── model │ ├── migrate.go │ └── repo.go ├── info │ └── info.go ├── set │ └── set.go ├── tabwriter │ └── tabwriter.go └── docker │ ├── fake │ └── cli.go │ └── cli.go ├── test ├── utils │ └── utils.go └── integration │ └── sync_test.go ├── cmd ├── yukictl │ ├── yukictl.go │ └── README.md └── yukid │ ├── yukid.go │ └── README.md ├── Makefile ├── .github ├── dependabot.yml └── workflows │ ├── release.yml │ ├── pr-presubmit-checks.yml │ └── codeql-analysis.yml ├── .golangci.yml ├── .goreleaser.yaml ├── go.mod ├── README.md ├── go.sum └── LICENSE /.dockerignore: -------------------------------------------------------------------------------- 1 | /yukictl 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:bookworm-slim 2 | COPY ./yukid /yukid 3 | CMD ["/yukid"] 4 | -------------------------------------------------------------------------------- /etc/prerm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Only run if systemd is running 4 | [ -d /run/systemd ] || exit 0 5 | 6 | systemctl disable --now yukid.service 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Build output files 2 | /yukictl 3 | /yukid 4 | /*.deb 5 | 6 | .idea/ 7 | /debian/ 8 | dist/ 9 | 10 | *.swp 11 | *.[oa] 12 | *~ 13 | *.bac 14 | -------------------------------------------------------------------------------- /pkg/server/validator.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | type echoValidator func(i any) error 4 | 5 | func (v echoValidator) Validate(i any) error { 6 | return v(i) 7 | } 8 | -------------------------------------------------------------------------------- /etc/postinst.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Only run if systemd is running 4 | [ -d /run/systemd ] || exit 0 5 | 6 | systemctl daemon-reload 7 | systemctl enable yukid.service 8 | -------------------------------------------------------------------------------- /pkg/api/const.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | const ( 4 | LabelRepoName = "org.ustcmirror.name" 5 | LabelStorageDir = "org.ustcmirror.storage-dir" 6 | LabelImages = "org.ustcmirror.images" 7 | ) 8 | -------------------------------------------------------------------------------- /pkg/yukictl/cmd/util.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import "strings" 4 | 5 | const suffixYAML = ".yaml" 6 | 7 | func stripSuffix(s string) string { 8 | return strings.TrimSuffix(s, suffixYAML) 9 | } 10 | -------------------------------------------------------------------------------- /pkg/yukictl/factory/factory.go: -------------------------------------------------------------------------------- 1 | package factory 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | 7 | "github.com/go-resty/resty/v2" 8 | ) 9 | 10 | type Factory interface { 11 | RESTClient() *resty.Client 12 | JSONEncoder(w io.Writer) *json.Encoder 13 | } 14 | -------------------------------------------------------------------------------- /pkg/fs/utils.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | // dirExists checks whether given path is an existing directory. 8 | func dirExists(path string) bool { 9 | stat, err := os.Stat(path) 10 | if err != nil { 11 | return false 12 | } 13 | return stat.IsDir() 14 | } 15 | -------------------------------------------------------------------------------- /pkg/yukictl/cmd/meta/meta.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | 6 | "github.com/ustclug/Yuki/pkg/yukictl/factory" 7 | ) 8 | 9 | func NewCmdMeta(f factory.Factory) *cobra.Command { 10 | cmd := &cobra.Command{ 11 | Use: "meta", 12 | Short: "List metas", 13 | } 14 | cmd.AddCommand( 15 | NewCmdMetaLs(f), 16 | ) 17 | return cmd 18 | } 19 | -------------------------------------------------------------------------------- /pkg/model/migrate.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "fmt" 5 | 6 | "gorm.io/gorm" 7 | ) 8 | 9 | func AutoMigrate(db *gorm.DB) error { 10 | // enable WAL mode by default to improve performance 11 | err := db.Exec("PRAGMA journal_mode=WAL").Error 12 | if err != nil { 13 | return fmt.Errorf("set WAL mode: %w", err) 14 | } 15 | return db.AutoMigrate(&Repo{}, &RepoMeta{}) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/yukictl/cmd/repo/repo.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | 6 | "github.com/ustclug/Yuki/pkg/yukictl/factory" 7 | ) 8 | 9 | func NewCmdRepo(f factory.Factory) *cobra.Command { 10 | cmd := &cobra.Command{ 11 | Use: "repo", 12 | Short: "Manage repositories", 13 | } 14 | cmd.AddCommand( 15 | NewCmdRepoLs(f), 16 | NewCmdRepoRm(f), 17 | ) 18 | return cmd 19 | } 20 | -------------------------------------------------------------------------------- /pkg/info/info.go: -------------------------------------------------------------------------------- 1 | package info 2 | 3 | import ( 4 | "runtime" 5 | ) 6 | 7 | var ( 8 | Version string 9 | BuildDate string 10 | GitCommit string 11 | ) 12 | 13 | var VersionInfo = struct { 14 | Version string 15 | GoVersion string 16 | BuildDate string 17 | GitCommit string 18 | }{ 19 | Version: Version, 20 | BuildDate: BuildDate, 21 | GitCommit: GitCommit, 22 | GoVersion: runtime.Version(), 23 | } 24 | -------------------------------------------------------------------------------- /etc/yukid.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Yuki - USTC Mirror Manager 3 | After=docker.service 4 | Requires=docker.service 5 | PartOf=docker.service 6 | ConditionPathExists=/etc/yuki/daemon.toml 7 | 8 | [Service] 9 | Type=exec 10 | User=mirror 11 | ExecStart=/usr/bin/yukid 12 | ExecReload=/usr/bin/yukictl reload 13 | Restart=on-failure 14 | RestartSec=5 15 | 16 | [Install] 17 | Alias=yuki.service 18 | WantedBy=multi-user.target 19 | -------------------------------------------------------------------------------- /pkg/yukictl/register.go: -------------------------------------------------------------------------------- 1 | package yukictl 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | 6 | "github.com/ustclug/Yuki/pkg/yukictl/cmd" 7 | "github.com/ustclug/Yuki/pkg/yukictl/cmd/meta" 8 | "github.com/ustclug/Yuki/pkg/yukictl/cmd/repo" 9 | "github.com/ustclug/Yuki/pkg/yukictl/factory" 10 | ) 11 | 12 | func Register(root *cobra.Command, f factory.Factory) { 13 | root.AddCommand( 14 | cmd.NewCmdCompletion(), 15 | cmd.NewCmdReload(f), 16 | cmd.NewCmdSync(f), 17 | meta.NewCmdMeta(f), 18 | repo.NewCmdRepo(f), 19 | ) 20 | } 21 | -------------------------------------------------------------------------------- /pkg/set/set.go: -------------------------------------------------------------------------------- 1 | package set 2 | 3 | type Set[T comparable] map[T]struct{} 4 | 5 | func New[T comparable](elements ...T) Set[T] { 6 | s := make(Set[T], len(elements)) 7 | s.Add(elements...) 8 | return s 9 | } 10 | 11 | func (s Set[T]) Add(elements ...T) { 12 | for _, v := range elements { 13 | s[v] = struct{}{} 14 | } 15 | } 16 | 17 | func (s Set[T]) Del(ele T) { 18 | delete(s, ele) 19 | } 20 | 21 | func (s Set[T]) ToList() []T { 22 | list := make([]T, 0, len(s)) 23 | for k := range s { 24 | list = append(list, k) 25 | } 26 | return list 27 | } 28 | -------------------------------------------------------------------------------- /pkg/yukictl/factory/impl.go: -------------------------------------------------------------------------------- 1 | package factory 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | 7 | "github.com/go-resty/resty/v2" 8 | "github.com/spf13/pflag" 9 | ) 10 | 11 | type factoryImpl struct { 12 | remote string 13 | } 14 | 15 | func (f *factoryImpl) RESTClient() *resty.Client { 16 | return resty.New().SetBaseURL(f.remote) 17 | } 18 | 19 | func (f *factoryImpl) JSONEncoder(w io.Writer) *json.Encoder { 20 | encoder := json.NewEncoder(w) 21 | encoder.SetIndent("", " ") 22 | return encoder 23 | } 24 | 25 | func New(flags *pflag.FlagSet) Factory { 26 | s := factoryImpl{} 27 | flags.StringVarP(&s.remote, "remote", "r", "http://127.0.0.1:9999/", "Remote address") 28 | return &s 29 | } 30 | -------------------------------------------------------------------------------- /test/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func PollUntilTimeout(t *testing.T, timeout time.Duration, f func() bool) { 12 | timer := time.NewTimer(timeout) 13 | ticker := time.NewTicker(time.Second) 14 | t.Cleanup(func() { 15 | if !timer.Stop() { 16 | <-timer.C 17 | } 18 | ticker.Stop() 19 | }) 20 | loop: 21 | for { 22 | select { 23 | case <-ticker.C: 24 | if f() { 25 | break loop 26 | } 27 | case <-timer.C: 28 | t.Fatal("Timeout") 29 | } 30 | } 31 | } 32 | 33 | func WriteFile(t *testing.T, path, content string) { 34 | require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) 35 | } 36 | -------------------------------------------------------------------------------- /pkg/server/config_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/require" 9 | 10 | testutils "github.com/ustclug/Yuki/test/utils" 11 | ) 12 | 13 | func TestLoadConfig(t *testing.T) { 14 | tmp, err := os.CreateTemp("", "TestLoadConfig*.toml") 15 | require.NoError(t, err) 16 | defer os.Remove(tmp.Name()) 17 | defer tmp.Close() 18 | testutils.WriteFile(t, tmp.Name(), ` 19 | db_url = ":memory:" 20 | repo_logs_dir = "/tmp" 21 | repo_config_dir = "/tmp" 22 | sync_timeout = "15s" 23 | `) 24 | srv, err := New(tmp.Name()) 25 | require.NoError(t, err) 26 | require.Equal(t, time.Second*15, srv.config.SyncTimeout) 27 | require.Equal(t, "/tmp", srv.config.RepoConfigDir[0]) 28 | } 29 | -------------------------------------------------------------------------------- /pkg/yukictl/cmd/completion.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | ) 6 | 7 | func NewCmdCompletion() *cobra.Command { 8 | rootCmd := &cobra.Command{ 9 | Use: "completion", 10 | Short: "Output shell completion code for the specified shell (bash or zsh).", 11 | } 12 | 13 | bashCmd := &cobra.Command{ 14 | Use: "bash", 15 | RunE: func(cmd *cobra.Command, args []string) error { 16 | return cmd.Root().GenBashCompletion(cmd.OutOrStdout()) 17 | }, 18 | } 19 | zshCmd := &cobra.Command{ 20 | Use: "zsh", 21 | RunE: func(cmd *cobra.Command, args []string) error { 22 | return cmd.Root().GenZshCompletion(cmd.OutOrStdout()) 23 | }, 24 | } 25 | rootCmd.AddCommand(bashCmd, zshCmd) 26 | return rootCmd 27 | } 28 | -------------------------------------------------------------------------------- /pkg/api/types.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | type ListRepoMetasResponse = []GetRepoMetaResponse 4 | 5 | type GetRepoMetaResponse struct { 6 | Name string `json:"name"` 7 | Upstream string `json:"upstream"` 8 | Syncing bool `json:"syncing"` 9 | Size int64 `json:"size"` 10 | ExitCode int `json:"exitCode"` 11 | LastSuccess int64 `json:"lastSuccess"` 12 | UpdatedAt int64 `json:"updatedAt"` 13 | PrevRun int64 `json:"prevRun"` 14 | NextRun int64 `json:"nextRun"` 15 | } 16 | 17 | type ListReposResponseItem struct { 18 | Name string `json:"name"` 19 | Cron string `json:"cron"` 20 | Image string `json:"image"` 21 | StorageDir string `json:"storageDir"` 22 | } 23 | 24 | type ListReposResponse = []ListReposResponseItem 25 | -------------------------------------------------------------------------------- /cmd/yukictl/yukictl.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/spf13/cobra" 7 | 8 | "github.com/ustclug/Yuki/pkg/info" 9 | "github.com/ustclug/Yuki/pkg/yukictl" 10 | "github.com/ustclug/Yuki/pkg/yukictl/factory" 11 | ) 12 | 13 | func main() { 14 | var printVersion bool 15 | rootCmd := &cobra.Command{ 16 | Use: "yukictl", 17 | SilenceUsage: true, 18 | RunE: func(cmd *cobra.Command, args []string) error { 19 | if printVersion { 20 | return json.NewEncoder(cmd.OutOrStdout()).Encode(info.VersionInfo) 21 | } 22 | return nil 23 | }, 24 | } 25 | rootCmd.Flags().BoolVarP(&printVersion, "version", "V", false, "Print version information and quit") 26 | f := factory.New(rootCmd.PersistentFlags()) 27 | yukictl.Register(rootCmd, f) 28 | _ = rootCmd.Execute() 29 | } 30 | -------------------------------------------------------------------------------- /pkg/server/middlewares.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "log/slog" 6 | 7 | "github.com/labstack/echo/v4" 8 | ) 9 | 10 | const ctxKeyLogger = "yukid-logger" 11 | 12 | func setLogger(logger *slog.Logger) echo.MiddlewareFunc { 13 | return func(next echo.HandlerFunc) echo.HandlerFunc { 14 | return func(c echo.Context) error { 15 | reqID := c.Response().Header().Get(echo.HeaderXRequestID) 16 | attrs := []any{ 17 | slog.String("req_id", reqID), 18 | } 19 | routePath := c.Path() 20 | if len(routePath) > 0 { 21 | attrs = append( 22 | attrs, 23 | slog.String("endpoint", fmt.Sprintf("%s %s", c.Request().Method, routePath)), 24 | ) 25 | } 26 | c.Set( 27 | ctxKeyLogger, 28 | logger.With(attrs...), 29 | ) 30 | return next(c) 31 | } 32 | } 33 | } 34 | 35 | func getLogger(c echo.Context) *slog.Logger { 36 | return c.Get(ctxKeyLogger).(*slog.Logger) 37 | } 38 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | all: yukid yukictl 3 | 4 | .PHONY: release 5 | release: 6 | goreleaser release --snapshot --clean 7 | 8 | .PHONY: clean 9 | clean: 10 | rm -rf yukid yukictl dist/ 11 | 12 | .PHONY: lint 13 | lint: 14 | golangci-lint run --fix ./... 15 | 16 | .PHONY: unit-test 17 | unit-test: 18 | go test -race -v ./pkg/... 19 | 20 | .PHONY: integration-test 21 | integration-test: 22 | go test -v ./test/integration/... 23 | 24 | git_commit := $(shell git rev-parse HEAD) 25 | build_date := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') 26 | version ?= $(shell git describe --tags) 27 | 28 | go_ldflags := -s -w \ 29 | -X github.com/ustclug/Yuki/pkg/info.BuildDate=$(build_date) \ 30 | -X github.com/ustclug/Yuki/pkg/info.GitCommit=$(git_commit) \ 31 | -X github.com/ustclug/Yuki/pkg/info.Version=$(version) 32 | 33 | .PHONY: yukid 34 | yukid: 35 | go build -ldflags "$(go_ldflags)" -trimpath ./cmd/yukid 36 | 37 | .PHONY: yukictl 38 | yukictl: 39 | go build -ldflags "$(go_ldflags)" -trimpath ./cmd/yukictl 40 | -------------------------------------------------------------------------------- /cmd/yukictl/README.md: -------------------------------------------------------------------------------- 1 | # yukictl 2 | 3 | ### Table of Content 4 | 5 | + [Introduction](#introduction) 6 | + [Handbook](#handbook) 7 | - [自动补全](#自动补全) 8 | - [获取同步状态](#获取同步状态) 9 | - [手动开始同步任务](#手动开始同步任务) 10 | - [更新仓库同步配置](#更新仓库同步配置) 11 | 12 | ### Introduction 13 | 14 | yuki 的命令行客户端。 15 | 16 | ### Handbook 17 | 18 | #### 自动补全 19 | 20 | ```bash 21 | # Zsh: 22 | $ yukictl completion zsh 23 | 24 | # Bash: 25 | $ yukictl completion bash 26 | ``` 27 | 28 | #### 获取同步状态 29 | 30 | ```bash 31 | $ yukictl meta ls [repo] 32 | ``` 33 | 34 | #### 手动开始同步任务 35 | 36 | ```bash 37 | $ yukictl sync 38 | ``` 39 | 40 | 开启同步任务的 debug 模式 41 | ```bash 42 | $ yukictl sync --debug 43 | ``` 44 | 45 | #### 更新仓库同步配置 46 | 47 | 新增或修改完仓库的 YAML 配置后，需要执行下面的命令来更新配置。 48 | ```bash 49 | $ yukictl reload 50 | ``` 51 | 注意：在新增配置前需要先创建仓库相应的 `storageDir`。 52 | 53 | 如果不带任何参数的话，则该命令会更新所有仓库的同步配置，并且删除配置里没有但数据库里有的仓库配置。 54 | ```bash 55 | $ yukictl reload 56 | ``` 57 | 58 | 若需要删除仓库，则可以删除相应的配置文件，然后执行 `yukictl repo rm ` 或 `yukictl reload` 来从数据库里删除配置。 59 | 60 | -------------------------------------------------------------------------------- /pkg/yukictl/cmd/repo/rm.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/labstack/echo/v4" 7 | "github.com/spf13/cobra" 8 | 9 | "github.com/ustclug/Yuki/pkg/yukictl/factory" 10 | ) 11 | 12 | type rmOptions struct { 13 | name string 14 | } 15 | 16 | func (o *rmOptions) Run(f factory.Factory) error { 17 | var errMsg echo.HTTPError 18 | resp, err := f.RESTClient().R(). 19 | SetError(&errMsg). 20 | SetPathParam("name", o.name). 21 | Delete("api/v1/repos/{name}") 22 | if err != nil { 23 | return err 24 | } 25 | if resp.IsError() { 26 | return fmt.Errorf("%s", errMsg.Message) 27 | } 28 | fmt.Printf("Successfully deleted from database: <%s>\n", o.name) 29 | return nil 30 | } 31 | 32 | func NewCmdRepoRm(f factory.Factory) *cobra.Command { 33 | o := rmOptions{} 34 | return &cobra.Command{ 35 | Use: "rm", 36 | Short: "Remove repository from database", 37 | Example: " yukictl repo rm REPO", 38 | Args: cobra.ExactArgs(1), 39 | RunE: func(cmd *cobra.Command, args []string) error { 40 | o.name = args[0] 41 | return o.Run(f) 42 | }, 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | # Maintain dependencies for GitHub Actions 9 | - package-ecosystem: "github-actions" 10 | # Workflow files stored in the default location of `.github/workflows`. (You don't need to specify `/.github/workflows` for `directory`. You can use `directory: "/"`.) 11 | directory: "/" 12 | schedule: 13 | interval: "weekly" 14 | groups: 15 | actions: 16 | update-types: 17 | - "major" 18 | - "minor" 19 | - "patch" 20 | - package-ecosystem: "gomod" # See documentation for possible values 21 | directory: "/" # Location of package manifests 22 | schedule: 23 | interval: "weekly" 24 | groups: 25 | gomod: 26 | update-types: 27 | - "minor" 28 | - "patch" 29 | -------------------------------------------------------------------------------- /cmd/yukid/yukid.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "os" 7 | "os/signal" 8 | 9 | "github.com/spf13/cobra" 10 | 11 | "github.com/ustclug/Yuki/pkg/info" 12 | "github.com/ustclug/Yuki/pkg/server" 13 | ) 14 | 15 | func main() { 16 | var ( 17 | configPath string 18 | printVersion bool 19 | ) 20 | cmd := cobra.Command{ 21 | Use: "yukid", 22 | SilenceUsage: true, 23 | RunE: func(cmd *cobra.Command, args []string) error { 24 | if printVersion { 25 | return json.NewEncoder(cmd.OutOrStdout()).Encode(info.VersionInfo) 26 | } 27 | 28 | s, err := server.New(configPath) 29 | if err != nil { 30 | return err 31 | } 32 | ctx, cancel := context.WithCancel(context.Background()) 33 | signals := make(chan os.Signal, 2) 34 | signal.Notify(signals, os.Interrupt) 35 | go func() { 36 | <-signals 37 | cancel() 38 | <-signals 39 | os.Exit(1) 40 | }() 41 | return s.Start(ctx) 42 | }, 43 | } 44 | cmd.Flags().StringVar(&configPath, "config", "/etc/yuki/daemon.toml", "The path to config file") 45 | cmd.Flags().BoolVarP(&printVersion, "version", "V", false, "Print version information and quit") 46 | 47 | _ = cmd.Execute() 48 | } 49 | -------------------------------------------------------------------------------- /pkg/yukictl/cmd/reload.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/labstack/echo/v4" 7 | "github.com/spf13/cobra" 8 | 9 | "github.com/ustclug/Yuki/pkg/yukictl/factory" 10 | ) 11 | 12 | type reloadOptions struct { 13 | repo string 14 | } 15 | 16 | func (o *reloadOptions) Run(f factory.Factory) error { 17 | req := f.RESTClient().R() 18 | path := "api/v1/repos" 19 | if len(o.repo) > 0 { 20 | path += "/" + o.repo 21 | } 22 | var errMsg echo.HTTPError 23 | resp, err := req.SetError(&errMsg).Post(path) 24 | if err != nil { 25 | return err 26 | } 27 | if resp.IsError() { 28 | return fmt.Errorf("%s", errMsg.Message) 29 | } 30 | if len(o.repo) > 0 { 31 | fmt.Printf("Successfully reloaded: <%s>\n", o.repo) 32 | } else { 33 | fmt.Println("Successfully reloaded all repositories") 34 | } 35 | return nil 36 | } 37 | 38 | func NewCmdReload(f factory.Factory) *cobra.Command { 39 | o := reloadOptions{} 40 | cmd := &cobra.Command{ 41 | Use: "reload [name]", 42 | Short: "Reload config of one or all repos", 43 | RunE: func(cmd *cobra.Command, args []string) error { 44 | if len(args) > 0 { 45 | o.repo = stripSuffix(args[0]) 46 | } 47 | return o.Run(f) 48 | }, 49 | } 50 | return cmd 51 | } 52 | -------------------------------------------------------------------------------- /pkg/yukictl/cmd/sync.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/labstack/echo/v4" 7 | "github.com/spf13/cobra" 8 | 9 | "github.com/ustclug/Yuki/pkg/yukictl/factory" 10 | ) 11 | 12 | type syncOptions struct { 13 | debug bool 14 | name string 15 | } 16 | 17 | func (o *syncOptions) Run(f factory.Factory) error { 18 | req := f.RESTClient().R() 19 | var errMsg echo.HTTPError 20 | if o.debug { 21 | req.SetQueryParam("debug", "true") 22 | } 23 | resp, err := req. 24 | SetError(&errMsg). 25 | SetPathParam("name", o.name). 26 | Post("api/v1/repos/{name}/sync") 27 | if err != nil { 28 | return err 29 | } 30 | if resp.IsError() { 31 | return fmt.Errorf("%s", errMsg.Message) 32 | } 33 | 34 | fmt.Printf("Syncing <%s>\n", o.name) 35 | return nil 36 | } 37 | 38 | func NewCmdSync(f factory.Factory) *cobra.Command { 39 | o := syncOptions{} 40 | cmd := &cobra.Command{ 41 | Use: "sync", 42 | Args: cobra.ExactArgs(1), 43 | Example: " yukictl sync REPO", 44 | Short: "Sync local repository with remote", 45 | RunE: func(cmd *cobra.Command, args []string) error { 46 | o.name = stripSuffix(args[0]) 47 | return o.Run(f) 48 | }, 49 | } 50 | cmd.Flags().BoolVarP(&o.debug, "debug", "v", false, "Debug mode") 51 | return cmd 52 | } 53 | -------------------------------------------------------------------------------- /pkg/model/repo.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type StringMap map[string]string 4 | 5 | // Repo represents a Repository. 6 | type Repo struct { 7 | Name string `gorm:"primaryKey" json:"name" validate:"required"` 8 | // NOTE: the cron validator does not support */number syntax 9 | Cron string `json:"cron" validate:"required"` 10 | Image string `json:"image" validate:"required"` 11 | StorageDir string `json:"storageDir" validate:"required,dir"` 12 | User string `json:"user"` 13 | BindIP string `json:"bindIP" validate:"omitempty,ip"` 14 | Network string `json:"network"` 15 | LogRotCycle int `json:"logRotCycle" validate:"min=0"` 16 | Retry int `json:"retry" validate:"min=0"` 17 | Envs StringMap `gorm:"type:text;serializer:json" json:"envs"` 18 | Volumes StringMap `gorm:"type:text;serializer:json" json:"volumes"` 19 | // sqlite3 does not have builtin datetime type 20 | CreatedAt int64 `gorm:"autoCreateTime" json:"-"` 21 | UpdatedAt int64 `gorm:"autoUpdateTime" json:"-"` 22 | } 23 | 24 | // RepoMeta represents the metadata of a Repository. 25 | type RepoMeta struct { 26 | Name string `gorm:"primaryKey"` 27 | Upstream string 28 | Size int64 29 | ExitCode int 30 | CreatedAt int64 `gorm:"autoCreateTime"` 31 | UpdatedAt int64 `gorm:"autoUpdateTime"` 32 | LastSuccess int64 33 | PrevRun int64 34 | NextRun int64 35 | Syncing bool 36 | } 37 | -------------------------------------------------------------------------------- /pkg/tabwriter/tabwriter.go: -------------------------------------------------------------------------------- 1 | package tabwriter 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "strings" 8 | "text/tabwriter" 9 | ) 10 | 11 | const ( 12 | minWidth = 6 13 | width = 4 14 | padding = 3 15 | padChar = ' ' 16 | ) 17 | 18 | type Writer struct { 19 | delegate *tabwriter.Writer 20 | 21 | header []string 22 | buf bytes.Buffer 23 | } 24 | 25 | func toStringList(args ...interface{}) []string { 26 | strLst := make([]string, 0, len(args)) 27 | for _, arg := range args { 28 | strLst = append(strLst, fmt.Sprint(arg)) 29 | } 30 | return strLst 31 | } 32 | 33 | func (w *Writer) Render() error { 34 | // print header 35 | _, err := fmt.Fprintln(w.delegate, strings.Join(w.header, "\t")) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | // print content 41 | _, err = w.buf.WriteTo(w.delegate) 42 | if err != nil { 43 | return err 44 | } 45 | return w.delegate.Flush() 46 | } 47 | 48 | func (w *Writer) Append(args ...interface{}) { 49 | _, _ = fmt.Fprintln(&w.buf, strings.Join(toStringList(args...), "\t")) 50 | } 51 | 52 | func (w *Writer) SetHeader(header []string) { 53 | upperHeader := make([]string, 0, len(header)) 54 | for _, col := range header { 55 | upperHeader = append(upperHeader, strings.ToUpper(col)) 56 | } 57 | w.header = upperHeader 58 | } 59 | 60 | func New(out io.Writer) *Writer { 61 | return &Writer{ 62 | delegate: tabwriter.NewWriter(out, 63 | minWidth, 64 | width, 65 | padding, 66 | padChar, 67 | 0), 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /pkg/server/meta_handlers_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/ustclug/Yuki/pkg/model" 10 | ) 11 | 12 | func TestHandlerListRepoMetas(t *testing.T) { 13 | te := NewTestEnv(t) 14 | require.NoError(t, te.server.db.Create([]model.RepoMeta{ 15 | { 16 | Name: "repo2", 17 | }, 18 | { 19 | Name: "repo1", 20 | }, 21 | }).Error) 22 | 23 | var metas []model.RepoMeta 24 | cli := te.RESTClient() 25 | resp, err := cli.R().SetResult(&metas).Get("/metas") 26 | require.NoError(t, err) 27 | require.True(t, resp.IsSuccess(), "Unexpected response: %s", resp.Body()) 28 | 29 | require.Len(t, metas, 2) 30 | require.Equal(t, "repo1", metas[0].Name) 31 | } 32 | 33 | func TestHandlerGetRepoMeta(t *testing.T) { 34 | te := NewTestEnv(t) 35 | require.NoError(t, te.server.db.Create([]model.RepoMeta{ 36 | { 37 | Name: t.Name(), 38 | }, 39 | }).Error) 40 | 41 | cli := te.RESTClient() 42 | testCases := map[string]struct { 43 | name string 44 | expectStatus int 45 | }{ 46 | "ok": { 47 | name: t.Name(), 48 | expectStatus: http.StatusOK, 49 | }, 50 | "not found": { 51 | name: "not found", 52 | expectStatus: http.StatusNotFound, 53 | }, 54 | } 55 | for name, tc := range testCases { 56 | t.Run(name, func(t *testing.T) { 57 | resp, err := cli.R().Get("/metas/" + tc.name) 58 | require.NoError(t, err) 59 | require.Equal(t, tc.expectStatus, resp.StatusCode()) 60 | }) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | # run only against tags 6 | tags: 7 | - "v*" 8 | 9 | concurrency: 10 | cancel-in-progress: true 11 | group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} 12 | 13 | permissions: 14 | contents: write 15 | packages: write 16 | 17 | jobs: 18 | releaser: 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v6 22 | 23 | - name: Install Go 24 | uses: actions/setup-go@v6 25 | with: 26 | go-version: stable 27 | check-latest: true 28 | 29 | - name: Set up Docker Buildx 30 | uses: docker/setup-buildx-action@v3 31 | 32 | - name: Log in to the Container registry 33 | uses: docker/login-action@v3 34 | with: 35 | registry: ghcr.io 36 | username: ${{ github.actor }} 37 | password: ${{ secrets.GITHUB_TOKEN }} 38 | 39 | - name: Build 40 | run: | 41 | CGO_ENABLED=0 make yukictl yukid 42 | 43 | - name: Build and Push Image 44 | uses: docker/build-push-action@v6 45 | with: 46 | context: . 47 | push: true 48 | tags: ghcr.io/ustclug/yukid:${{ github.ref_name }} 49 | 50 | # More assembly might be required: Docker logins, GPG, etc. 51 | # It all depends on your needs. 52 | - uses: goreleaser/goreleaser-action@v6 53 | with: 54 | version: latest 55 | args: release --clean 56 | env: 57 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 58 | -------------------------------------------------------------------------------- /.github/workflows/pr-presubmit-checks.yml: -------------------------------------------------------------------------------- 1 | name: Presubmit Checks 2 | 3 | on: 4 | pull_request: 5 | types: [opened, synchronize, reopened] 6 | push: 7 | branches: 8 | - master 9 | paths: 10 | - '**.go' 11 | 12 | concurrency: 13 | cancel-in-progress: true 14 | group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} 15 | 16 | jobs: 17 | lint: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Checkout code 21 | uses: actions/checkout@v6 22 | - name: Install Go 23 | uses: actions/setup-go@v6 24 | with: 25 | go-version: stable 26 | check-latest: true 27 | - name: Run golangci-lint 28 | uses: golangci/golangci-lint-action@v9 29 | with: 30 | version: latest 31 | 32 | unit-test: 33 | name: Unit Test 34 | runs-on: ubuntu-latest 35 | steps: 36 | - name: Checkout code 37 | uses: actions/checkout@v6 38 | - name: Install Go 39 | uses: actions/setup-go@v6 40 | with: 41 | go-version: stable 42 | check-latest: true 43 | - name: Test 44 | run: | 45 | make unit-test 46 | 47 | integration-test: 48 | name: Integration Test 49 | runs-on: ubuntu-latest 50 | steps: 51 | - name: Checkout code 52 | uses: actions/checkout@v6 53 | - name: Install Go 54 | uses: actions/setup-go@v6 55 | with: 56 | go-version: stable 57 | check-latest: true 58 | - name: Integration Test 59 | run: | 60 | set -euo pipefail 61 | docker pull ustcmirror/test 62 | make integration-test 63 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | run: 3 | modules-download-mode: readonly 4 | linters: 5 | enable: 6 | - copyloopvar 7 | - depguard 8 | - exhaustive 9 | - gochecknoinits 10 | - goconst 11 | - gocritic 12 | - importas 13 | - misspell 14 | - nolintlint 15 | - prealloc 16 | - revive 17 | - testifylint 18 | - unconvert 19 | - unparam 20 | - usestdlibvars 21 | - whitespace 22 | settings: 23 | depguard: 24 | rules: 25 | main: 26 | deny: 27 | - pkg: github.com/docker/docker 28 | desc: https://github.com/ustclug/Yuki/issues/44 29 | exhaustive: 30 | explicit-exhaustive-switch: true 31 | govet: 32 | enable: 33 | - nilness 34 | revive: 35 | confidence: 0.6 36 | rules: 37 | - name: blank-imports 38 | - name: dot-imports 39 | - name: error-strings 40 | - name: errorf 41 | exclusions: 42 | generated: lax 43 | presets: 44 | - comments 45 | - common-false-positives 46 | - legacy 47 | - std-error-handling 48 | rules: 49 | - linters: 50 | - errcheck 51 | path: _test.go 52 | paths: 53 | - third_party$ 54 | - builtin$ 55 | - examples$ 56 | formatters: 57 | enable: 58 | - gci 59 | - gofmt 60 | - goimports 61 | settings: 62 | gci: 63 | sections: 64 | - standard 65 | - default 66 | - prefix(github.com/ustclug/Yuki) 67 | exclusions: 68 | generated: lax 69 | paths: 70 | - third_party$ 71 | - builtin$ 72 | - examples$ 73 | -------------------------------------------------------------------------------- /pkg/server/meta_handlers.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/labstack/echo/v4" 7 | 8 | "github.com/ustclug/Yuki/pkg/api" 9 | "github.com/ustclug/Yuki/pkg/model" 10 | ) 11 | 12 | func (s *Server) handlerListRepoMetas(c echo.Context) error { 13 | l := getLogger(c) 14 | l.Debug("Invoked") 15 | 16 | var metas []model.RepoMeta 17 | err := s.getDB(c).Order("name").Find(&metas).Error 18 | if err != nil { 19 | const msg = "Fail to list RepoMetas" 20 | l.Error(msg, slogErrAttr(err)) 21 | return &echo.HTTPError{ 22 | Code: http.StatusInternalServerError, 23 | Message: msg, 24 | } 25 | } 26 | resp := make(api.ListRepoMetasResponse, len(metas)) 27 | for i, meta := range metas { 28 | resp[i] = s.convertModelRepoMetaToGetMetaResponse(meta) 29 | } 30 | return c.JSON(http.StatusOK, resp) 31 | } 32 | 33 | func (s *Server) handlerGetRepoMeta(c echo.Context) error { 34 | l := getLogger(c) 35 | l.Debug("Invoked") 36 | 37 | name, err := getRepoNameFromRoute(c) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | var meta model.RepoMeta 43 | res := s.getDB(c). 44 | Where(model.RepoMeta{ 45 | Name: name, 46 | }). 47 | Limit(1). 48 | Find(&meta) 49 | if res.Error != nil { 50 | const msg = "Fail to get RepoMeta" 51 | l.Error(msg, slogErrAttr(res.Error)) 52 | return &echo.HTTPError{ 53 | Code: http.StatusInternalServerError, 54 | Message: msg, 55 | } 56 | } 57 | if res.RowsAffected == 0 { 58 | return &echo.HTTPError{ 59 | Code: http.StatusNotFound, 60 | Message: "RepoMeta not found", 61 | } 62 | } 63 | 64 | resp := s.convertModelRepoMetaToGetMetaResponse(meta) 65 | return c.JSON(http.StatusOK, resp) 66 | } 67 | -------------------------------------------------------------------------------- /pkg/yukictl/cmd/repo/ls.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/labstack/echo/v4" 8 | "github.com/spf13/cobra" 9 | 10 | "github.com/ustclug/Yuki/pkg/api" 11 | "github.com/ustclug/Yuki/pkg/model" 12 | "github.com/ustclug/Yuki/pkg/tabwriter" 13 | "github.com/ustclug/Yuki/pkg/yukictl/factory" 14 | ) 15 | 16 | type repoLsOptions struct { 17 | name string 18 | } 19 | 20 | func (o *repoLsOptions) Run(f factory.Factory) error { 21 | var ( 22 | err error 23 | errMsg echo.HTTPError 24 | ) 25 | req := f.RESTClient().R().SetError(&errMsg) 26 | encoder := f.JSONEncoder(os.Stdout) 27 | if len(o.name) > 0 { 28 | var result model.Repo 29 | resp, err := req.SetResult(&result).SetPathParam("name", o.name).Get("api/v1/repos/{name}") 30 | if err != nil { 31 | return err 32 | } 33 | if resp.IsError() { 34 | return fmt.Errorf("%s", errMsg.Message) 35 | } 36 | return encoder.Encode(result) 37 | } 38 | 39 | var result api.ListReposResponse 40 | resp, err := req.SetResult(&result).Get("api/v1/repos") 41 | if err != nil { 42 | return err 43 | } 44 | if resp.IsError() { 45 | return fmt.Errorf("%s", errMsg.Message) 46 | } 47 | printer := tabwriter.New(os.Stdout) 48 | printer.SetHeader([]string{ 49 | "name", 50 | "cron", 51 | "image", 52 | "storage-dir", 53 | }) 54 | for _, r := range result { 55 | printer.Append(r.Name, r.Cron, r.Image, r.StorageDir) 56 | } 57 | return printer.Render() 58 | } 59 | 60 | func NewCmdRepoLs(f factory.Factory) *cobra.Command { 61 | o := repoLsOptions{} 62 | cmd := &cobra.Command{ 63 | Use: "ls", 64 | Short: "List one or all repositories", 65 | RunE: func(cmd *cobra.Command, args []string) error { 66 | if len(args) > 0 { 67 | o.name = args[0] 68 | } 69 | return o.Run(f) 70 | }, 71 | } 72 | return cmd 73 | } 74 | -------------------------------------------------------------------------------- /pkg/server/config.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | ) 8 | 9 | type Config struct { 10 | Debug bool `mapstructure:"debug"` 11 | DbURL string `mapstructure:"db_url" validate:"required"` 12 | FileSystem string `mapstructure:"fs" validate:"oneof=xfs zfs default"` 13 | DockerEndpoint string `mapstructure:"docker_endpoint" validate:"unix_addr|tcp_addr"` 14 | Owner string `mapstructure:"owner"` 15 | LogFile string `mapstructure:"log_file" validate:"filepath"` 16 | RepoLogsDir string `mapstructure:"repo_logs_dir" validate:"dir"` 17 | RepoConfigDir []string `mapstructure:"repo_config_dir" validate:"required,dive,dir"` 18 | LogLevel string `mapstructure:"log_level" validate:"oneof=debug info warn error"` 19 | ListenAddr string `mapstructure:"listen_addr" validate:"hostname_port"` 20 | BindIP string `mapstructure:"bind_ip" validate:"omitempty,ip"` 21 | NamePrefix string `mapstructure:"name_prefix"` 22 | PostSync []string `mapstructure:"post_sync"` 23 | ImagesUpgradeInterval time.Duration `mapstructure:"images_upgrade_interval" validate:"min=0"` 24 | SyncTimeout time.Duration `mapstructure:"sync_timeout" validate:"min=0"` 25 | } 26 | 27 | var DefaultConfig = Config{ 28 | FileSystem: "default", 29 | DockerEndpoint: "unix:///var/run/docker.sock", 30 | LogFile: "/dev/stderr", 31 | Owner: fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()), 32 | RepoLogsDir: "/var/log/yuki/", 33 | ListenAddr: "127.0.0.1:9999", 34 | NamePrefix: "syncing-", 35 | LogLevel: "info", 36 | ImagesUpgradeInterval: time.Hour, 37 | } 38 | -------------------------------------------------------------------------------- /etc/daemon.example.toml: -------------------------------------------------------------------------------- 1 | ## 设置 debug 为 true 后会打开 echo web 框架的 debug 模式 2 | ## 以及在日志里输出程序里打印日志的位置 3 | #debug = true 4 | 5 | ## 设置 sqlite 数据库文件的路径 6 | ## 格式可以是文件路径或者是 url（如果需要设置特定参数的话）。例如： 7 | ## /var/run/yukid/data.db 8 | ## file:///home/fred/data.db?mode=ro&cache=private 9 | ## 参考 https://www.sqlite.org/c3ref/open.html 10 | db_url = "/path/to/yukid.db" 11 | 12 | ## 每个仓库的同步配置存放的文件夹 13 | ## 每个配置的后缀名必须是 `.yaml` 14 | ## 配置的格式参考下方 Repo Configuration 15 | repo_config_dir = ["/path/to/config-dir"] 16 | 17 | ## 设置同步日志存放的文件夹 18 | ## 默认值是 /var/log/yuki/ 19 | #repo_logs_dir = "/var/log/yuki/" 20 | 21 | ## 数据所在位置的文件系统 22 | ## 可选的值为 "zfs" | "xfs" | "default" 23 | ## 影响获取仓库大小的方式，如果是 "default" 的话仓库大小恒为 `-1` 24 | ## 默认值是 "default" 25 | #fs = "default" 26 | 27 | ## 设置 Docker Daemon 地址 28 | ## unix local socket: unix:///var/run/docker.sock 29 | ## tcp: tcp://127.0.0.1:2375 30 | ## 默认值是 "unix:///var/run/docker.sock" 31 | #docker_endpoint = "unix:///var/run/docker.sock" 32 | 33 | ## 设置同步程序的运行时的 uid 跟 gid，会影响仓库文件的 uid 跟 gid 34 | ## 格式为 uid:gid 35 | ## 默认值是 yukid 进程的 uid 跟 gid 36 | #owner = "1000:1000" 37 | 38 | ## 设置 yukid 的日志文件 39 | ## 默认值是 "/dev/stderr" 40 | #log_file = "/path/to/yukid.log" 41 | 42 | ## 设置 log level 43 | ## 可选的值为 "debug" | "info" | "warn" | "error" 44 | ## 默认值是 "info" 45 | #log_level = "info" 46 | 47 | ## 设置监听地址 48 | ## 默认值是 "127.0.0.1:9999" 49 | #listen_addr = "127.0.0.1:9999" 50 | 51 | ## 设置同步仓库的时候默认绑定的 IP 52 | ## 默认值为空，即不绑定 53 | #bind_ip = "1.2.3.4" 54 | 55 | ## 设置创建的 container 的名字前缀 56 | ## 默认值是 "syncing-" 57 | #name_prefix = "syncing-" 58 | 59 | ## 设置同步完后执行的命令 60 | ## 默认值为空 61 | #post_sync = ["/path/to/the/program"] 62 | 63 | ## 设置更新用到的 docker images 的频率 64 | ## 默认值为 "1h" 65 | #images_upgrade_interval = "1h" 66 | 67 | ## 同步超时时间，如果超过了这个时间，同步容器会被强制停止 68 | ## 支持使用 time.ParseDuration() 支持的时间格式，诸如 "10m", "1h" 等 69 | ## 如果为 0 的话则不会超时。注意修改的配置仅对新启动的同步容器生效 70 | ## 默认值为 0 71 | #sync_timeout = "48h" 72 | -------------------------------------------------------------------------------- /pkg/docker/fake/cli.go: -------------------------------------------------------------------------------- 1 | package fake 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "time" 8 | 9 | "github.com/cpuguy83/go-docker/errdefs" 10 | 11 | "github.com/ustclug/Yuki/pkg/docker" 12 | ) 13 | 14 | type Client struct { 15 | mu sync.Mutex 16 | containers map[string]docker.ContainerSummary 17 | } 18 | 19 | func (f *Client) RunContainer(ctx context.Context, config docker.RunContainerConfig) (id string, err error) { 20 | f.mu.Lock() 21 | defer f.mu.Unlock() 22 | _, ok := f.containers[config.Name] 23 | if ok { 24 | return "", errdefs.Conflict("container already exists") 25 | } 26 | f.containers[config.Name] = docker.ContainerSummary{ 27 | ID: config.Name, 28 | Labels: config.Labels, 29 | } 30 | return config.Name, nil 31 | } 32 | 33 | func (f *Client) WaitContainerWithTimeout(id string, timeout time.Duration) (int, error) { 34 | f.mu.Lock() 35 | defer f.mu.Unlock() 36 | 37 | _, ok := f.containers[id] 38 | if !ok { 39 | return 0, fmt.Errorf("container %s not found", id) 40 | } 41 | const delay = 5 * time.Second 42 | if timeout > 0 && timeout < delay { 43 | time.Sleep(timeout) 44 | return 0, context.DeadlineExceeded 45 | } 46 | time.Sleep(delay) 47 | return 0, nil 48 | } 49 | 50 | func (f *Client) RemoveContainerWithTimeout(id string, timeout time.Duration) error { 51 | f.mu.Lock() 52 | defer f.mu.Unlock() 53 | delete(f.containers, id) 54 | return nil 55 | } 56 | 57 | func (f *Client) ListContainersWithTimeout(running bool, timeout time.Duration) ([]docker.ContainerSummary, error) { 58 | f.mu.Lock() 59 | defer f.mu.Unlock() 60 | l := make([]docker.ContainerSummary, 0, len(f.containers)) 61 | for _, ct := range f.containers { 62 | l = append(l, ct) 63 | } 64 | return l, nil 65 | } 66 | 67 | func (f *Client) UpgradeImages(refs []string) error { 68 | panic("not implemented") 69 | } 70 | 71 | func NewClient() docker.Client { 72 | return &Client{ 73 | containers: make(map[string]docker.ContainerSummary), 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /pkg/yukictl/cmd/meta/ls.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "github.com/docker/go-units" 9 | "github.com/labstack/echo/v4" 10 | "github.com/spf13/cobra" 11 | 12 | "github.com/ustclug/Yuki/pkg/api" 13 | "github.com/ustclug/Yuki/pkg/tabwriter" 14 | "github.com/ustclug/Yuki/pkg/yukictl/factory" 15 | ) 16 | 17 | type lsOptions struct { 18 | name string 19 | } 20 | 21 | func (o *lsOptions) Run(f factory.Factory) error { 22 | var ( 23 | err error 24 | errMsg echo.HTTPError 25 | ) 26 | req := f.RESTClient().R().SetError(&errMsg) 27 | encoder := f.JSONEncoder(os.Stdout) 28 | if len(o.name) > 0 { 29 | var result api.GetRepoMetaResponse 30 | resp, err := req. 31 | SetResult(&result). 32 | SetPathParam("name", o.name). 33 | Get("api/v1/metas/{name}") 34 | if err != nil { 35 | return err 36 | } 37 | if resp.IsError() { 38 | return fmt.Errorf("%s", errMsg.Message) 39 | } 40 | return encoder.Encode(result) 41 | } 42 | 43 | var result api.ListRepoMetasResponse 44 | resp, err := req.SetResult(&result).Get("api/v1/metas") 45 | if err != nil { 46 | return err 47 | } 48 | if resp.IsError() { 49 | return fmt.Errorf("%s", errMsg.Message) 50 | } 51 | tw := tabwriter.New(os.Stdout) 52 | tw.SetHeader([]string{"name", "upstream", "syncing", "size", "last-success", "next-run"}) 53 | for _, r := range result { 54 | lastSuccess := "" 55 | nextRun := "" 56 | if r.LastSuccess > 0 { 57 | lastSuccess = time.Unix(r.LastSuccess, 0).Format(time.RFC3339) 58 | } 59 | if r.NextRun > 0 { 60 | nextRun = time.Unix(r.NextRun, 0).Format(time.RFC3339) 61 | } 62 | tw.Append( 63 | r.Name, 64 | r.Upstream, 65 | r.Syncing, 66 | units.BytesSize(float64(r.Size)), 67 | lastSuccess, 68 | nextRun, 69 | ) 70 | } 71 | return tw.Render() 72 | } 73 | 74 | func NewCmdMetaLs(f factory.Factory) *cobra.Command { 75 | o := lsOptions{} 76 | cmd := &cobra.Command{ 77 | Use: "ls", 78 | Short: "List one or all metadata", 79 | RunE: func(cmd *cobra.Command, args []string) error { 80 | if len(args) > 0 { 81 | o.name = args[0] 82 | } 83 | return o.Run(f) 84 | }, 85 | } 86 | return cmd 87 | } 88 | -------------------------------------------------------------------------------- /test/integration/sync_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | "time" 9 | 10 | "github.com/go-resty/resty/v2" 11 | "github.com/stretchr/testify/require" 12 | 13 | "github.com/ustclug/Yuki/pkg/api" 14 | "github.com/ustclug/Yuki/pkg/server" 15 | testutils "github.com/ustclug/Yuki/test/utils" 16 | ) 17 | 18 | func TestSyncRepo(t *testing.T) { 19 | tmpdir, err := os.MkdirTemp("", t.Name()) 20 | require.NoError(t, err) 21 | t.Cleanup(func() { 22 | _ = os.RemoveAll(tmpdir) 23 | }) 24 | logDir := filepath.Join(tmpdir, "log") 25 | cfgDir := filepath.Join(tmpdir, "config") 26 | os.MkdirAll(logDir, 0755) 27 | os.MkdirAll(cfgDir, 0755) 28 | 29 | cfg := server.DefaultConfig 30 | cfg.DbURL = filepath.Join(tmpdir, "yukid.db") 31 | cfg.RepoConfigDir = []string{cfgDir} 32 | cfg.RepoLogsDir = logDir 33 | cfg.ListenAddr = "127.0.0.1:0" 34 | srv, err := server.NewWithConfig(cfg) 35 | require.NoError(t, err) 36 | ctx, cancel := context.WithCancel(context.Background()) 37 | defer cancel() 38 | go func() { 39 | err := srv.Start(ctx) 40 | if err != nil { 41 | t.Errorf("Fail to start server: %v", err) 42 | } 43 | cancel() 44 | }() 45 | 46 | testutils.WriteFile(t, filepath.Join(cfgDir, "foo.yaml"), ` 47 | name: "foo" 48 | cron: "@every 1h" 49 | image: "ustcmirror/test:latest" 50 | storageDir: "/tmp" 51 | `) 52 | 53 | time.Sleep(3 * time.Second) 54 | if t.Failed() { 55 | return 56 | } 57 | restCli := resty.New().SetBaseURL("http://" + srv.ListenAddr()) 58 | resp, err := restCli.R().Post("/api/v1/repos/foo") 59 | require.NoError(t, err) 60 | require.True(t, resp.IsSuccess(), "Unexpected response: %s", resp.Body()) 61 | 62 | resp, err = restCli.R().Post("/api/v1/repos/foo/sync") 63 | require.NoError(t, err) 64 | require.True(t, resp.IsSuccess(), "Unexpected response: %s", resp.Body()) 65 | 66 | var meta api.GetRepoMetaResponse 67 | testutils.PollUntilTimeout(t, 5*time.Minute, func() bool { 68 | resp, err = restCli.R().SetResult(&meta).Get("/api/v1/metas/foo") 69 | require.NoError(t, err) 70 | require.True(t, resp.IsSuccess(), "Unexpected response: %s", resp.Body()) 71 | if !meta.Syncing { 72 | return true 73 | } 74 | t.Log("Waiting for syncing to finish") 75 | return false 76 | }) 77 | } 78 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | before: 3 | hooks: 4 | - go mod tidy 5 | builds: 6 | - id: yukid 7 | binary: yukid 8 | env: 9 | - CGO_ENABLED=0 10 | main: ./cmd/yukid 11 | goos: 12 | - linux 13 | goarch: 14 | - amd64 15 | flags: 16 | - -trimpath 17 | ldflags: 18 | - -s -w -X github.com/ustclug/Yuki/pkg/info.Version={{.Version}} -X github.com/ustclug/Yuki/pkg/info.BuildDate={{.Date}} -X github.com/ustclug/Yuki/pkg/info.GitCommit={{.Commit}} 19 | - id: yukictl 20 | binary: yukictl 21 | env: 22 | - CGO_ENABLED=0 23 | main: ./cmd/yukictl 24 | goos: 25 | - linux 26 | goarch: 27 | - amd64 28 | flags: 29 | - -trimpath 30 | ldflags: 31 | - -s -w -X github.com/ustclug/Yuki/pkg/info.Version={{.Version}} -X github.com/ustclug/Yuki/pkg/info.BuildDate={{.Date}} -X github.com/ustclug/Yuki/pkg/info.GitCommit={{.Commit}} 32 | hooks: 33 | post: 34 | - cmd: sh -c "{{ .Path }} completion bash > bash_completion" 35 | dir: "{{ dir (dir .Path) }}" 36 | output: true 37 | archives: 38 | - format: binary 39 | name_template: "{{ .Binary }}_{{ .Os }}_{{ .Arch }}" 40 | checksum: 41 | name_template: 'checksums.txt' 42 | snapshot: 43 | name_template: "{{ incpatch .Version }}-next" 44 | changelog: 45 | use: github-native 46 | nfpms: 47 | - id: default 48 | package_name: yuki 49 | homepage: https://github.com/ustclug/Yuki 50 | maintainer: "USTC LUG " 51 | description: |- 52 | USTC Mirror Manager 53 | formats: 54 | - deb 55 | umask: 0o022 56 | dependencies: 57 | - "docker.io | docker-engine | docker-ce" 58 | section: admin 59 | priority: extra 60 | provides: 61 | - yukid 62 | - yukictl 63 | scripts: 64 | postinstall: etc/postinst.sh 65 | preremove: etc/prerm.sh 66 | contents: 67 | - src: dist/bash_completion 68 | dst: /etc/bash_completion.d/yukictl 69 | - src: etc/daemon.example.toml 70 | dst: /etc/yuki/ 71 | - src: etc/yukid.service 72 | dst: /lib/systemd/system/ 73 | - src: yukictl 74 | dst: /usr/bin/yuki 75 | type: symlink 76 | 77 | # modelines, feel free to remove those if you don't want/use them: 78 | # yaml-language-server: $schema=https://goreleaser.com/static/schema.json 79 | # vim: set ts=2 sw=2 tw=0 fo=cnqoj 80 | -------------------------------------------------------------------------------- /pkg/server/main_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "crypto/rand" 6 | "encoding/base64" 7 | "log/slog" 8 | "net/http/httptest" 9 | "os" 10 | "testing" 11 | 12 | "github.com/glebarez/sqlite" 13 | "github.com/go-playground/validator/v10" 14 | "github.com/go-resty/resty/v2" 15 | "github.com/labstack/echo/v4" 16 | "github.com/labstack/echo/v4/middleware" 17 | cmap "github.com/orcaman/concurrent-map/v2" 18 | "github.com/robfig/cron/v3" 19 | "github.com/stretchr/testify/require" 20 | "gorm.io/gorm" 21 | 22 | fakedocker "github.com/ustclug/Yuki/pkg/docker/fake" 23 | "github.com/ustclug/Yuki/pkg/fs" 24 | "github.com/ustclug/Yuki/pkg/model" 25 | ) 26 | 27 | type TestEnv struct { 28 | t *testing.T 29 | httpSrv *httptest.Server 30 | server *Server 31 | } 32 | 33 | func (t *TestEnv) RESTClient() *resty.Client { 34 | return resty.New().SetBaseURL(t.httpSrv.URL + "/api/v1") 35 | } 36 | 37 | func (t *TestEnv) RandomString() string { 38 | var buf [6]byte 39 | _, _ = rand.Read(buf[:]) 40 | suffix := base64.RawURLEncoding.EncodeToString(buf[:]) 41 | return t.t.Name() + suffix 42 | } 43 | 44 | func NewTestEnv(t *testing.T) *TestEnv { 45 | slogger := newSlogger(os.Stderr, true, slog.LevelInfo) 46 | 47 | e := echo.New() 48 | e.HideBanner = true 49 | e.HidePort = true 50 | v := validator.New() 51 | e.Validator = echoValidator(v.Struct) 52 | 53 | dbFile, err := os.CreateTemp("", "yukid*.db") 54 | require.NoError(t, err) 55 | t.Cleanup(func() { 56 | _ = dbFile.Close() 57 | _ = os.Remove(dbFile.Name()) 58 | }) 59 | // Switch to WAL mode to avoid "database is locked" error. 60 | db, err := gorm.Open(sqlite.Open(dbFile.Name()), &gorm.Config{ 61 | QueryFields: true, 62 | }) 63 | require.NoError(t, err) 64 | require.NoError(t, model.AutoMigrate(db)) 65 | 66 | s := &Server{ 67 | e: e, 68 | db: db, 69 | logger: slogger, 70 | dockerCli: fakedocker.NewClient(), 71 | getSize: fs.New(fs.DEFAULT).GetSize, 72 | 73 | repoSchedules: cmap.New[cron.Schedule](), 74 | } 75 | s.e.Use(setLogger(slogger)) 76 | s.e.Use(middleware.RequestLoggerWithConfig(middleware.RequestLoggerConfig{ 77 | LogStatus: true, 78 | LogMethod: true, 79 | LogURI: true, 80 | LogValuesFunc: func(c echo.Context, v middleware.RequestLoggerValues) error { 81 | l := getLogger(c) 82 | l.LogAttrs(context.Background(), slog.LevelInfo, "REQUEST", slog.Int("status", v.Status)) 83 | return nil 84 | }, 85 | })) 86 | s.registerAPIs(e) 87 | srv := httptest.NewServer(e) 88 | return &TestEnv{ 89 | t: t, 90 | httpSrv: srv, 91 | server: s, 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '45 5 * * 3' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v6 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v4 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 52 | 53 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 54 | # If this step fails, then you should remove it and run the build manually (see below) 55 | - name: Autobuild 56 | uses: github/codeql-action/autobuild@v4 57 | 58 | # ℹ️ Command-line programs to run using the OS shell. 59 | # 📚 https://git.io/JvXDl 60 | 61 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 62 | # and modify them (or add more) to build your code if your project 63 | # uses a compiled language 64 | 65 | #- run: | 66 | # make bootstrap 67 | # make release 68 | 69 | - name: Perform CodeQL Analysis 70 | uses: github/codeql-action/analyze@v4 71 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ustclug/Yuki 2 | 3 | go 1.24.0 4 | 5 | require ( 6 | github.com/cpuguy83/go-docker v0.4.0 7 | github.com/docker/go-units v0.5.0 8 | github.com/glebarez/sqlite v1.11.0 9 | github.com/go-playground/validator/v10 v10.30.0 10 | github.com/go-resty/resty/v2 v2.17.1 11 | github.com/labstack/echo/v4 v4.14.0 12 | github.com/orcaman/concurrent-map/v2 v2.0.1 13 | github.com/robfig/cron/v3 v3.0.1 14 | github.com/spf13/cobra v1.10.2 15 | github.com/spf13/pflag v1.0.10 16 | github.com/spf13/viper v1.21.0 17 | github.com/stretchr/testify v1.11.1 18 | golang.org/x/sync v0.19.0 19 | gorm.io/gorm v1.31.1 20 | sigs.k8s.io/yaml v1.6.0 21 | ) 22 | 23 | require ( 24 | github.com/Microsoft/go-winio v0.6.2 // indirect 25 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 26 | github.com/dustin/go-humanize v1.0.1 // indirect 27 | github.com/fsnotify/fsnotify v1.9.0 // indirect 28 | github.com/gabriel-vasile/mimetype v1.4.12 // indirect 29 | github.com/glebarez/go-sqlite v1.21.2 // indirect 30 | github.com/go-playground/locales v0.14.1 // indirect 31 | github.com/go-playground/universal-translator v0.18.1 // indirect 32 | github.com/go-viper/mapstructure/v2 v2.4.0 // indirect 33 | github.com/google/uuid v1.6.0 // indirect 34 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 35 | github.com/jinzhu/inflection v1.0.0 // indirect 36 | github.com/jinzhu/now v1.1.5 // indirect 37 | github.com/labstack/gommon v0.4.2 // indirect 38 | github.com/leodido/go-urn v1.4.0 // indirect 39 | github.com/mattn/go-colorable v0.1.14 // indirect 40 | github.com/mattn/go-isatty v0.0.20 // indirect 41 | github.com/pelletier/go-toml/v2 v2.2.4 // indirect 42 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 43 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect 44 | github.com/sagikazarmark/locafero v0.11.0 // indirect 45 | github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect 46 | github.com/spf13/afero v1.15.0 // indirect 47 | github.com/spf13/cast v1.10.0 // indirect 48 | github.com/subosito/gotenv v1.6.0 // indirect 49 | github.com/valyala/bytebufferpool v1.0.0 // indirect 50 | github.com/valyala/fasttemplate v1.2.2 // indirect 51 | go.yaml.in/yaml/v2 v2.4.2 // indirect 52 | go.yaml.in/yaml/v3 v3.0.4 // indirect 53 | golang.org/x/crypto v0.46.0 // indirect 54 | golang.org/x/net v0.48.0 // indirect 55 | golang.org/x/sys v0.39.0 // indirect 56 | golang.org/x/text v0.32.0 // indirect 57 | golang.org/x/time v0.14.0 // indirect 58 | gopkg.in/yaml.v3 v3.0.1 // indirect 59 | modernc.org/libc v1.22.5 // indirect 60 | modernc.org/mathutil v1.5.0 // indirect 61 | modernc.org/memory v1.5.0 // indirect 62 | modernc.org/sqlite v1.23.1 // indirect 63 | ) 64 | -------------------------------------------------------------------------------- /pkg/fs/fs.go: -------------------------------------------------------------------------------- 1 | // Package fs implements function for getting the size of a given directory. 2 | package fs 3 | 4 | import ( 5 | "bufio" 6 | "bytes" 7 | "fmt" 8 | "os" 9 | "os/exec" 10 | "path" 11 | "strconv" 12 | "strings" 13 | ) 14 | 15 | // Type represents different kinds of file system. 16 | type Type byte 17 | 18 | const ( 19 | // DEFAULT represents the default file system. Always return -1 as size. 20 | DEFAULT Type = iota 21 | // XFS is the XFS file system. Getting the size by running `sudo -n xfs_quota -c "quota -pN $name"`. 22 | XFS 23 | // ZFS is the ZFS file system. Getting the size by running `df -B1 --output=used`. 24 | ZFS 25 | ) 26 | 27 | // GetSizer is the interface that wraps the `GetSize` method. 28 | type GetSizer interface { 29 | GetSize(string) int64 30 | } 31 | 32 | // New returns different `GetSizer` depending on the passed `Type`. 33 | func New(ty Type) GetSizer { 34 | switch ty { 35 | case XFS: 36 | return &xfs{} 37 | case ZFS: 38 | return &zfs{} 39 | default: 40 | return &defaultFs{} 41 | } 42 | } 43 | 44 | type defaultFs struct{} 45 | 46 | func (f *defaultFs) GetSize(d string) int64 { 47 | return -1 48 | } 49 | 50 | type zfs struct{} 51 | 52 | func getMountSource(d string) (string, error) { 53 | if !dirExists(d) { 54 | return "", os.ErrNotExist 55 | } 56 | var buf bytes.Buffer 57 | cmd := exec.Command("df", "--output=source", d) 58 | cmd.Stdout = &buf 59 | if err := cmd.Run(); err != nil { 60 | return "", err 61 | } 62 | scanner := bufio.NewScanner(&buf) 63 | scanner.Scan() 64 | scanner.Scan() 65 | return strings.TrimSpace(scanner.Text()), nil 66 | } 67 | 68 | func (f *zfs) GetSize(d string) int64 { 69 | if !dirExists(d) { 70 | return -1 71 | } 72 | src, err := getMountSource(d) 73 | if err != nil { 74 | return -1 75 | } 76 | var buf bytes.Buffer 77 | cmd := exec.Command("zfs", "get", "-H", "-p", "-o", "value", "logicalused", src) 78 | cmd.Stdout = &buf 79 | if err := cmd.Run(); err != nil { 80 | return -1 81 | } 82 | scanner := bufio.NewScanner(&buf) 83 | scanner.Scan() 84 | bs, err := strconv.ParseInt(scanner.Text(), 10, 64) 85 | if err != nil { 86 | return -1 87 | } 88 | return bs 89 | } 90 | 91 | type xfs struct{} 92 | 93 | func (f *xfs) GetSize(d string) int64 { 94 | if !dirExists(d) { 95 | return -1 96 | } 97 | 98 | var buf bytes.Buffer 99 | var kbs int64 100 | var err error 101 | name := path.Base(d) 102 | cmd := exec.Command("sudo", "-n", "xfs_quota", "-c", fmt.Sprintf("quota -pN %s", name)) 103 | cmd.Stdout = &buf 104 | if err := cmd.Run(); err != nil { 105 | return -1 106 | } 107 | scanner := bufio.NewScanner(&buf) 108 | scanner.Scan() 109 | fields := strings.Fields(scanner.Text()) 110 | switch { 111 | case len(fields) == 0: 112 | return -1 113 | case len(fields) >= 2: 114 | kbs, err = strconv.ParseInt(fields[1], 10, 64) 115 | default: 116 | scanner.Scan() 117 | fields = strings.Fields(scanner.Text()) 118 | kbs, err = strconv.ParseInt(fields[0], 10, 64) 119 | } 120 | if err != nil { 121 | return -1 122 | } 123 | return kbs * 1024 124 | } 125 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | README 2 | ======= 3 | 4 | [![Presubmit Checks](https://github.com/ustclug/Yuki/actions/workflows/pr-presubmit-checks.yml/badge.svg)](https://github.com/ustclug/Yuki/actions/workflows/pr-presubmit-checks.yml) 5 | [![Go Report](https://goreportcard.com/badge/github.com/ustclug/Yuki)](https://goreportcard.com/report/github.com/ustclug/Yuki) 6 | 7 | - [Requirements](#requirements) 8 | - [Quickstart](#quickstart) 9 | - [Handbook](#handbook) 10 | - [Development](#development) 11 | 12 | Sync local repositories with remote. 13 | 14 | ## Requirements 15 | 16 | * Docker 17 | * SQLite 18 | 19 | ## Quickstart 20 | 21 | ### Setup 22 | 23 | #### Debian and Ubuntu 24 | 25 | Download `yuki_*_amd64.deb` from the [latest release][latest-release] and install it: 26 | 27 | [latest-release]: https://github.com/ustclug/Yuki/releases/latest 28 | 29 | ```shell 30 | # Using v0.6.1 for example 31 | wget https://github.com/ustclug/Yuki/releases/download/v0.6.1/yuki_0.6.1_amd64.deb 32 | sudo dpkg -i yuki_0.6.1_amd64.deb 33 | ``` 34 | 35 | Copy `/etc/yuki/daemon.example.toml` to `/etc/yuki/daemon.toml` and edit accordingly. 36 | 37 | Create the `mirror` user and start the system service: 38 | 39 | ```shell 40 | sudo useradd -m mirror 41 | sudo systemctl enable --now yukid.service 42 | ``` 43 | 44 | #### Other distros 45 | 46 | Download the binaries from the [latest release][latest-release]. For example: 47 | 48 | ```bash 49 | wget https://github.com/ustclug/Yuki/releases/latest/download/yukictl_linux_amd64 50 | wget https://github.com/ustclug/Yuki/releases/latest/download/yukid_linux_amd64 51 | 52 | sudo cp yukictl_linux_amd64 /usr/local/bin/yukictl 53 | sudo cp yukid_linux_amd64 /usr/local/bin/yukid 54 | sudo chmod +x /usr/local/bin/{yukid,yukictl} 55 | ``` 56 | 57 | Configure yukid: 58 | 59 | ```bash 60 | sudo mkdir /etc/yuki/ 61 | sudo useradd -m mirror 62 | mkdir /tmp/repo-logs/ /tmp/repo-configs/ 63 | 64 | cat < /tmp/repo-configs/docker-ce.yaml 92 | name: docker-ce 93 | # every 1 hour 94 | cron: "0 * * * *" 95 | storageDir: /tmp/repo-data/docker-ce 96 | image: ustcmirror/rsync:latest 97 | logRotCycle: 2 98 | envs: 99 | RSYNC_HOST: rsync.mirrors.ustc.edu.cn 100 | RSYNC_PATH: docker-ce/ 101 | RSYNC_EXCLUDE: --exclude=.~tmp~/ 102 | RSYNC_EXTRA: --size-only 103 | RSYNC_MAXDELETE: "50000" 104 | EOF 105 | 106 | yukictl reload 107 | # Verify 108 | yukictl repo ls 109 | 110 | # Trigger synchronization immediately 111 | yukictl sync docker-ce 112 | ``` 113 | 114 | For more details of the configuration file, please refer to the [yukid handbook](./cmd/yukid/README.md). 115 | 116 | ## Handbook 117 | 118 | * [yukid](./cmd/yukid/README.md): Yuki daemon 119 | * [yukictl](./cmd/yukictl/README.md): Yuki cli 120 | 121 | ## Migration Guide 122 | 123 | ### v0.3.x -> v0.4.x 124 | 125 | For configuration: 126 | 127 | ```bash 128 | sed -i.bak 's/log_dir/repo_logs_dir/' /etc/yuki/daemon.toml 129 | # Also remember to update the `images_upgrade_interval` field in /etc/yuki/daemon.toml if it is set. 130 | 131 | sed -i.bak 's/interval/cron/' /path/to/repo/configs/*.yaml 132 | ``` 133 | 134 | For post sync hook, the environment variables that are passed to the hook script are changed: 135 | 136 | * `Dir` -> `DIR`: the directory of the repository 137 | * `Name` -> `NAME`: the name of the repository 138 | 139 | ## Development 140 | 141 | * Build `yukid`: 142 | 143 | ```shell 144 | make yukid 145 | ``` 146 | 147 | * Build `yukictl`: 148 | 149 | ```shell 150 | make yukictl 151 | ``` 152 | 153 | * Build Debian package: 154 | 155 | ```shell 156 | make deb 157 | ``` 158 | 159 | * Lint the whole project: 160 | 161 | ```shell 162 | make lint 163 | ``` 164 | -------------------------------------------------------------------------------- /cmd/yukid/README.md: -------------------------------------------------------------------------------- 1 | # yukid 2 | 3 | ### Table of Content 4 | 5 | - [yukid](#yukid) 6 | - [Table of Content](#table-of-content) 7 | - [Introduction](#introduction) 8 | - [Server Configuration](#server-configuration) 9 | - [Repo Configuration](#repo-configuration) 10 | 11 | ### Introduction 12 | 13 | yukid 是 yuki 的服务端，负责定期同步仓库，并且提供 RESTful API 用于管理。 14 | 15 | ### Server Configuration 16 | 17 | yukid 的配置，路径 `/etc/yuki/daemon.toml` 18 | 19 | ```toml 20 | ## 设置 debug 为 true 后会打开 echo web 框架的 debug 模式 21 | ## 以及在日志里输出程序里打印日志的位置 22 | #debug = true 23 | 24 | ## 设置 sqlite 数据库文件的路径 25 | ## 格式可以是文件路径或者是 url（如果需要设置特定参数的话）。例如： 26 | ## /var/run/yukid/data.db 27 | ## file:///home/fred/data.db?mode=ro&cache=private 28 | ## 参考 https://www.sqlite.org/c3ref/open.html 29 | db_url = "/path/to/yukid.db" 30 | 31 | ## 每个仓库的同步配置存放的文件夹 32 | ## 每个配置的后缀名必须是 `.yaml` 33 | ## 配置的格式参考下方 Repo Configuration 34 | repo_config_dir = ["/path/to/config-dir"] 35 | 36 | ## 设置同步日志存放的文件夹 37 | ## 默认值是 /var/log/yuki/ 38 | #repo_logs_dir = "/var/log/yuki/" 39 | 40 | ## 数据所在位置的文件系统 41 | ## 可选的值为 "zfs" | "xfs" | "default" 42 | ## 影响获取仓库大小的方式，如果是 "default" 的话仓库大小恒为 `-1` 43 | ## 默认值是 "default" 44 | #fs = "default" 45 | 46 | ## 设置 Docker Daemon 地址 47 | ## unix local socket: unix:///var/run/docker.sock 48 | ## tcp: tcp://127.0.0.1:2375 49 | ## 默认值是 "unix:///var/run/docker.sock" 50 | #docker_endpoint = "unix:///var/run/docker.sock" 51 | 52 | ## 设置同步程序的运行时的 uid 跟 gid，会影响仓库文件的 uid 跟 gid 53 | ## 格式为 uid:gid 54 | ## 默认值是 yukid 进程的 uid 跟 gid 55 | #owner = "1000:1000" 56 | 57 | ## 设置 yukid 的日志文件 58 | ## 默认值是 "/dev/stderr" 59 | #log_file = "/path/to/yukid.log" 60 | 61 | ## 设置 log level 62 | ## 可选的值为 "debug" | "info" | "warn" | "error" 63 | ## 默认值是 "info" 64 | #log_level = "info" 65 | 66 | ## 设置监听地址 67 | ## 默认值是 "127.0.0.1:9999" 68 | #listen_addr = "127.0.0.1:9999" 69 | 70 | ## 设置同步仓库的时候默认绑定的 IP 71 | ## 默认值为空，即不绑定 72 | #bind_ip = "1.2.3.4" 73 | 74 | ## 设置创建的 container 的名字前缀 75 | ## 默认值是 "syncing-" 76 | #name_prefix = "syncing-" 77 | 78 | ## 设置同步完后执行的命令 79 | ## 默认值为空 80 | #post_sync = ["/path/to/the/program"] 81 | 82 | ## 设置更新用到的 docker images 的频率 83 | ## 默认值为 "1h" 84 | #images_upgrade_interval = "1h" 85 | 86 | ## 同步超时时间，如果超过了这个时间，同步容器会被强制停止 87 | ## 支持使用 time.ParseDuration() 支持的时间格式，诸如 "10m", "1h" 等 88 | ## 如果为 0 的话则不会超时。注意修改的配置仅对新启动的同步容器生效 89 | ## 默认值为 0 90 | #sync_timeout = "48h" 91 | ``` 92 | 93 | ### Repo Configuration 94 | 95 | yukid 启动的时候只会从数据库里读取仓库的同步配置，不会读取 `repo_config_dir` 下的配置，所以如果有新增配置的话需要执行 `yukictl reload` 来把配置写到数据库中。 96 | 97 | 存放在 `repo_config_dir` 下的每个仓库的同步配置，文件名必须以 `.yaml` 结尾。 98 | 99 | 示例如下。不同的 image 需要的 envs 可参考 [这里](https://github.com/ustclug/ustcmirror-images#table-of-content)。 100 | 101 | ```yaml 102 | name: bioc # required 103 | image: ustcmirror/rsync:latest # required 104 | interval: 2 2 31 4 * # required 105 | storageDir: /srv/repo/bioc # required 106 | logRotCycle: 1 # 保留多少次同步日志 107 | bindIP: 1.2.3.4 # 同步的时候绑定的 IP，可选，默认为空 108 | network: host # 容器所属的 docker network，可选，默认为 host 109 | retry: 2 # 同步失败后的重试次数 110 | envs: # 传给同步程序的环境变量 111 | RSYNC_HOST: rsync.exmaple.com 112 | RSYNC_PATH: / 113 | RSYNC_RSH: ssh -i /home/mirror/.ssh/id_rsa 114 | RSYNC_USER: bioc-rsync 115 | $UPSTREAM: rsync://rsync.example.com/ # 可选变量，设置 yuki 显示的同步上游 116 | volumes: # 同步的时候需要挂载的 volume 117 | /etc/passwd: /etc/passwd:ro 118 | /home/mirror/.ssh: /home/mirror/.ssh:ro 119 | ``` 120 | 121 | 当存在多个目录时，配置将被字段级合并，同名字段 last win。举例： 122 | 123 | daemon.toml 124 | 125 | ```yaml 126 | repo_config_dir = ["common/", "override/"] 127 | ``` 128 | 129 | common/centos.yaml 130 | 131 | ```yaml 132 | name: centos 133 | storageDir: /srv/repo/centos/ 134 | image: ustcmirror/rsync:latest 135 | interval: 0 0 * * * 136 | envs: 137 | RSYNC_HOST: msync.centos.org 138 | RSYNC_PATH: CentOS/ 139 | logRotCycle: 10 140 | retry: 1 141 | ``` 142 | 143 | override/centos.yaml 144 | 145 | ```yaml 146 | interval: 17 3-23/4 * * * 147 | envs: 148 | RSYNC_MAXDELETE: "200000" 149 | ``` 150 | 151 | `yukictl repo ls centos` 152 | 153 | ```json 154 | { 155 | "name": "centos", 156 | "interval": "17 3-23/4 * * *", 157 | "image": "ustcmirror/rsync:latest", 158 | "storageDir": "/srv/repo/centos/", 159 | "logRotCycle": 10, 160 | "retry": 2, 161 | "envs": { 162 | "RSYNC_HOST": "msync.centos.org", 163 | "RSYNC_MAXDELETE": "200000", 164 | "RSYNC_PATH": "CentOS/" 165 | } 166 | } 167 | ``` 168 | 169 | ### RESTful API 170 | 171 | yukid 提供的 API 参考 [`registerAPIs` 函数](../../pkg/server/main.go) 的实现。其中 `/api/v1/metas` 和 `/api/v1/metas/{name}` 是可公开访问的，可以用于搭建状态页。 172 | 173 | yukictl 也会使用这些 API 来操作 yukid。 174 | -------------------------------------------------------------------------------- /pkg/server/utils_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/ustclug/Yuki/pkg/api" 12 | "github.com/ustclug/Yuki/pkg/docker" 13 | "github.com/ustclug/Yuki/pkg/model" 14 | testutils "github.com/ustclug/Yuki/test/utils" 15 | ) 16 | 17 | func TestInitRepoMetas(t *testing.T) { 18 | te := NewTestEnv(t) 19 | require.NoError(t, te.server.db.Create([]model.Repo{ 20 | { 21 | Name: "repo0", 22 | Cron: "@every 1h", 23 | }, 24 | { 25 | Name: "repo1", 26 | Cron: "@every 1h", 27 | }, 28 | }).Error) 29 | require.NoError(t, te.server.db.Create([]model.RepoMeta{ 30 | { 31 | Name: "repo0", 32 | Size: 100, 33 | ExitCode: 0, 34 | }, 35 | }).Error) 36 | 37 | require.NoError(t, te.server.initRepoMetas()) 38 | 39 | var metas []model.RepoMeta 40 | require.NoError(t, te.server.db.Order("name").Find(&metas).Error) 41 | require.Len(t, metas, 2) 42 | require.Equal(t, int64(-1), metas[0].Size) 43 | require.Equal(t, 0, metas[0].ExitCode) 44 | 45 | require.Equal(t, int64(-1), metas[1].Size) 46 | require.Equal(t, -1, metas[1].ExitCode) 47 | } 48 | 49 | type fakeImageClient struct { 50 | docker.Client 51 | pullImage func(ctx context.Context, image string) error 52 | } 53 | 54 | func (f *fakeImageClient) UpgradeImages(refs []string) error { 55 | for _, ref := range refs { 56 | err := f.pullImage(context.Background(), ref) 57 | if err != nil { 58 | return err 59 | } 60 | } 61 | return nil 62 | } 63 | 64 | func TestUpgradeImages(t *testing.T) { 65 | te := NewTestEnv(t) 66 | var ( 67 | mu sync.Mutex 68 | pulledImages []string 69 | ) 70 | dockerCli := &fakeImageClient{ 71 | Client: te.server.dockerCli, 72 | pullImage: func(ctx context.Context, image string) error { 73 | mu.Lock() 74 | defer mu.Unlock() 75 | pulledImages = append(pulledImages, image) 76 | return nil 77 | }, 78 | } 79 | te.server.dockerCli = dockerCli 80 | 81 | require.NoError(t, te.server.db.Create([]model.Repo{ 82 | { 83 | Name: "repo0", 84 | Image: "image0", 85 | }, 86 | { 87 | Name: "repo1", 88 | Image: "image1", 89 | }, 90 | { 91 | Name: "repo2", 92 | Image: "image0", 93 | }, 94 | }).Error) 95 | te.server.upgradeImages() 96 | 97 | require.Len(t, pulledImages, 2) 98 | require.Contains(t, pulledImages, "image0") 99 | require.Contains(t, pulledImages, "image1") 100 | } 101 | 102 | func TestWaitRunningContainers(t *testing.T) { 103 | te := NewTestEnv(t) 104 | require.NoError(t, te.server.db.Create(&model.RepoMeta{ 105 | Name: "repo0", 106 | }).Error) 107 | _, err := te.server.dockerCli.RunContainer( 108 | context.TODO(), 109 | docker.RunContainerConfig{ 110 | Name: "sync-repo0", 111 | Labels: map[string]string{ 112 | api.LabelRepoName: "repo0", 113 | api.LabelStorageDir: "/data", 114 | }, 115 | }, 116 | ) 117 | require.NoError(t, err) 118 | require.NoError(t, te.server.waitRunningContainers()) 119 | 120 | meta := model.RepoMeta{ 121 | Name: "repo0", 122 | } 123 | require.NoError(t, te.server.db.First(&meta).Error) 124 | require.True(t, meta.Syncing) 125 | 126 | testutils.PollUntilTimeout(t, time.Minute, func() bool { 127 | require.NoError(t, te.server.db.First(&meta).Error) 128 | return !meta.Syncing 129 | }) 130 | } 131 | 132 | func TestWaitForSync(t *testing.T) { 133 | const name = "repo0" 134 | t.Run("last_success should be updated", func(t *testing.T) { 135 | te := NewTestEnv(t) 136 | require.NoError(t, te.server.db.Create([]model.RepoMeta{ 137 | { 138 | Name: name, 139 | Syncing: true, 140 | ExitCode: 2, 141 | Size: 3, 142 | }, 143 | }).Error) 144 | 145 | id, err := te.server.dockerCli.RunContainer(context.TODO(), docker.RunContainerConfig{ 146 | Name: name, 147 | }) 148 | require.NoError(t, err) 149 | te.server.waitForSync(name, id, "") 150 | 151 | meta := model.RepoMeta{Name: name} 152 | require.NoError(t, te.server.db.Take(&meta).Error) 153 | require.Equal(t, name, meta.Name) 154 | require.Equal(t, int64(-1), meta.Size) 155 | require.False(t, meta.Syncing) 156 | require.Empty(t, meta.ExitCode) 157 | require.NotEmpty(t, meta.LastSuccess) 158 | }) 159 | 160 | t.Run("last_success should not be updated upon sync failure", func(t *testing.T) { 161 | te := NewTestEnv(t) 162 | lastSuccess := time.Now().Add(-time.Hour * 24).Unix() 163 | require.NoError(t, te.server.db.Create([]model.RepoMeta{ 164 | { 165 | Name: name, 166 | LastSuccess: lastSuccess, 167 | ExitCode: 2, 168 | Size: 3, 169 | }, 170 | }).Error) 171 | 172 | te.server.config.SyncTimeout = time.Second 173 | id, err := te.server.dockerCli.RunContainer(context.TODO(), docker.RunContainerConfig{ 174 | Name: name, 175 | }) 176 | require.NoError(t, err) 177 | te.server.waitForSync(name, id, "") 178 | 179 | meta := model.RepoMeta{Name: name} 180 | require.NoError(t, te.server.db.Take(&meta).Error) 181 | require.Equal(t, name, meta.Name) 182 | require.Equal(t, int64(-1), meta.Size) 183 | require.False(t, meta.Syncing) 184 | require.Equal(t, -2, meta.ExitCode) 185 | require.Equal(t, lastSuccess, meta.LastSuccess) 186 | }) 187 | } 188 | -------------------------------------------------------------------------------- /pkg/server/repo_handlers_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | "time" 9 | 10 | "github.com/robfig/cron/v3" 11 | "github.com/stretchr/testify/require" 12 | 13 | "github.com/ustclug/Yuki/pkg/api" 14 | "github.com/ustclug/Yuki/pkg/model" 15 | testutils "github.com/ustclug/Yuki/test/utils" 16 | ) 17 | 18 | func TestHandlerListRepos(t *testing.T) { 19 | te := NewTestEnv(t) 20 | require.NoError(t, te.server.db.Create([]model.Repo{ 21 | { 22 | Name: te.RandomString(), 23 | StorageDir: "/data/1", 24 | }, 25 | { 26 | Name: te.RandomString(), 27 | StorageDir: "/data/2", 28 | }, 29 | }).Error) 30 | 31 | var repos api.ListReposResponse 32 | cli := te.RESTClient() 33 | resp, err := cli.R().SetResult(&repos).Get("/repos") 34 | require.NoError(t, err) 35 | require.True(t, resp.IsSuccess(), "Unexpected response: %s", resp.Body()) 36 | 37 | require.Len(t, repos, 2) 38 | require.Equal(t, "/data/2", repos[1].StorageDir) 39 | } 40 | 41 | func TestHandlerReloadAllRepos(t *testing.T) { 42 | te := NewTestEnv(t) 43 | rootDir, err := os.MkdirTemp("", t.Name()) 44 | require.NoError(t, err) 45 | t.Cleanup(func() { 46 | _ = os.RemoveAll(rootDir) 47 | }) 48 | cfgDir1 := filepath.Join(rootDir, "cfg1") 49 | cfgDir2 := filepath.Join(rootDir, "cfg2") 50 | require.NoError(t, os.Mkdir(cfgDir1, 0o755)) 51 | require.NoError(t, os.Mkdir(cfgDir2, 0o755)) 52 | te.server.config = Config{ 53 | RepoLogsDir: filepath.Join(rootDir, "logs"), 54 | RepoConfigDir: []string{"/no/such/dir", cfgDir1, cfgDir2}, 55 | } 56 | te.server.repoSchedules.Set("should-be-deleted", cron.Schedule(nil)) 57 | 58 | require.NoError(t, te.server.db.Create([]model.Repo{ 59 | { 60 | Name: "should-be-deleted", 61 | }, 62 | { 63 | Name: "repo0", 64 | Cron: "1 * * * *", 65 | }, 66 | }).Error) 67 | 68 | require.NoError(t, te.server.db.Create([]model.RepoMeta{ 69 | { 70 | Name: "should-be-deleted", 71 | }, 72 | { 73 | Name: "repo0", 74 | Upstream: "http://foo.com", 75 | }, 76 | }).Error) 77 | 78 | for i := 0; i < 2; i++ { 79 | testutils.WriteFile( 80 | t, 81 | filepath.Join(cfgDir1, fmt.Sprintf("repo%d.yaml", i)), 82 | fmt.Sprintf(` 83 | name: repo%d 84 | cron: "* * * * *" 85 | image: "alpine:latest" 86 | storageDir: /tmp 87 | `, i), 88 | ) 89 | } 90 | testutils.WriteFile(t, filepath.Join(cfgDir2, "repo0.yaml"), ` 91 | image: ubuntu 92 | envs: 93 | UPSTREAM: http://bar.com 94 | `) 95 | 96 | cli := te.RESTClient() 97 | resp, err := cli.R().Post("/repos") 98 | require.NoError(t, err) 99 | require.True(t, resp.IsSuccess(), "Unexpected response: %s", resp.Body()) 100 | 101 | require.Equal(t, 2, te.server.repoSchedules.Count()) 102 | 103 | var repos []model.Repo 104 | require.NoError(t, te.server.db.Order("name").Find(&repos).Error) 105 | require.Len(t, repos, 2) 106 | 107 | require.Equal(t, "repo0", repos[0].Name) 108 | require.Equal(t, "ubuntu", repos[0].Image) 109 | require.Equal(t, "* * * * *", repos[0].Cron) 110 | require.NotEmpty(t, repos[0].Envs) 111 | 112 | require.Equal(t, "repo1", repos[1].Name) 113 | require.Equal(t, "alpine:latest", repos[1].Image) 114 | 115 | var metas []model.RepoMeta 116 | require.NoError(t, te.server.db.Order("name").Find(&metas).Error) 117 | require.Len(t, repos, 2) 118 | require.Equal(t, "repo0", metas[0].Name) 119 | require.Equal(t, "http://bar.com", metas[0].Upstream) 120 | 121 | require.Equal(t, "repo1", metas[1].Name) 122 | } 123 | 124 | func TestHandlerSyncRepo(t *testing.T) { 125 | te := NewTestEnv(t) 126 | name := te.RandomString() 127 | require.NoError(t, te.server.db.Create(&model.Repo{ 128 | Name: name, 129 | Cron: "@every 1h", 130 | Image: "alpine:latest", 131 | StorageDir: "/data", 132 | }).Error) 133 | schedule, _ := cron.ParseStandard("@every 1h") 134 | te.server.repoSchedules.Set(name, schedule) 135 | 136 | require.NoError(t, te.server.db.Create(&model.RepoMeta{Name: name}).Error) 137 | 138 | cli := te.RESTClient() 139 | resp, err := cli.R().Post(fmt.Sprintf("/repos/%s/sync", name)) 140 | require.NoError(t, err) 141 | require.True(t, resp.IsSuccess(), "Unexpected response: %s", resp.Body()) 142 | 143 | meta := model.RepoMeta{ 144 | Name: name, 145 | } 146 | testutils.PollUntilTimeout(t, time.Minute, func() bool { 147 | require.NoError(t, te.server.db.Take(&meta).Error) 148 | return !meta.Syncing 149 | }) 150 | 151 | require.NoError(t, te.server.db.Take(&meta).Error) 152 | require.NotEmpty(t, meta.PrevRun, "PrevRun") 153 | require.Empty(t, meta.ExitCode, "ExitCode") 154 | require.NotEmpty(t, meta.LastSuccess, "LastSuccess") 155 | require.NotEmpty(t, meta.NextRun, "NextRun") 156 | } 157 | 158 | func TestHandlerRemoveRepo(t *testing.T) { 159 | te := NewTestEnv(t) 160 | name := te.RandomString() 161 | require.NoError(t, te.server.db.Create(&model.Repo{ 162 | Name: name, 163 | Cron: "@every 1h", 164 | Image: "alpine:latest", 165 | StorageDir: "/data", 166 | }).Error) 167 | require.NoError(t, te.server.db.Create(&model.RepoMeta{Name: name}).Error) 168 | schedule, _ := cron.ParseStandard("@every 1h") 169 | te.server.repoSchedules.Set(name, schedule) 170 | 171 | cli := te.RESTClient() 172 | resp, err := cli.R().Delete(fmt.Sprintf("/repos/%s", name)) 173 | require.NoError(t, err) 174 | require.True(t, resp.IsSuccess(), "Unexpected response: %s", resp.Body()) 175 | 176 | require.False(t, te.server.repoSchedules.Has(name)) 177 | require.ErrorContains(t, te.server.db.First(&model.Repo{Name: name}).Error, "record not found") 178 | require.ErrorContains(t, te.server.db.First(&model.RepoMeta{Name: name}).Error, "record not found") 179 | 180 | resp, err = cli.R().Delete("/repos/nonexist") 181 | require.NoError(t, err) 182 | require.Equal(t, 404, resp.StatusCode(), "Removing non-exist repo does not return 404") 183 | } 184 | -------------------------------------------------------------------------------- /pkg/server/main.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "log/slog" 9 | "net/http" 10 | "os" 11 | 12 | "github.com/glebarez/sqlite" 13 | "github.com/go-playground/validator/v10" 14 | "github.com/labstack/echo/v4" 15 | "github.com/labstack/echo/v4/middleware" 16 | cmap "github.com/orcaman/concurrent-map/v2" 17 | "github.com/robfig/cron/v3" 18 | "github.com/spf13/viper" 19 | "gorm.io/gorm" 20 | 21 | "github.com/ustclug/Yuki/pkg/docker" 22 | "github.com/ustclug/Yuki/pkg/fs" 23 | "github.com/ustclug/Yuki/pkg/model" 24 | ) 25 | 26 | type Server struct { 27 | repoSchedules cmap.ConcurrentMap[string, cron.Schedule] 28 | 29 | e *echo.Echo 30 | dockerCli docker.Client 31 | config Config 32 | db *gorm.DB 33 | logger *slog.Logger 34 | getSize func(string) int64 35 | } 36 | 37 | func New(configPath string) (*Server, error) { 38 | v := viper.New() 39 | v.SetConfigFile(configPath) 40 | err := v.ReadInConfig() 41 | if err != nil { 42 | return nil, err 43 | } 44 | cfg := DefaultConfig 45 | if err := v.Unmarshal(&cfg); err != nil { 46 | return nil, err 47 | } 48 | validate := validator.New() 49 | if err := validate.Struct(&cfg); err != nil { 50 | return nil, err 51 | } 52 | return NewWithConfig(cfg) 53 | } 54 | 55 | func NewWithConfig(cfg Config) (*Server, error) { 56 | db, err := gorm.Open(sqlite.Open(cfg.DbURL), &gorm.Config{ 57 | QueryFields: true, 58 | }) 59 | if err != nil { 60 | return nil, fmt.Errorf("open db: %w", err) 61 | } 62 | 63 | dockerCli, err := docker.NewClient(cfg.DockerEndpoint) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | // workaround a systemd bug. 69 | // See also https://github.com/ustclug/Yuki/issues/4 70 | logfile, err := os.OpenFile(cfg.LogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 71 | if err != nil { 72 | return nil, err 73 | } 74 | 75 | var logLvl slog.Level 76 | switch cfg.LogLevel { 77 | case "debug": 78 | logLvl = slog.LevelDebug 79 | case "warn": 80 | logLvl = slog.LevelWarn 81 | case "error": 82 | logLvl = slog.LevelError 83 | default: 84 | logLvl = slog.LevelInfo 85 | } 86 | 87 | slogger := newSlogger(logfile, cfg.Debug, logLvl) 88 | 89 | s := Server{ 90 | e: echo.New(), 91 | db: db, 92 | logger: slogger, 93 | dockerCli: dockerCli, 94 | config: cfg, 95 | repoSchedules: cmap.New[cron.Schedule](), 96 | } 97 | switch cfg.FileSystem { 98 | case "zfs": 99 | s.getSize = fs.New(fs.ZFS).GetSize 100 | case "xfs": 101 | s.getSize = fs.New(fs.XFS).GetSize 102 | default: 103 | s.getSize = fs.New(fs.DEFAULT).GetSize 104 | } 105 | 106 | validate := validator.New() 107 | s.e.Validator = echoValidator(validate.Struct) 108 | s.e.Debug = cfg.Debug 109 | s.e.HideBanner = true 110 | s.e.Logger.SetOutput(io.Discard) 111 | 112 | // Middlewares. 113 | // The order matters. 114 | s.e.Use(middleware.RequestID()) 115 | s.e.Use(setLogger(slogger)) 116 | s.e.Use(middleware.RequestLoggerWithConfig(middleware.RequestLoggerConfig{ 117 | LogStatus: true, 118 | LogLatency: true, 119 | LogUserAgent: true, 120 | LogValuesFunc: func(c echo.Context, v middleware.RequestLoggerValues) error { 121 | attrs := []slog.Attr{ 122 | slog.Int("status", v.Status), 123 | slog.String("user_agent", v.UserAgent), 124 | slog.Duration("latency", v.Latency), 125 | } 126 | l := getLogger(c) 127 | l.LogAttrs(context.Background(), slog.LevelDebug, "REQUEST", attrs...) 128 | return nil 129 | }, 130 | })) 131 | 132 | s.registerAPIs(s.e) 133 | 134 | return &s, nil 135 | } 136 | 137 | func (s *Server) Start(rootCtx context.Context) error { 138 | l := s.logger 139 | ctx, cancel := context.WithCancelCause(rootCtx) 140 | defer cancel(context.Canceled) 141 | 142 | l.Info("Initializing database") 143 | err := model.AutoMigrate(s.db) 144 | if err != nil { 145 | return fmt.Errorf("init db: %w", err) 146 | } 147 | 148 | l.Info("Initializing repo metas") 149 | err = s.initRepoMetas() 150 | if err != nil { 151 | return fmt.Errorf("init meta: %w", err) 152 | } 153 | 154 | l.Info("Cleaning dead containers") 155 | err = s.cleanDeadContainers() 156 | if err != nil { 157 | return fmt.Errorf("clean dead containers: %w", err) 158 | } 159 | 160 | l.Info("Waiting running containers") 161 | err = s.waitRunningContainers() 162 | if err != nil { 163 | return fmt.Errorf("wait running containers: %w", err) 164 | } 165 | 166 | l.Info("Scheduling tasks") 167 | s.scheduleTasks(ctx) 168 | 169 | go func() { 170 | l.Info("Running HTTP server", slog.String("addr", s.config.ListenAddr)) 171 | if err := s.e.Start(s.config.ListenAddr); err != nil && !errors.Is(err, http.ErrServerClosed) { 172 | l.Error("Fail to run HTTP server", slogErrAttr(err)) 173 | cancel(err) 174 | } 175 | }() 176 | 177 | <-ctx.Done() 178 | l.Info("Shutting down HTTP server") 179 | _ = s.e.Shutdown(context.Background()) 180 | 181 | caused := context.Cause(ctx) 182 | if errors.Is(caused, context.Canceled) { 183 | return nil 184 | } 185 | return caused 186 | } 187 | 188 | // ListenAddr returns the actual address the server is listening on. 189 | // It is useful when the server is configured to listen on a random port. 190 | func (s *Server) ListenAddr() string { 191 | return s.e.Listener.Addr().String() 192 | } 193 | 194 | func (s *Server) registerAPIs(e *echo.Echo) { 195 | v1API := e.Group("/api/v1/") 196 | 197 | // public APIs 198 | v1API.GET("metas", s.handlerListRepoMetas) 199 | v1API.GET("metas/:name", s.handlerGetRepoMeta) 200 | 201 | // private APIs 202 | v1API.GET("repos", s.handlerListRepos) 203 | v1API.GET("repos/:name", s.handlerGetRepo) 204 | v1API.DELETE("repos/:name", s.handlerRemoveRepo) 205 | v1API.POST("repos/:name", s.handlerReloadRepo) 206 | v1API.POST("repos", s.handlerReloadAllRepos) 207 | v1API.POST("repos/:name/sync", s.handlerSyncRepo) 208 | } 209 | -------------------------------------------------------------------------------- /pkg/docker/cli.go: -------------------------------------------------------------------------------- 1 | package docker 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/cpuguy83/go-docker" 9 | "github.com/cpuguy83/go-docker/container" 10 | "github.com/cpuguy83/go-docker/container/containerapi" 11 | "github.com/cpuguy83/go-docker/container/containerapi/mount" 12 | "github.com/cpuguy83/go-docker/errdefs" 13 | "github.com/cpuguy83/go-docker/image" 14 | "github.com/cpuguy83/go-docker/image/imageapi" 15 | "github.com/cpuguy83/go-docker/transport" 16 | "golang.org/x/sync/errgroup" 17 | 18 | "github.com/ustclug/Yuki/pkg/api" 19 | ) 20 | 21 | type RunContainerConfig struct { 22 | // ContainerConfig 23 | Labels map[string]string 24 | Env []string 25 | Image string 26 | Name string 27 | 28 | // HostConfig 29 | Binds []string 30 | 31 | // NetworkingConfig 32 | Network string 33 | } 34 | 35 | type ContainerSummary struct { 36 | ID string 37 | Labels map[string]string 38 | } 39 | 40 | type Client interface { 41 | // RunContainer creates and starts a container with the given config. 42 | // The specified image will be pulled automatically if it does not exist. 43 | RunContainer(ctx context.Context, config RunContainerConfig) (id string, err error) 44 | WaitContainerWithTimeout(id string, timeout time.Duration) (int, error) 45 | RemoveContainerWithTimeout(id string, timeout time.Duration) error 46 | ListContainersWithTimeout(running bool, timeout time.Duration) ([]ContainerSummary, error) 47 | UpgradeImages(refs []string) error 48 | } 49 | 50 | func NewClient(endpoint string) (Client, error) { 51 | tr, err := transport.FromConnectionString(endpoint) 52 | if err != nil { 53 | return nil, err 54 | } 55 | return &clientImpl{ 56 | client: docker.NewClient(docker.WithTransport(tr)), 57 | }, nil 58 | } 59 | 60 | func getTimeoutContext(timeout time.Duration) (context.Context, context.CancelFunc) { 61 | if timeout == 0 { 62 | return context.WithCancel(context.Background()) 63 | } 64 | return context.WithTimeout(context.Background(), timeout) 65 | } 66 | 67 | type clientImpl struct { 68 | client *docker.Client 69 | } 70 | 71 | func (c *clientImpl) RunContainer(ctx context.Context, config RunContainerConfig) (id string, err error) { 72 | setCfg := func(cfg *container.CreateConfig) { 73 | cfg.Name = config.Name 74 | cfg.Spec.Config = containerapi.Config{ 75 | Image: config.Image, 76 | OpenStdin: true, 77 | Env: config.Env, 78 | Labels: config.Labels, 79 | } 80 | 81 | cfg.Spec.HostConfig = containerapi.HostConfig{ 82 | Binds: config.Binds, 83 | } 84 | cfg.Spec.HostConfig.Mounts = []mount.Mount{ 85 | { 86 | Type: mount.TypeTmpfs, 87 | Target: "/tmp", 88 | }, 89 | } 90 | 91 | cfg.Spec.NetworkConfig.EndpointsConfig = make(map[string]*containerapi.EndpointSettings) 92 | switch config.Network { 93 | case "host", "": 94 | cfg.Spec.HostConfig.NetworkMode = "host" 95 | default: 96 | cfg.Spec.HostConfig.NetworkMode = config.Network 97 | } 98 | } 99 | ct, err := c.client.ContainerService().Create(ctx, "", setCfg) 100 | if err != nil { 101 | if errdefs.IsNotFound(err) { 102 | err = c.pullImage(ctx, config.Image) 103 | if err != nil { 104 | return "", fmt.Errorf("pull image: %w", err) 105 | } 106 | ct, err = c.client.ContainerService().Create(ctx, "", setCfg) 107 | } 108 | if err != nil { 109 | return "", fmt.Errorf("create container: %w", err) 110 | } 111 | } 112 | err = ct.Start(ctx) 113 | if err != nil { 114 | return "", fmt.Errorf("start container: %w", err) 115 | } 116 | return ct.ID(), nil 117 | } 118 | 119 | func (c *clientImpl) ListContainersWithTimeout(running bool, timeout time.Duration) ([]ContainerSummary, error) { 120 | ctx, cancel := getTimeoutContext(timeout) 121 | defer cancel() 122 | 123 | var statuses []string 124 | if running { 125 | statuses = append(statuses, "running") 126 | } else { 127 | statuses = append(statuses, "exited", "created", "dead") 128 | } 129 | 130 | cts, err := c.client.ContainerService().List(ctx, func(config *container.ListConfig) { 131 | config.Filter = container.ListFilter{ 132 | Status: statuses, 133 | Label: []string{api.LabelRepoName}, 134 | } 135 | }) 136 | if err != nil { 137 | return nil, err 138 | } 139 | 140 | result := make([]ContainerSummary, len(cts)) 141 | for i, ct := range cts { 142 | result[i] = ContainerSummary{ 143 | ID: ct.ID, 144 | Labels: ct.Labels, 145 | } 146 | } 147 | return result, nil 148 | } 149 | 150 | func (c *clientImpl) RemoveContainerWithTimeout(id string, timeout time.Duration) error { 151 | ctx, cancel := getTimeoutContext(timeout) 152 | defer cancel() 153 | return c.client.ContainerService().Remove(ctx, id, func(cfg *container.RemoveConfig) { 154 | cfg.RemoveVolumes = true 155 | cfg.Force = true 156 | }) 157 | } 158 | 159 | func (c *clientImpl) WaitContainerWithTimeout(id string, timeout time.Duration) (int, error) { 160 | ctx, cancel := getTimeoutContext(timeout) 161 | defer cancel() 162 | ct := c.client.ContainerService().NewContainer(ctx, id) 163 | status, err := ct.Wait(ctx, container.WithWaitCondition(container.WaitConditionNotRunning)) 164 | if err != nil { 165 | return 0, fmt.Errorf("wait container: %w", err) 166 | } 167 | return status.ExitCode() 168 | } 169 | 170 | func (c *clientImpl) pullImage(ctx context.Context, ref string) error { 171 | remote, err := image.ParseRef(ref) 172 | if err != nil { 173 | return fmt.Errorf("invalid image ref: %w", err) 174 | } 175 | return c.client.ImageService().Pull(ctx, remote) 176 | } 177 | 178 | func (c *clientImpl) removeImage(id string, timeout time.Duration) error { 179 | ctx, cancel := getTimeoutContext(timeout) 180 | defer cancel() 181 | _, err := c.client.ImageService().Remove(ctx, id) 182 | return err 183 | } 184 | 185 | func (c *clientImpl) removeDanglingImages() error { 186 | images, err := c.listDanglingImages(time.Second * 5) 187 | if err != nil { 188 | return fmt.Errorf("list images: %w", err) 189 | } 190 | for _, img := range images { 191 | err = c.removeImage(img.ID, time.Second*20) 192 | if err != nil { 193 | return fmt.Errorf("remove image: %q: %w", img.ID, err) 194 | } 195 | } 196 | return nil 197 | } 198 | 199 | func (c *clientImpl) listDanglingImages(timeout time.Duration) ([]imageapi.Image, error) { 200 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 201 | defer cancel() 202 | return c.client.ImageService().List(ctx, func(cfg *image.ListConfig) { 203 | cfg.Filter = image.ListFilter{ 204 | Label: []string{api.LabelImages}, 205 | Dangling: []string{"true"}, 206 | } 207 | }) 208 | } 209 | 210 | func (c *clientImpl) UpgradeImages(refs []string) error { 211 | eg, ctx := errgroup.WithContext(context.Background()) 212 | eg.SetLimit(5) 213 | for _, ref := range refs { 214 | img := ref 215 | eg.Go(func() error { 216 | pullCtx, cancel := context.WithTimeout(ctx, time.Minute*10) 217 | defer cancel() 218 | return c.pullImage(pullCtx, img) 219 | }) 220 | } 221 | err := eg.Wait() 222 | if err != nil { 223 | return err 224 | } 225 | 226 | return c.removeDanglingImages() 227 | } 228 | -------------------------------------------------------------------------------- /pkg/server/repo_handlers.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log/slog" 7 | "net/http" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | "time" 12 | 13 | "github.com/cpuguy83/go-docker/errdefs" 14 | "github.com/cpuguy83/go-docker/image" 15 | "github.com/labstack/echo/v4" 16 | "github.com/robfig/cron/v3" 17 | "gorm.io/gorm/clause" 18 | "sigs.k8s.io/yaml" 19 | 20 | "github.com/ustclug/Yuki/pkg/api" 21 | "github.com/ustclug/Yuki/pkg/model" 22 | "github.com/ustclug/Yuki/pkg/set" 23 | ) 24 | 25 | func (s *Server) handlerListRepos(c echo.Context) error { 26 | l := getLogger(c) 27 | l.Debug("Invoked") 28 | 29 | var repos []model.Repo 30 | err := s.getDB(c). 31 | Select("name", "cron", "image", "storage_dir"). 32 | Find(&repos).Error 33 | if err != nil { 34 | const msg = "Fail to list Repos" 35 | l.Error(msg, slogErrAttr(err)) 36 | return newHTTPError(http.StatusInternalServerError, msg) 37 | } 38 | 39 | resp := make(api.ListReposResponse, len(repos)) 40 | for i, repo := range repos { 41 | resp[i] = api.ListReposResponseItem{ 42 | Name: repo.Name, 43 | Cron: repo.Cron, 44 | Image: repo.Image, 45 | StorageDir: repo.StorageDir, 46 | } 47 | } 48 | return c.JSON(http.StatusOK, resp) 49 | } 50 | 51 | func (s *Server) handlerGetRepo(c echo.Context) error { 52 | l := getLogger(c) 53 | l.Debug("Invoked") 54 | 55 | name, err := getRepoNameFromRoute(c) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | var repo model.Repo 61 | res := s.getDB(c). 62 | Where(model.Repo{Name: name}). 63 | Limit(1). 64 | Find(&repo) 65 | if res.Error != nil { 66 | const msg = "Fail to get Repo" 67 | l.Error(msg, slogErrAttr(err)) 68 | return newHTTPError(http.StatusInternalServerError, msg) 69 | } 70 | if res.RowsAffected == 0 { 71 | return newHTTPError(http.StatusNotFound, "Repo not found") 72 | } 73 | 74 | return c.JSON(http.StatusOK, repo) 75 | } 76 | 77 | func (s *Server) handlerRemoveRepo(c echo.Context) error { 78 | l := getLogger(c) 79 | l.Debug("Invoked") 80 | 81 | name, err := getRepoNameFromRoute(c) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | db := s.getDB(c) 87 | res := db.Where(model.Repo{Name: name}).Delete(&model.Repo{}) 88 | if res.Error != nil { 89 | const msg = "Fail to delete Repo" 90 | l.Error(msg, slogErrAttr(err), slog.String("repo", name)) 91 | return newHTTPError(http.StatusInternalServerError, msg) 92 | } 93 | err = db.Where(model.RepoMeta{Name: name}).Delete(&model.RepoMeta{}).Error 94 | if err != nil { 95 | l.Error("Fail to delete RepoMeta", slogErrAttr(err), slog.String("repo", name)) 96 | } 97 | s.repoSchedules.Remove(name) 98 | // Check repo existence after RepoMeta and schedule removal, to prevent inconsistency 99 | if res.RowsAffected == 0 { 100 | return newHTTPError(http.StatusNotFound, "Repo not found") 101 | } 102 | return c.NoContent(http.StatusNoContent) 103 | } 104 | 105 | func (s *Server) loadRepo(c echo.Context, logger *slog.Logger, dirs []string, file string) (*model.Repo, error) { 106 | l := logger.With(slog.String("config", file)) 107 | 108 | var repo model.Repo 109 | errn := len(dirs) 110 | for _, dir := range dirs { 111 | data, err := os.ReadFile(filepath.Join(dir, file)) 112 | if err != nil { 113 | errn-- 114 | if errn > 0 && os.IsNotExist(err) { 115 | continue 116 | } else { 117 | return nil, newHTTPError(http.StatusNotFound, fmt.Sprintf("File not found: %q", file)) 118 | } 119 | } 120 | err = yaml.Unmarshal(data, &repo) 121 | if err != nil { 122 | return nil, newHTTPError(http.StatusBadRequest, fmt.Sprintf("Fail to parse config: %q: %v", file, err)) 123 | } 124 | } 125 | 126 | err := s.e.Validator.Validate(&repo) 127 | if err != nil { 128 | return nil, newHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid config: %q: %v", file, err)) 129 | } 130 | 131 | _, err = image.ParseRef(repo.Image) 132 | if err != nil { 133 | return nil, newHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid image: %q: %v", repo.Image, err)) 134 | } 135 | 136 | schedule, err := cron.ParseStandard(repo.Cron) 137 | if err != nil { 138 | return nil, newHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid cron: %q: %v", repo.Cron, err)) 139 | } 140 | s.repoSchedules.Set(repo.Name, schedule) 141 | 142 | logDir := filepath.Join(s.config.RepoLogsDir, repo.Name) 143 | err = os.MkdirAll(logDir, 0o755) 144 | if err != nil { 145 | return nil, newHTTPError(http.StatusInternalServerError, fmt.Sprintf("Fail to create log dir: %q", logDir)) 146 | } 147 | 148 | db := s.getDB(c) 149 | err = db. 150 | Clauses(clause.OnConflict{UpdateAll: true}). 151 | Create(&repo).Error 152 | if err != nil { 153 | const msg = "Fail to create Repo" 154 | l.Error(msg, slogErrAttr(err)) 155 | return nil, newHTTPError(http.StatusInternalServerError, msg) 156 | } 157 | 158 | upstream := getUpstream(repo.Image, repo.Envs) 159 | nextRun := schedule.Next(time.Now()).Unix() 160 | err = db. 161 | Clauses(clause.OnConflict{ 162 | DoUpdates: clause.Assignments(map[string]any{ 163 | "upstream": upstream, 164 | "next_run": nextRun, 165 | }), 166 | }). 167 | Create(&model.RepoMeta{ 168 | Name: repo.Name, 169 | Upstream: upstream, 170 | Size: s.getSize(repo.StorageDir), 171 | NextRun: nextRun, 172 | }).Error 173 | if err != nil { 174 | const msg = "Fail to create RepoMeta" 175 | l.Error(msg, slogErrAttr(err)) 176 | return nil, newHTTPError(http.StatusInternalServerError, msg) 177 | } 178 | return &repo, nil 179 | } 180 | 181 | func (s *Server) handlerReloadAllRepos(c echo.Context) error { 182 | l := getLogger(c) 183 | l.Debug("Invoked") 184 | 185 | var repoNames []string 186 | db := s.getDB(c) 187 | err := db.Model(&model.Repo{}).Pluck("name", &repoNames).Error 188 | if err != nil { 189 | const msg = "Fail to list Repos" 190 | l.Error(msg, slogErrAttr(err)) 191 | return newHTTPError(http.StatusInternalServerError, msg) 192 | } 193 | 194 | l.Debug("Reloading all repos") 195 | toDelete := set.New(repoNames...) 196 | for _, dir := range s.config.RepoConfigDir { 197 | infos, err := os.ReadDir(dir) 198 | if err != nil { 199 | if !os.IsNotExist(err) { 200 | l.Warn("Fail to list dir", slogErrAttr(err), slog.String("dir", dir)) 201 | } 202 | continue 203 | } 204 | for _, info := range infos { 205 | fileName := info.Name() 206 | if info.IsDir() || fileName[0] == '.' || !strings.HasSuffix(fileName, suffixYAML) { 207 | continue 208 | } 209 | repo, err := s.loadRepo(c, l, s.config.RepoConfigDir, fileName) 210 | if err != nil { 211 | return err 212 | } 213 | toDelete.Del(repo.Name) 214 | } 215 | } 216 | 217 | toDeleteNames := toDelete.ToList() 218 | l.Debug("Deleting unnecessary repos", slog.Any("repos", toDeleteNames)) 219 | err = db.Where("name IN ?", toDeleteNames).Delete(&model.Repo{}).Error 220 | if err != nil { 221 | const msg = "Fail to delete Repos" 222 | l.Error(msg, slogErrAttr(err)) 223 | return newHTTPError(http.StatusInternalServerError, msg) 224 | } 225 | err = db.Where("name IN ?", toDeleteNames).Delete(&model.RepoMeta{}).Error 226 | if err != nil { 227 | const msg = "Fail to delete RepoMetas" 228 | l.Error(msg, slogErrAttr(err)) 229 | } 230 | for name := range toDelete { 231 | s.repoSchedules.Remove(name) 232 | } 233 | return c.NoContent(http.StatusNoContent) 234 | } 235 | 236 | func (s *Server) handlerReloadRepo(c echo.Context) error { 237 | l := getLogger(c) 238 | l.Debug("Invoked") 239 | 240 | name, err := getRepoNameFromRoute(c) 241 | if err != nil { 242 | return err 243 | } 244 | _, err = s.loadRepo(c, l.With(slog.String("repo", name)), s.config.RepoConfigDir, name+suffixYAML) 245 | if err != nil { 246 | return err 247 | } 248 | return c.NoContent(http.StatusNoContent) 249 | } 250 | 251 | func (s *Server) handlerSyncRepo(c echo.Context) error { 252 | l := getLogger(c) 253 | l.Debug("Invoked") 254 | 255 | name, err := getRepoNameFromRoute(c) 256 | if err != nil { 257 | return err 258 | } 259 | l = l.With(slog.String("repo", name)) 260 | 261 | debug := len(c.QueryParam("debug")) > 0 262 | err = s.syncRepo(c.Request().Context(), name, debug) 263 | if err != nil { 264 | if errors.Is(err, errNotFound) { 265 | return newHTTPError(http.StatusNotFound, "Repo not found") 266 | } 267 | if errdefs.IsConflict(err) { 268 | return newHTTPError(http.StatusConflict, "Repo is syncing") 269 | } 270 | const msg = "Fail to sync Repo" 271 | l.Error(msg, slogErrAttr(err)) 272 | return newHTTPError(http.StatusInternalServerError, msg) 273 | } 274 | return c.NoContent(http.StatusCreated) 275 | } 276 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= 2 | github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= 3 | github.com/cpuguy83/go-docker v0.4.0 h1:q4HTRaVdiom8I8esE2bbOiqkukew6/ZLdlKfLzpWuSI= 4 | github.com/cpuguy83/go-docker v0.4.0/go.mod h1:UCLSZgZWsumf0GPx4D7t58VQqLLyJqrCTrY9A/v5w7c= 5 | github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= 6 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 7 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= 9 | github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 10 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 11 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 12 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 13 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 14 | github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= 15 | github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= 16 | github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= 17 | github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= 18 | github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= 19 | github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= 20 | github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= 21 | github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= 22 | github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= 23 | github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 24 | github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= 25 | github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= 26 | github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= 27 | github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= 28 | github.com/go-playground/validator/v10 v10.30.0 h1:5YBPNs273uzsZJD1I8uiB4Aqg9sN6sMDVX3s6LxmhWU= 29 | github.com/go-playground/validator/v10 v10.30.0/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= 30 | github.com/go-resty/resty/v2 v2.17.1 h1:x3aMpHK1YM9e4va/TMDRlusDDoZiQ+ViDu/WpA6xTM4= 31 | github.com/go-resty/resty/v2 v2.17.1/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA= 32 | github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= 33 | github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= 34 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 35 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 36 | github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= 37 | github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= 38 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 39 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 40 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 41 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 42 | github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= 43 | github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= 44 | github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= 45 | github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= 46 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 47 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 48 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 49 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 50 | github.com/labstack/echo/v4 v4.14.0 h1:+tiMrDLxwv6u0oKtD03mv+V1vXXB3wCqPHJqPuIe+7M= 51 | github.com/labstack/echo/v4 v4.14.0/go.mod h1:xmw1clThob0BSVRX1CRQkGQ/vjwcpOMjQZSZa9fKA/c= 52 | github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= 53 | github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= 54 | github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= 55 | github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= 56 | github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= 57 | github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= 58 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 59 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 60 | github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= 61 | github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= 62 | github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= 63 | github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= 64 | github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= 65 | github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= 66 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 67 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 68 | github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= 69 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= 70 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= 71 | github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= 72 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 73 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 74 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 75 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 76 | github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= 77 | github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= 78 | github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= 79 | github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= 80 | github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= 81 | github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= 82 | github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= 83 | github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= 84 | github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= 85 | github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= 86 | github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 87 | github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= 88 | github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 89 | github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= 90 | github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= 91 | github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 92 | github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 93 | github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= 94 | github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= 95 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 96 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 97 | github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= 98 | github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= 99 | go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= 100 | go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= 101 | go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= 102 | go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= 103 | golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= 104 | golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= 105 | golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= 106 | golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= 107 | golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= 108 | golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= 109 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 110 | golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= 111 | golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 112 | golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= 113 | golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= 114 | golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= 115 | golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= 116 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 117 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 118 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 119 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 120 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 121 | gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= 122 | gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= 123 | gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= 124 | gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= 125 | modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= 126 | modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= 127 | modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= 128 | modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= 129 | modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= 130 | modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= 131 | modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= 132 | modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= 133 | sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= 134 | sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= 135 | -------------------------------------------------------------------------------- /pkg/server/utils.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "log/slog" 9 | "net/http" 10 | "os/exec" 11 | "path/filepath" 12 | "strconv" 13 | "strings" 14 | "time" 15 | 16 | "github.com/cpuguy83/go-docker/errdefs" 17 | "github.com/labstack/echo/v4" 18 | "github.com/robfig/cron/v3" 19 | "gorm.io/gorm" 20 | "gorm.io/gorm/clause" 21 | 22 | "github.com/ustclug/Yuki/pkg/api" 23 | "github.com/ustclug/Yuki/pkg/docker" 24 | "github.com/ustclug/Yuki/pkg/model" 25 | ) 26 | 27 | const suffixYAML = ".yaml" 28 | 29 | var errNotFound = errors.New("not found") 30 | 31 | func (s *Server) getDB(c echo.Context) *gorm.DB { 32 | return s.db.WithContext(c.Request().Context()) 33 | } 34 | 35 | func getRepoNameFromRoute(c echo.Context) (string, error) { 36 | val := c.Param("name") 37 | if len(val) == 0 { 38 | return "", newHTTPError(http.StatusBadRequest, "Missing required repo name") 39 | } 40 | return val, nil 41 | } 42 | 43 | func (s *Server) convertModelRepoMetaToGetMetaResponse(in model.RepoMeta) api.GetRepoMetaResponse { 44 | return api.GetRepoMetaResponse{ 45 | Name: in.Name, 46 | Upstream: in.Upstream, 47 | Syncing: in.Syncing, 48 | Size: in.Size, 49 | ExitCode: in.ExitCode, 50 | LastSuccess: in.LastSuccess, 51 | UpdatedAt: in.UpdatedAt, 52 | PrevRun: in.PrevRun, 53 | NextRun: in.NextRun, 54 | } 55 | } 56 | 57 | func slogErrAttr(err error) slog.Attr { 58 | return slog.Any("err", err) 59 | } 60 | 61 | func newHTTPError(code int, msg string) error { 62 | return &echo.HTTPError{ 63 | Code: code, 64 | Message: msg, 65 | } 66 | } 67 | 68 | func (s *Server) waitForSync(name, ctID, storageDir string) { 69 | l := s.logger.With(slog.String("repo", name)) 70 | code, err := s.dockerCli.WaitContainerWithTimeout(ctID, s.config.SyncTimeout) 71 | if err != nil { 72 | if !errors.Is(err, context.DeadlineExceeded) { 73 | l.Error("Fail to wait for container", slogErrAttr(err)) 74 | return 75 | } else { 76 | // Here we set a special exit code to indicate that the container is timeout in meta. 77 | code = -2 78 | } 79 | } 80 | err = s.dockerCli.RemoveContainerWithTimeout(ctID, time.Second*20) 81 | if err != nil { 82 | l.Error("Fail to remove container", slogErrAttr(err)) 83 | } 84 | 85 | updates := map[string]any{ 86 | "size": s.getSize(storageDir), 87 | "exit_code": code, 88 | "syncing": false, 89 | } 90 | if code == 0 { 91 | updates["last_success"] = time.Now().Unix() 92 | } 93 | 94 | err = s.db. 95 | Model(&model.RepoMeta{}). 96 | Where(model.RepoMeta{Name: name}). 97 | Updates(updates).Error 98 | if err != nil { 99 | l.Error("Fail to update RepoMeta", slogErrAttr(err)) 100 | } 101 | 102 | if len(s.config.PostSync) == 0 { 103 | return 104 | } 105 | go func() { 106 | envs := []string{ 107 | fmt.Sprintf("NAME=%s", name), 108 | fmt.Sprintf("DIR=%s", storageDir), 109 | } 110 | for _, cmd := range s.config.PostSync { 111 | prog := exec.Command("sh", "-c", cmd) 112 | prog.Env = envs 113 | output, err := prog.CombinedOutput() 114 | if err != nil { 115 | l.Error("PostSync program exit abnormally", 116 | slog.String("output", string(output)), 117 | slog.String("command", cmd), 118 | ) 119 | } 120 | } 121 | }() 122 | } 123 | 124 | func getUpstream(image string, envs model.StringMap) (upstream string) { 125 | image = strings.SplitN(image, ":", 2)[0] 126 | parts := strings.Split(image, "/") 127 | t := parts[len(parts)-1] 128 | 129 | var ok bool 130 | if upstream, ok = envs["$UPSTREAM"]; ok { 131 | return upstream 132 | } 133 | if upstream, ok = envs["UPSTREAM"]; ok { 134 | return upstream 135 | } 136 | switch t { 137 | case "archvsync", "rsync": 138 | return fmt.Sprintf("rsync://%s/%s", envs["RSYNC_HOST"], envs["RSYNC_PATH"]) 139 | case "aptsync", "apt-sync": 140 | return envs["APTSYNC_URL"] 141 | case "crates-io-index": 142 | return "https://github.com/rust-lang/crates.io-index" 143 | case "debian-cd": 144 | return fmt.Sprintf("rsync://%s/%s", envs["RSYNC_HOST"], envs["RSYNC_MODULE"]) 145 | case "docker-ce": 146 | return "https://download.docker.com/" 147 | case "fedora": 148 | remote, ok := envs["REMOTE"] 149 | if !ok { 150 | remote = "rsync://dl.fedoraproject.org" 151 | } 152 | return fmt.Sprintf("%s/%s", remote, envs["MODULE"]) 153 | case "freebsd-pkg": 154 | if upstream, ok = envs["FBSD_PKG_UPSTREAM"]; !ok { 155 | return "http://pkg.freebsd.org/" 156 | } 157 | case "freebsd-ports": 158 | if upstream, ok = envs["FBSD_PORTS_DISTFILES_UPSTREAM"]; !ok { 159 | return "http://distcache.freebsd.org/ports-distfiles/" 160 | } 161 | case "ghcup": 162 | return "https://www.haskell.org/ghcup/" 163 | case "github-release": 164 | return "https://github.com" 165 | case "gitsync": 166 | return envs["GITSYNC_URL"] 167 | case "google-repo": 168 | return "https://android.googlesource.com/mirror/manifest" 169 | case "gsutil-rsync": 170 | return envs["GS_URL"] 171 | case "hackage": 172 | if upstream, ok = envs["HACKAGE_BASE_URL"]; !ok { 173 | return "https://hackage.haskell.org/" 174 | } 175 | case "homebrew-bottles": 176 | if upstream, ok = envs["HOMEBREW_BOTTLE_DOMAIN"]; !ok { 177 | return "https://ghcr.io/v2/homebrew/" 178 | } 179 | case "julia-storage": 180 | return "https://us-east.storage.juliahub.com, https://kr.storage.juliahub.com" 181 | case "nix-channels": 182 | if upstream, ok = envs["NIX_MIRROR_UPSTREAM"]; !ok { 183 | return "https://nixos.org/channels" 184 | } 185 | case "lftpsync": 186 | return fmt.Sprintf("%s/%s", envs["LFTPSYNC_HOST"], envs["LFTPSYNC_PATH"]) 187 | case "nodesource": 188 | return "https://nodesource.com/" 189 | case "pypi": 190 | return "https://pypi.python.org/" 191 | case "rclone": 192 | remoteType := envs["RCLONE_CONFIG_REMOTE_TYPE"] 193 | path := envs["RCLONE_PATH"] 194 | domain := "" 195 | switch remoteType { 196 | case "swift": 197 | domain = envs["RCLONE_SWIFT_STORAGE_URL"] + "/" 198 | case "http": 199 | domain = envs["RCLONE_CONFIG_REMOTE_URL"] 200 | case "s3": 201 | domain = envs["RCLONE_CONFIG_REMOTE_ENDPOINT"] 202 | case "webdav": 203 | domain = envs["RCLONE_CONFIG_REMOTE_URL"] 204 | } 205 | return fmt.Sprintf("%s%s", domain, path) 206 | case "rubygems": 207 | return "http://rubygems.org/" 208 | case "stackage": 209 | upstream = "https://github.com/commercialhaskell/" 210 | case "tsumugu": 211 | return envs["UPSTREAM"] 212 | case "winget-source": 213 | if upstream, ok = envs["WINGET_REPO_URL"]; !ok { 214 | return "https://cdn.winget.microsoft.com/cache" 215 | } 216 | case "yum-sync": 217 | return envs["YUMSYNC_URL"] 218 | } 219 | return 220 | } 221 | 222 | // cleanDeadContainers removes containers which status are `created`, `exited` or `dead`. 223 | func (s *Server) cleanDeadContainers() error { 224 | cts, err := s.dockerCli.ListContainersWithTimeout(false, time.Second*10) 225 | if err != nil { 226 | return fmt.Errorf("list containers: %w", err) 227 | } 228 | 229 | for _, ct := range cts { 230 | err := s.dockerCli.RemoveContainerWithTimeout(ct.ID, time.Second*20) 231 | if err != nil { 232 | return fmt.Errorf("remove container %q: %w", ct.ID, err) 233 | } 234 | } 235 | return nil 236 | } 237 | 238 | // waitRunningContainers waits for all syncing containers to stop and remove them. 239 | func (s *Server) waitRunningContainers() error { 240 | cts, err := s.dockerCli.ListContainersWithTimeout(true, time.Second*10) 241 | if err != nil { 242 | // logger.Error("Fail to list containers", slogErrAttr(err)) 243 | return fmt.Errorf("list containers: %w", err) 244 | } 245 | for _, ct := range cts { 246 | name := ct.Labels[api.LabelRepoName] 247 | dir := ct.Labels[api.LabelStorageDir] 248 | ctID := ct.ID 249 | err := s.db. 250 | Where(model.RepoMeta{Name: name}). 251 | Updates(&model.RepoMeta{Syncing: true}). 252 | Error 253 | if err != nil { 254 | s.logger.Error("Fail to set syncing to true", slogErrAttr(err), slog.String("repo", name)) 255 | } 256 | go s.waitForSync(name, ctID, dir) 257 | } 258 | return nil 259 | } 260 | 261 | func (s *Server) upgradeImages() { 262 | db := s.db 263 | logger := s.logger 264 | logger.Debug("Upgrading images") 265 | 266 | var images []string 267 | err := db.Model(&model.Repo{}). 268 | Distinct("image"). 269 | Pluck("image", &images).Error 270 | if err != nil { 271 | logger.Error("Fail to query images", slogErrAttr(err)) 272 | return 273 | } 274 | err = s.dockerCli.UpgradeImages(images) 275 | if err != nil { 276 | logger.Error("Fail to upgrade images", slogErrAttr(err)) 277 | } 278 | } 279 | 280 | func (s *Server) scheduleTasks(ctx context.Context) { 281 | // sync repos 282 | go func() { 283 | ticker := time.NewTicker(time.Second * 10) 284 | defer ticker.Stop() 285 | for { 286 | var metas []model.RepoMeta 287 | s.db.Select("name").Where("next_run <= ?", time.Now().Unix()).Find(&metas) 288 | for _, meta := range metas { 289 | name := meta.Name 290 | l := s.logger.With(slog.String("repo", name)) 291 | err := s.syncRepo(context.Background(), name, false) 292 | if err != nil { 293 | if errdefs.IsConflict(err) { 294 | l.Warn("Still syncing") 295 | } else { 296 | l.Error("Fail to sync", slogErrAttr(err)) 297 | } 298 | } 299 | } 300 | select { 301 | case <-ctx.Done(): 302 | return 303 | case <-ticker.C: 304 | } 305 | } 306 | }() 307 | 308 | // upgrade images 309 | if s.config.ImagesUpgradeInterval > 0 { 310 | go func() { 311 | ticker := time.NewTicker(s.config.ImagesUpgradeInterval) 312 | defer ticker.Stop() 313 | for { 314 | s.upgradeImages() 315 | select { 316 | case <-ctx.Done(): 317 | return 318 | case <-ticker.C: 319 | } 320 | } 321 | }() 322 | } 323 | } 324 | 325 | func (s *Server) initRepoMetas() error { 326 | db := s.db 327 | var repos []model.Repo 328 | return db.Select("name", "storage_dir", "cron"). 329 | FindInBatches(&repos, 10, func(*gorm.DB, int) error { 330 | for _, repo := range repos { 331 | schedule, _ := cron.ParseStandard(repo.Cron) 332 | s.repoSchedules.Set(repo.Name, schedule) 333 | nextRun := schedule.Next(time.Now()).Unix() 334 | size := s.getSize(repo.StorageDir) 335 | err := db.Clauses(clause.OnConflict{ 336 | DoUpdates: clause.Assignments(map[string]any{ 337 | "size": size, 338 | "syncing": false, 339 | "next_run": nextRun, 340 | }), 341 | }).Create(&model.RepoMeta{ 342 | Name: repo.Name, 343 | Size: size, 344 | NextRun: nextRun, 345 | ExitCode: -1, 346 | }).Error 347 | if err != nil { 348 | return fmt.Errorf("init meta for repo %q: %w", repo.Name, err) 349 | } 350 | } 351 | return nil 352 | }).Error 353 | } 354 | 355 | func (s *Server) syncRepo(ctx context.Context, name string, debug bool) error { 356 | db := s.db.WithContext(ctx) 357 | var repo model.Repo 358 | res := db.Where(model.Repo{Name: name}).Limit(1).Find(&repo) 359 | if res.Error != nil { 360 | return fmt.Errorf("get repo %q: %w", name, res.Error) 361 | } 362 | if res.RowsAffected == 0 { 363 | return fmt.Errorf("get repo %q: %w", name, errNotFound) 364 | } 365 | 366 | // Update next_run unconditionally 367 | logger := s.logger.With(slog.String("repo", name)) 368 | now := time.Now() 369 | var nextRun int64 370 | schedule, ok := s.repoSchedules.Get(repo.Name) 371 | if ok { 372 | nextRun = schedule.Next(now).Unix() 373 | } else { 374 | logger.Warn("No schedule found for repo. Fallback to 1 hour") 375 | nextRun = now.Add(time.Hour).Unix() 376 | } 377 | err := db. 378 | Where(model.RepoMeta{Name: name}). 379 | Updates(&model.RepoMeta{NextRun: nextRun}).Error 380 | if err != nil { 381 | logger.Error("Fail to update next_run", slogErrAttr(err)) 382 | } 383 | 384 | if len(repo.BindIP) == 0 { 385 | repo.BindIP = s.config.BindIP 386 | } 387 | if len(repo.User) == 0 { 388 | repo.User = s.config.Owner 389 | } 390 | 391 | envMap := repo.Envs 392 | if len(envMap) == 0 { 393 | envMap = make(map[string]string) 394 | } 395 | envMap["REPO"] = repo.Name 396 | envMap["OWNER"] = repo.User 397 | if repo.BindIP != "" { 398 | envMap["BIND_ADDRESS"] = repo.BindIP 399 | } 400 | envMap["RETRY"] = strconv.Itoa(repo.Retry) 401 | envMap["LOG_ROTATE_CYCLE"] = strconv.Itoa(repo.LogRotCycle) 402 | if debug { 403 | envMap["DEBUG"] = "true" 404 | } 405 | 406 | envs := make([]string, 0, len(envMap)) 407 | for k, v := range envMap { 408 | envs = append(envs, k+"="+v) 409 | } 410 | 411 | binds := []string{ 412 | repo.StorageDir + ":/data", 413 | filepath.Join(s.config.RepoLogsDir, name) + ":/log", 414 | } 415 | for k, v := range repo.Volumes { 416 | binds = append(binds, k+":"+v) 417 | } 418 | ctName := s.config.NamePrefix + name 419 | 420 | ctID, err := s.dockerCli.RunContainer( 421 | ctx, 422 | docker.RunContainerConfig{ 423 | Labels: map[string]string{ 424 | api.LabelRepoName: repo.Name, 425 | api.LabelStorageDir: repo.StorageDir, 426 | }, 427 | Env: envs, 428 | Image: repo.Image, 429 | Name: ctName, 430 | Binds: binds, 431 | Network: repo.Network, 432 | }, 433 | ) 434 | if err != nil { 435 | return fmt.Errorf("run container: %w", err) 436 | } 437 | 438 | err = db. 439 | Where(model.RepoMeta{Name: name}). 440 | Updates(&model.RepoMeta{ 441 | PrevRun: now.Unix(), 442 | Syncing: true, 443 | }).Error 444 | if err != nil { 445 | logger.Error("Fail to update RepoMeta", slogErrAttr(err)) 446 | } 447 | go s.waitForSync(name, ctID, repo.StorageDir) 448 | 449 | return nil 450 | } 451 | 452 | func newSlogger(writer io.Writer, addSource bool, level slog.Leveler) *slog.Logger { 453 | return slog.New(slog.NewTextHandler(writer, &slog.HandlerOptions{ 454 | AddSource: addSource, 455 | Level: level, 456 | ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { 457 | // Taken from https://gist.github.com/HalCanary/6bd335057c65f3b803088cc55b9ebd2b 458 | if a.Key == slog.SourceKey { 459 | source, _ := a.Value.Any().(*slog.Source) 460 | if source != nil { 461 | _, after, _ := strings.Cut(source.File, "Yuki") 462 | source.File = after 463 | } 464 | } 465 | return a 466 | }, 467 | })) 468 | } 469 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | --------------------------------------------------------------------------------