├── .github └── workflows │ ├── build-release.yml │ └── docker-image.yml ├── .gitignore ├── Dockerfile ├── README.md ├── aiengine ├── aiengine.go ├── gemini │ ├── config.go │ ├── gemini.go │ └── requests.go └── register.go ├── capture ├── capture.go ├── config.go └── ruleapi │ ├── matcher.go │ ├── rewriter.go │ └── tester.go ├── client ├── client.go ├── config.go └── io.go ├── config ├── config.go ├── config_test.go ├── dependency_config.go ├── handler_config.go ├── log_config.go ├── plugin_config.go └── rule_config.go ├── dependency └── dependency.go ├── downloadmgr ├── .gitignore ├── download_manager.go └── download_manager_test.go ├── dynscript ├── dynscript.go ├── number_categorier.go ├── number_categorier_test.go ├── number_rewriter.go ├── number_rewriter_test.go ├── number_uncensor_checker.go └── number_uncensor_checker_test.go ├── enum └── movie_meta.go ├── face ├── constant.go ├── face_rec.go ├── goface │ ├── go_face_linux.go │ └── go_face_other.go ├── group.go └── pigo │ └── pigo.go ├── ffmpeg ├── ffmpeg.go └── ffprobe.go ├── flarerr ├── client.go ├── client_test.go └── model.go ├── go.mod ├── go.sum ├── hasher └── hasher.go ├── image ├── .gitignore ├── image.go ├── image_cutter.go ├── image_cutter_test.go ├── watermark.go └── watermark_test.go ├── main.go ├── model └── model.go ├── nfo ├── model.go ├── nfo.go └── nfo_test.go ├── number ├── constant.go ├── model.go ├── number.go └── number_test.go ├── numberkit └── numberkit.go ├── processor ├── default.go ├── group.go ├── handler │ ├── actor_split_handler.go │ ├── actor_split_handler_test.go │ ├── ai_tagger_handler.go │ ├── ai_tagger_handler_test.go │ ├── chinese_title_translate_optimizer.go │ ├── chinese_title_translate_optimizer_test.go │ ├── constant.go │ ├── duration_fixer_handler.go │ ├── duration_fixer_handler_test.go │ ├── handler.go │ ├── hd_cover_handler.go │ ├── image_transcode_handler.go │ ├── number_title_handler.go │ ├── poster_crop_handler.go │ ├── tag_padder_handler.go │ ├── tag_padder_handler_test.go │ ├── translate_handler.go │ └── watermark_handler.go └── processor.go ├── resource ├── image │ ├── 4k.png │ ├── 8k.png │ ├── README.md │ ├── hack.png │ ├── leak.png │ ├── subtitle.png │ ├── uncensored.png │ └── vr.png ├── json │ └── c_number.json.gz └── resource.go ├── scripts ├── build_archive.sh ├── download_models.sh └── install_deps.sh ├── searcher ├── category_searcher.go ├── config.go ├── decoder │ ├── config.go │ └── xpath_decoder.go ├── default_searcher.go ├── group_searcher.go ├── parser │ ├── date_parser.go │ ├── duration_parser.go │ └── duration_parser_test.go ├── plugin │ ├── api │ │ ├── api.go │ │ ├── container.go │ │ ├── defaults.go │ │ └── domain_selector.go │ ├── constant │ │ └── constant.go │ ├── factory │ │ └── factory.go │ ├── impl │ │ ├── 18av.go │ │ ├── airav │ │ │ ├── airav.go │ │ │ └── model.go │ │ ├── avsox.go │ │ ├── caribpr.go │ │ ├── cospuri.go │ │ ├── cospuri_test.go │ │ ├── fc2.go │ │ ├── fc2ppvdb.go │ │ ├── freejavbt.go │ │ ├── jav321.go │ │ ├── javbus.go │ │ ├── javbus_test.go │ │ ├── javdb.go │ │ ├── javhoo.go │ │ ├── javlibrary.go │ │ ├── jvrporn.go │ │ ├── madouqu.go │ │ ├── missav.go │ │ ├── njav.go │ │ └── tktube.go │ ├── meta │ │ └── meta.go │ ├── register │ │ └── register.go │ └── twostep │ │ ├── multilink.go │ │ └── twostep.go ├── searcher.go └── utils │ └── http_utils.go ├── store ├── mem_storage.go ├── sqlite_storage.go ├── sqlite_storate_test.go └── storage.go ├── translator ├── ai │ ├── ai_translator.go │ ├── ai_translator_test.go │ └── config.go ├── constant.go ├── google │ ├── config.go │ ├── google_translator.go │ └── google_translator_test.go ├── group.go └── translator.go └── utils ├── file_utils.go ├── name_utils.go ├── name_utils_test.go ├── nfo_utils.go ├── string_utils.go └── time_utils.go /.github/workflows/build-release.yml: -------------------------------------------------------------------------------- 1 | name: build_release 2 | 3 | on: 4 | create: 5 | tags: 6 | - 'v*' # 触发条件为以 'v' 开头的 tag 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | 12 | strategy: 13 | matrix: 14 | go-os: [windows, linux, darwin] # 可以根据需要添加或修改 15 | go-arch: [amd64] # 可以根据需要添加或修改 16 | steps: 17 | - name: Checkout code 18 | uses: actions/checkout@v2 19 | 20 | - name: Set up Go 21 | uses: actions/setup-go@v2 22 | with: 23 | go-version: 1.21 24 | 25 | - name: Build 26 | run: | 27 | sudo apt-get install libdlib-dev libblas-dev libatlas-base-dev liblapack-dev libjpeg-turbo8-dev gfortran 28 | ./scripts/build_archive.sh ${{matrix.go-os}} ${{matrix.go-arch}} ${{secrets.DOCKER_IMAGE_NAME}} 29 | 30 | - name: Upload binaries to release 31 | uses: svenstaro/upload-release-action@v2 32 | if: startsWith(github.ref, 'refs/tags/') 33 | with: 34 | repo_token: ${{ secrets.REPO_TOKEN }} 35 | file: ./*.tar.gz 36 | tag: ${{ github.ref }} 37 | file_glob: true 38 | -------------------------------------------------------------------------------- /.github/workflows/docker-image.yml: -------------------------------------------------------------------------------- 1 | name: build_tag 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Get git tag 13 | id: tag 14 | uses: dawidd6/action-get-tag@v1 15 | with: 16 | strip_v: false 17 | - name: Set up QEMU 18 | uses: docker/setup-qemu-action@v1 19 | - name: Set up Docker Buildx 20 | uses: docker/setup-buildx-action@v1 21 | - name: Login to DockerHub 22 | uses: docker/login-action@v1 23 | with: 24 | username: ${{ secrets.DOCKER_USER }} 25 | password: ${{ secrets.DOCKER_PASSWORD }} 26 | - name: Checkout 27 | uses: actions/checkout@v3 28 | with: 29 | ref: ${{steps.tag.outputs.tag}} 30 | - name: Build and push the docker image 31 | uses: docker/build-push-action@v2 32 | with: 33 | platforms: linux/amd64 34 | push: true 35 | tags: ${{ secrets.DOCKER_USER }}/${{ secrets.DOCKER_IMAGE_NAME }}:latest,${{ secrets.DOCKER_USER }}/${{ secrets.DOCKER_IMAGE_NAME }}:${{steps.tag.outputs.tag}} 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | test_data* 2 | __debug_bin* 3 | .vscode 4 | models/ 5 | yamdc -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.24 2 | 3 | RUN apt update && apt install libdlib-dev libblas-dev libatlas-base-dev liblapack-dev libjpeg62-turbo-dev gfortran -y 4 | WORKDIR /build 5 | COPY . ./ 6 | RUN CGO_LDFLAGS="-static" CGO_ENABLED=1 go build -a -tags netgo -ldflags '-w' -o yamdc ./ 7 | 8 | FROM alpine:3.13 9 | 10 | COPY --from=0 /build/yamdc /bin/ 11 | 12 | RUN apk add --no-cache ffmpeg 13 | 14 | ENTRYPOINT [ "/bin/yamdc" ] -------------------------------------------------------------------------------- /aiengine/aiengine.go: -------------------------------------------------------------------------------- 1 | package aiengine 2 | 3 | import "context" 4 | 5 | var ( 6 | defaultAIEngine IAIEngine 7 | ) 8 | 9 | type IAIEngine interface { 10 | Name() string 11 | Complete(ctx context.Context, prompt string, args map[string]interface{}) (string, error) 12 | } 13 | 14 | func SetAIEngine(engine IAIEngine) { 15 | defaultAIEngine = engine 16 | } 17 | 18 | func Complete(ctx context.Context, prompt string, args map[string]interface{}) (string, error) { 19 | return defaultAIEngine.Complete(ctx, prompt, args) 20 | } 21 | 22 | func IsAIEngineEnabled() bool { 23 | return defaultAIEngine != nil 24 | } 25 | -------------------------------------------------------------------------------- /aiengine/gemini/config.go: -------------------------------------------------------------------------------- 1 | package gemini 2 | 3 | type config struct { 4 | Key string `json:"key"` 5 | Model string `json:"model"` 6 | } 7 | 8 | type Option func(*config) 9 | 10 | func WithKey(key string) Option { 11 | return func(c *config) { 12 | c.Key = key 13 | } 14 | } 15 | 16 | func WithModel(model string) Option { 17 | return func(c *config) { 18 | c.Model = model 19 | } 20 | } 21 | 22 | func applyOpts(opts ...Option) *config { 23 | c := &config{ 24 | Model: "gemini-2.0-flash", 25 | } 26 | for _, opt := range opts { 27 | opt(c) 28 | } 29 | return c 30 | } 31 | -------------------------------------------------------------------------------- /aiengine/gemini/gemini.go: -------------------------------------------------------------------------------- 1 | package gemini 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "net/http" 9 | "strings" 10 | "yamdc/aiengine" 11 | "yamdc/client" 12 | 13 | "github.com/xxxsen/common/utils" 14 | ) 15 | 16 | const ( 17 | defaultGeminiEngineName = "gemini" 18 | ) 19 | 20 | type geminiEngine struct { 21 | c *config 22 | } 23 | 24 | func (g *geminiEngine) Name() string { 25 | return defaultGeminiEngineName 26 | } 27 | 28 | func (g *geminiEngine) Complete(ctx context.Context, prompt string, args map[string]interface{}) (string, error) { 29 | bodyRes := buildRequest(prompt, args) 30 | raw, err := json.Marshal(bodyRes) 31 | if err != nil { 32 | return "", err 33 | } 34 | req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("https://generativelanguage.googleapis.com/v1beta/models/%s:generateContent?key=%s", g.c.Model, g.c.Key), bytes.NewReader(raw)) 35 | if err != nil { 36 | return "", err 37 | } 38 | req.Header.Set("Content-Type", "application/json") 39 | req.Header.Set("Accept", "application/json") 40 | rsp, err := client.DefaultClient().Do(req) 41 | if err != nil { 42 | return "", err 43 | } 44 | defer rsp.Body.Close() 45 | if rsp.StatusCode != http.StatusOK { 46 | return "", fmt.Errorf("gemini response err, code:%d", rsp.StatusCode) 47 | } 48 | var res Response 49 | if err := json.NewDecoder(rsp.Body).Decode(&res); err != nil { 50 | return "", err 51 | } 52 | if len(res.Candidates) == 0 { 53 | return "", fmt.Errorf("no translate result found, maybe blocked, prompt feedback:%s", res.PromptFeedback.BlockReason) 54 | } 55 | if len(res.Candidates[0].Content.Parts) == 0 { 56 | return "", fmt.Errorf("no translate result part found, reason:%s", res.Candidates[0].FinishReason) 57 | } 58 | content := strings.TrimSpace(res.Candidates[0].Content.Parts[0].Text) 59 | if len(content) == 0 { 60 | return "", fmt.Errorf("no translate result text found") 61 | } 62 | return content, nil 63 | } 64 | 65 | func New(opts ...Option) (aiengine.IAIEngine, error) { 66 | c := applyOpts(opts...) 67 | return newGeminiEngine(c) 68 | } 69 | 70 | func newGeminiEngine(c *config) (*geminiEngine, error) { 71 | if c.Key == "" { 72 | return nil, fmt.Errorf("key is empty") 73 | } 74 | if c.Model == "" { 75 | return nil, fmt.Errorf("model is empty") 76 | } 77 | return &geminiEngine{c: c}, nil 78 | } 79 | 80 | func createGeminiEngine(args interface{}) (aiengine.IAIEngine, error) { 81 | c := &config{} 82 | if err := utils.ConvStructJson(args, c); err != nil { 83 | return nil, err 84 | } 85 | return newGeminiEngine(c) 86 | } 87 | 88 | func init() { 89 | aiengine.Register(defaultGeminiEngineName, createGeminiEngine) 90 | } 91 | -------------------------------------------------------------------------------- /aiengine/gemini/requests.go: -------------------------------------------------------------------------------- 1 | package gemini 2 | 3 | import "github.com/xxxsen/common/replacer" 4 | 5 | type Request struct { 6 | Contents []Content `json:"contents"` 7 | GenerationConfig GenerationConfig `json:"generationConfig"` 8 | } 9 | 10 | type GenerationConfig struct { 11 | Temperature float64 `json:"temperature"` 12 | TopK int `json:"topK"` 13 | TopP float64 `json:"topP"` 14 | } 15 | 16 | type PromptFeedback struct { 17 | BlockReason string `json:"blockReason"` 18 | } 19 | 20 | type Response struct { 21 | PromptFeedback PromptFeedback `json:"promptFeedback"` 22 | Candidates []Candidate `json:"candidates"` 23 | UsageMetadata UsageMetadata `json:"usageMetadata"` 24 | ModelVersion string `json:"modelVersion"` 25 | } 26 | 27 | type Candidate struct { 28 | Content Content `json:"content"` 29 | FinishReason string `json:"finishReason"` 30 | AvgLogprobs float64 `json:"avgLogprobs"` 31 | } 32 | 33 | type Content struct { 34 | Parts []Part `json:"parts"` 35 | Role string `json:"role"` 36 | } 37 | 38 | type Part struct { 39 | Text string `json:"text"` 40 | } 41 | 42 | type UsageMetadata struct { 43 | PromptTokenCount int `json:"promptTokenCount"` 44 | CandidatesTokenCount int `json:"candidatesTokenCount"` 45 | TotalTokenCount int `json:"totalTokenCount"` 46 | PromptTokensDetails []TokenDetail `json:"promptTokensDetails"` 47 | CandidatesTokensDetails []TokenDetail `json:"candidatesTokensDetails"` 48 | } 49 | 50 | type TokenDetail struct { 51 | Modality string `json:"modality"` 52 | TokenCount int `json:"tokenCount"` 53 | } 54 | 55 | func buildRequest(prompt string, m map[string]interface{}) *Request { 56 | res := replacer.ReplaceByMap(prompt, m) 57 | content := Content{ 58 | Parts: []Part{ 59 | { 60 | Text: res, 61 | }, 62 | }, 63 | } 64 | return &Request{ 65 | Contents: []Content{content}, 66 | GenerationConfig: GenerationConfig{ 67 | Temperature: 0, 68 | TopK: 1, 69 | TopP: 0.1, 70 | }, 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /aiengine/register.go: -------------------------------------------------------------------------------- 1 | package aiengine 2 | 3 | var ( 4 | m = make(map[string]AIEngineCreator) 5 | ) 6 | 7 | type AIEngineCreator func(args interface{}) (IAIEngine, error) 8 | 9 | func Create(name string, args interface{}) (IAIEngine, error) { 10 | if creator, ok := m[name]; ok { 11 | return creator(args) 12 | } 13 | return nil, nil 14 | } 15 | func Register(name string, creator AIEngineCreator) { 16 | if _, ok := m[name]; ok { 17 | panic("ai engine already registered") 18 | } 19 | m[name] = creator 20 | } 21 | -------------------------------------------------------------------------------- /capture/config.go: -------------------------------------------------------------------------------- 1 | package capture 2 | 3 | import ( 4 | "fmt" 5 | "yamdc/capture/ruleapi" 6 | "yamdc/processor" 7 | "yamdc/searcher" 8 | ) 9 | 10 | const ( 11 | NamingReleaseDate = "DATE" 12 | NamingReleaseYear = "YEAR" 13 | NamingReleaseMonth = "MONTH" 14 | NamingActor = "ACTOR" 15 | NamingNumber = "NUMBER" 16 | NamingTitle = "TITLE" 17 | NamingTitleTranslated = "TITLE_TRANSLATED" 18 | ) 19 | 20 | var ( 21 | defaultNamingRule = fmt.Sprintf("{%s}/{%s}", NamingReleaseYear, NamingNumber) 22 | ) 23 | 24 | type config struct { 25 | ScanDir string 26 | Searcher searcher.ISearcher 27 | Processor processor.IProcessor 28 | SaveDir string 29 | Naming string 30 | ExtraMediaExtList []string 31 | UncensorTester ruleapi.ITester 32 | NumberRewriter ruleapi.IRewriter 33 | NumberCategorier ruleapi.IMatcher 34 | DiscardTranslatedTitle bool 35 | DiscardTranslatedPlot bool 36 | LinkMode bool 37 | } 38 | 39 | type Option func(c *config) 40 | 41 | func WithScanDir(dir string) Option { 42 | return func(c *config) { 43 | c.ScanDir = dir 44 | } 45 | } 46 | 47 | func WithSaveDir(dir string) Option { 48 | return func(c *config) { 49 | c.SaveDir = dir 50 | } 51 | } 52 | 53 | func WithSeacher(ss searcher.ISearcher) Option { 54 | return func(c *config) { 55 | c.Searcher = ss 56 | } 57 | } 58 | 59 | func WithProcessor(p processor.IProcessor) Option { 60 | return func(c *config) { 61 | c.Processor = p 62 | } 63 | } 64 | 65 | func WithNamingRule(r string) Option { 66 | return func(c *config) { 67 | c.Naming = r 68 | } 69 | } 70 | 71 | func WithExtraMediaExtList(lst []string) Option { 72 | return func(c *config) { 73 | c.ExtraMediaExtList = lst 74 | } 75 | } 76 | 77 | func WithUncensorTester(t ruleapi.ITester) Option { 78 | return func(c *config) { 79 | c.UncensorTester = t 80 | } 81 | } 82 | 83 | func WithNumberRewriter(t ruleapi.IRewriter) Option { 84 | return func(c *config) { 85 | c.NumberRewriter = t 86 | } 87 | } 88 | 89 | func WithNumberCategorier(t ruleapi.IMatcher) Option { 90 | return func(c *config) { 91 | c.NumberCategorier = t 92 | } 93 | } 94 | 95 | func WithTransalteTitleDiscard(v bool) Option { 96 | return func(c *config) { 97 | c.DiscardTranslatedTitle = v 98 | } 99 | } 100 | 101 | func WithTranslatedPlotDiscard(v bool) Option { 102 | return func(c *config) { 103 | c.DiscardTranslatedPlot = v 104 | } 105 | } 106 | 107 | func WithLinkMode(v bool) Option { 108 | return func(c *config) { 109 | c.LinkMode = v 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /capture/ruleapi/matcher.go: -------------------------------------------------------------------------------- 1 | package ruleapi 2 | 3 | type MatcherFunc func(res string) (string, bool, error) 4 | 5 | type IMatcher interface { 6 | Match(res string) (string, bool, error) 7 | } 8 | 9 | type fnMatcherWrap struct { 10 | fn MatcherFunc 11 | } 12 | 13 | func WrapFuncAsMatcher(in MatcherFunc) IMatcher { 14 | return &fnMatcherWrap{fn: in} 15 | } 16 | 17 | func (f *fnMatcherWrap) Match(res string) (string, bool, error) { 18 | return f.fn(res) 19 | } 20 | -------------------------------------------------------------------------------- /capture/ruleapi/rewriter.go: -------------------------------------------------------------------------------- 1 | package ruleapi 2 | 3 | type RewriterFunc func(res string) (string, error) 4 | 5 | type IRewriter interface { 6 | Rewrite(res string) (string, error) 7 | } 8 | 9 | type fnRewriterWrap struct { 10 | fn RewriterFunc 11 | } 12 | 13 | func WrapFuncAsRewriter(in RewriterFunc) IRewriter { 14 | return &fnRewriterWrap{fn: in} 15 | } 16 | 17 | func (f *fnRewriterWrap) Rewrite(res string) (string, error) { 18 | return f.fn(res) 19 | } 20 | -------------------------------------------------------------------------------- /capture/ruleapi/tester.go: -------------------------------------------------------------------------------- 1 | package ruleapi 2 | 3 | type TesterFunc func(res string) (bool, error) 4 | 5 | type ITester interface { 6 | Test(res string) (bool, error) 7 | } 8 | 9 | type fnTesterWrap struct { 10 | fn TesterFunc 11 | } 12 | 13 | func WrapFuncAsTester(in TesterFunc) ITester { 14 | return &fnTesterWrap{fn: in} 15 | } 16 | func (f *fnTesterWrap) Test(res string) (bool, error) { 17 | return f.fn(res) 18 | } 19 | -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/cookiejar" 7 | "net/url" 8 | 9 | "github.com/imroc/req/v3" 10 | ) 11 | 12 | var defaultInst IHTTPClient 13 | 14 | func init() { 15 | SetDefault(MustNewClient()) //初始化default, 避免无初始化使用直接炸了 16 | } 17 | 18 | func SetDefault(c IHTTPClient) { 19 | defaultInst = c 20 | } 21 | 22 | func DefaultClient() IHTTPClient { 23 | return defaultInst 24 | } 25 | 26 | type IHTTPClient interface { 27 | Do(req *http.Request) (*http.Response, error) 28 | } 29 | 30 | type clientWrap struct { 31 | client *http.Client 32 | } 33 | 34 | var defaultChromeHeaders = map[string]string{ 35 | "pragma": "no-cache", 36 | "cache-control": "no-cache", 37 | "sec-ch-ua": `"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"`, 38 | "sec-ch-ua-mobile": "?0", 39 | "sec-ch-ua-platform": `"macOS"`, 40 | "upgrade-insecure-requests": "1", 41 | "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36", 42 | "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", 43 | "sec-fetch-site": "none", 44 | "sec-fetch-mode": "navigate", 45 | "sec-fetch-user": "?1", 46 | "sec-fetch-dest": "document", 47 | "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7,it;q=0.6", 48 | } 49 | 50 | func NewClient(opts ...Option) (IHTTPClient, error) { 51 | c := applyOpts(opts...) 52 | // 第三方客户端用着不是很习惯, 考虑到我们需要用到的功能都是在transport里面, 53 | // 所以这里直接把第三方客户端的transport提出来用... 54 | reqClient := req.NewClient() 55 | reqClient.ImpersonateChrome() //fixme: 部分逻辑看着, 有使用到底层的client, 但是, 貌似不使用这部分东西也能正常绕过cf? 56 | t := reqClient.Transport 57 | t.WrapRoundTripFunc(func(rt http.RoundTripper) req.HttpRoundTripFunc { 58 | return func(req *http.Request) (resp *http.Response, err error) { 59 | for k, v := range defaultChromeHeaders { 60 | req.Header.Set(k, v) 61 | } 62 | return rt.RoundTrip(req) 63 | } 64 | }) 65 | jar, _ := cookiejar.New(nil) 66 | client := &http.Client{ 67 | Transport: t, 68 | Jar: jar, 69 | Timeout: c.timeout, 70 | } 71 | if len(c.proxy) > 0 { 72 | proxyUrl, err := url.Parse(c.proxy) 73 | if err != nil { 74 | return nil, fmt.Errorf("parse proxy link failed, err:%w", err) 75 | } 76 | t.Proxy = http.ProxyURL(proxyUrl) // set proxy 77 | } 78 | return &clientWrap{client: client}, nil 79 | } 80 | 81 | func MustNewClient(opts ...Option) IHTTPClient { 82 | c, err := NewClient(opts...) 83 | if err != nil { 84 | panic(err) 85 | } 86 | return c 87 | } 88 | 89 | func (c *clientWrap) Do(req *http.Request) (*http.Response, error) { 90 | return c.client.Do(req) 91 | } 92 | -------------------------------------------------------------------------------- /client/config.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import "time" 4 | 5 | type config struct { 6 | timeout time.Duration 7 | proxy string 8 | } 9 | 10 | type Option func(c *config) 11 | 12 | func WithTimeout(t time.Duration) Option { 13 | return func(c *config) { 14 | c.timeout = t 15 | } 16 | } 17 | 18 | func WithProxy(link string) Option { 19 | return func(c *config) { 20 | c.proxy = link 21 | } 22 | } 23 | 24 | func applyOpts(opts ...Option) *config { 25 | c := &config{} 26 | for _, opt := range opts { 27 | opt(c) 28 | } 29 | if c.timeout == 0 { 30 | c.timeout = 10 * time.Second 31 | } 32 | return c 33 | } 34 | -------------------------------------------------------------------------------- /client/io.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "compress/flate" 5 | "compress/gzip" 6 | "io" 7 | "net/http" 8 | 9 | "github.com/klauspost/compress/zstd" 10 | "github.com/xxxsen/common/iotool" 11 | ) 12 | 13 | func getResponseBody(rsp *http.Response) (io.ReadCloser, error) { 14 | switch rsp.Header.Get("Content-Encoding") { 15 | case "gzip": 16 | return gzip.NewReader(rsp.Body) 17 | case "deflate": 18 | return flate.NewReader(rsp.Body), nil 19 | case "zstd": 20 | r, err := zstd.NewReader(rsp.Body) 21 | if err != nil { 22 | return nil, err 23 | } 24 | return iotool.WrapReadWriteCloser(r, nil, rsp.Body), nil 25 | default: 26 | return rsp.Body, nil 27 | } 28 | } 29 | 30 | func BuildReaderFromHTTPResponse(rsp *http.Response) (io.ReadCloser, error) { 31 | return getResponseBody(rsp) 32 | } 33 | 34 | func ReadHTTPData(rsp *http.Response) ([]byte, error) { 35 | defer rsp.Body.Close() 36 | reader, err := BuildReaderFromHTTPResponse(rsp) 37 | if err != nil { 38 | return nil, err 39 | } 40 | defer reader.Close() 41 | return io.ReadAll(reader) 42 | } 43 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/tailscale/hujson" 9 | ) 10 | 11 | const ( 12 | testData = ` 13 | { 14 | /* this is a test comment */ 15 | "a": 1, 16 | "b": 3.14, // hello? 17 | "c": true, 18 | //also comment here 19 | "d": ["a", "b"], //asdasdsadasd 20 | } 21 | ` 22 | ) 23 | 24 | type testSt struct { 25 | A int `json:"a"` 26 | B float64 `json:"b"` 27 | C bool `json:"c"` 28 | D []string `json:"d"` 29 | } 30 | 31 | func TestJsonWithComments(t *testing.T) { 32 | st := &testSt{} 33 | data, err := hujson.Standardize([]byte(testData)) 34 | assert.NoError(t, err) 35 | err = json.Unmarshal(data, st) 36 | assert.NoError(t, err) 37 | t.Logf("%+v", *st) 38 | assert.Equal(t, 1, st.A) 39 | assert.Equal(t, 3.14, st.B) 40 | assert.Equal(t, true, st.C) 41 | } 42 | -------------------------------------------------------------------------------- /config/dependency_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | var sysDependencies = []Dependency{ 4 | {Link: "https://github.com/Kagami/go-face-testdata/raw/master/models/shape_predictor_5_face_landmarks.dat", RelPath: "models/shape_predictor_5_face_landmarks.dat"}, 5 | {Link: "https://github.com/Kagami/go-face-testdata/raw/master/models/dlib_face_recognition_resnet_model_v1.dat", RelPath: "models/dlib_face_recognition_resnet_model_v1.dat"}, 6 | {Link: "https://github.com/Kagami/go-face-testdata/raw/master/models/mmod_human_face_detector.dat", RelPath: "models/mmod_human_face_detector.dat"}, 7 | {Link: "https://github.com/esimov/pigo/raw/master/cascade/facefinder", RelPath: "models/facefinder"}, 8 | } 9 | -------------------------------------------------------------------------------- /config/handler_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | var sysHandler = []string{ 4 | "hd_cover", 5 | "image_transcoder", 6 | "poster_cropper", 7 | "watermark_maker", 8 | "actor_spliter", 9 | "duration_fixer", 10 | "translater", 11 | "chinese_title_translate_optimizer", 12 | "number_title", 13 | "ai_tagger", 14 | "tag_padder", 15 | } 16 | -------------------------------------------------------------------------------- /config/log_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "github.com/xxxsen/common/logger" 4 | 5 | var sysLogConfig = logger.LogConfig{ 6 | Level: "info", 7 | Console: true, 8 | } 9 | -------------------------------------------------------------------------------- /config/plugin_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | var sysPlugins = []string{ 4 | "javbus", 5 | "javhoo", 6 | "airav", 7 | "javdb", 8 | "jav321", 9 | "javlibrary", 10 | "caribpr", 11 | "18av", 12 | "njav", 13 | "missav", 14 | "freejavbt", 15 | "tktube", 16 | "avsox", 17 | } 18 | 19 | var sysCategoryPlugins = []CategoryPlugin{ 20 | //如果存在分配配置, 那么当番号被识别为特定分类的场景下, 将会使用分类插件直接查询 21 | {Name: "FC2", Plugins: []string{"fc2", "18av", "njav", "freejavbt", "tktube", "avsox", "fc2ppvdb"}}, 22 | {Name: "JVR", Plugins: []string{"jvrporn"}}, 23 | {Name: "COSPURI", Plugins: []string{"cospuri"}}, 24 | {Name: "MD", Plugins: []string{"madouqu"}}, 25 | } 26 | -------------------------------------------------------------------------------- /config/rule_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | var sysRuleConfig = RuleConfig{ 4 | NumberRewriter: LinkConfig{ 5 | Type: "network", 6 | Link: "https://raw.githubusercontent.com/xxxsen/yamdc-script/refs/heads/master/number_rewriter.yaml", 7 | }, 8 | NumberCategorier: LinkConfig{ 9 | Type: "network", 10 | Link: "https://raw.githubusercontent.com/xxxsen/yamdc-script/refs/heads/master/number_categorier.yaml", 11 | }, 12 | NumberUncensorTester: LinkConfig{ 13 | Type: "network", 14 | Link: "https://raw.githubusercontent.com/xxxsen/yamdc-script/refs/heads/master/number_uncensor_tester.yaml", 15 | }, 16 | } 17 | -------------------------------------------------------------------------------- /dependency/dependency.go: -------------------------------------------------------------------------------- 1 | package dependency 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "time" 8 | "yamdc/client" 9 | "yamdc/downloadmgr" 10 | 11 | "github.com/xxxsen/common/logutil" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | const ( 16 | defaultSuffix = ".ts" 17 | ) 18 | 19 | type Dependency struct { 20 | URL string 21 | Target string 22 | } 23 | 24 | func Resolve(cli client.IHTTPClient, deps []*Dependency) error { 25 | m := downloadmgr.NewManager(cli) 26 | for _, dep := range deps { 27 | if err := checkAndDownload(m, dep.URL, dep.Target); err != nil { 28 | return fmt.Errorf("download link:%s to target:%s failed, err:%w", dep.URL, dep.Target, err) 29 | } 30 | } 31 | return nil 32 | } 33 | 34 | func checkAndDownload(m *downloadmgr.DownloadManager, link string, target string) error { 35 | if _, err := os.Stat(target + defaultSuffix); err == nil { 36 | return nil 37 | } 38 | logutil.GetLogger(context.Background()).Debug("start download link", zap.String("link", link)) 39 | if err := m.Download(link, target); err != nil { 40 | return err 41 | } 42 | logutil.GetLogger(context.Background()).Debug("download link succ", zap.String("link", link)) 43 | if err := os.WriteFile(target+defaultSuffix, []byte(fmt.Sprintf("%d", time.Now().Unix())), 0644); err != nil { 44 | return fmt.Errorf("write ts file failed, err:%w", err) 45 | } 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /downloadmgr/.gitignore: -------------------------------------------------------------------------------- 1 | /testdata -------------------------------------------------------------------------------- /downloadmgr/download_manager.go: -------------------------------------------------------------------------------- 1 | package downloadmgr 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "os" 8 | "path/filepath" 9 | "yamdc/client" 10 | ) 11 | 12 | type DownloadManager struct { 13 | cli client.IHTTPClient 14 | } 15 | 16 | func NewManager(cli client.IHTTPClient) *DownloadManager { 17 | return &DownloadManager{cli: cli} 18 | } 19 | 20 | func (m *DownloadManager) ensureDir(dst string) error { 21 | dir := filepath.Dir(dst) 22 | if err := os.MkdirAll(dir, 0755); err != nil { 23 | return fmt.Errorf("mkdir failed, path:%s, err:%w", dir, err) 24 | } 25 | return nil 26 | } 27 | 28 | func (m *DownloadManager) createHTTPStream(src string) (io.ReadCloser, error) { 29 | req, err := http.NewRequest(http.MethodGet, src, nil) 30 | if err != nil { 31 | return nil, err 32 | } 33 | rsp, err := m.cli.Do(req) 34 | if err != nil { 35 | return nil, fmt.Errorf("do request failed, err:%w", err) 36 | } 37 | if rsp.StatusCode != http.StatusOK { 38 | return nil, fmt.Errorf("status code:%d not ok", rsp.StatusCode) 39 | } 40 | rc, err := client.BuildReaderFromHTTPResponse(rsp) 41 | if err != nil { 42 | rsp.Body.Close() 43 | return nil, fmt.Errorf("build reader failed, err:%w", err) 44 | } 45 | return rc, nil 46 | } 47 | 48 | func (m *DownloadManager) writeToFile(rc io.Reader, dst string) error { 49 | tmp := dst + ".temp" 50 | f, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) 51 | if err != nil { 52 | return fmt.Errorf("open temp file for read failed, err:%w", err) 53 | } 54 | 55 | if _, err := io.Copy(f, rc); err != nil { 56 | _ = f.Close() 57 | return fmt.Errorf("transfer data failed, err:%w", err) 58 | } 59 | _ = f.Close() 60 | if err := os.Rename(tmp, dst); err != nil { 61 | return fmt.Errorf("unable to move file:%w", err) 62 | } 63 | return nil 64 | } 65 | 66 | func (m *DownloadManager) Download(src string, dst string) error { 67 | if err := m.ensureDir(dst); err != nil { 68 | return err 69 | } 70 | rc, err := m.createHTTPStream(src) 71 | if err != nil { 72 | return err 73 | } 74 | defer rc.Close() 75 | if err := m.writeToFile(rc, dst); err != nil { 76 | return err 77 | } 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /downloadmgr/download_manager_test.go: -------------------------------------------------------------------------------- 1 | package downloadmgr 2 | 3 | import ( 4 | "testing" 5 | "yamdc/client" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestDownloa(t *testing.T) { 11 | m := NewManager(client.MustNewClient()) 12 | err := m.Download("https://github.com/Kagami/go-face-testdata/raw/master/models/shape_predictor_5_face_landmarks.dat", "testdata/abc.dat") 13 | assert.NoError(t, err) 14 | } 15 | -------------------------------------------------------------------------------- /dynscript/dynscript.go: -------------------------------------------------------------------------------- 1 | package dynscript 2 | 3 | import "strings" 4 | 5 | func rewriteTabToSpace(input string) string { 6 | var result []string 7 | for _, line := range strings.Split(input, "\n") { 8 | i := 0 9 | for i < len(line) && line[i] == '\t' { 10 | i++ 11 | } 12 | newIndent := strings.Repeat(" ", i) 13 | result = append(result, newIndent+line[i:]) 14 | } 15 | 16 | return strings.Join(result, "\n") 17 | } 18 | -------------------------------------------------------------------------------- /dynscript/number_categorier.go: -------------------------------------------------------------------------------- 1 | package dynscript 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/xxxsen/common/logutil" 7 | "github.com/xxxsen/picker" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | type NumberCategoryFunc = func(ctx context.Context, number string) (string, bool, error) 12 | 13 | type INumberCategorier interface { 14 | Category(ctx context.Context, number string) (string, bool, error) 15 | } 16 | 17 | type numberCategorierImpl struct { 18 | pk picker.IPicker[NumberCategoryFunc] 19 | } 20 | 21 | func (n *numberCategorierImpl) Category(ctx context.Context, number string) (string, bool, error) { 22 | for _, name := range n.pk.List() { 23 | f, err := n.pk.Get(name) 24 | if err != nil { 25 | logutil.GetLogger(ctx).Error("get number category plugin failed", zap.String("name", name), zap.Error(err)) 26 | continue 27 | } 28 | category, matched, err := f(ctx, number) 29 | if err != nil { 30 | logutil.GetLogger(ctx).Error("call number category plugin failed", zap.String("name", name), zap.Error(err)) 31 | continue 32 | } 33 | if matched { 34 | return category, true, nil 35 | } 36 | } 37 | return "", false, nil 38 | } 39 | 40 | func NewNumberCategorier(rule string) (INumberCategorier, error) { 41 | rule = rewriteTabToSpace(rule) 42 | pk, err := picker.ParseData[NumberCategoryFunc]([]byte(rule), picker.YamlDecoder, picker.WithSafeFuncWrap(true)) 43 | if err != nil { 44 | return nil, err 45 | } 46 | return &numberCategorierImpl{ 47 | pk: pk, 48 | }, nil 49 | } 50 | -------------------------------------------------------------------------------- /dynscript/number_categorier_test.go: -------------------------------------------------------------------------------- 1 | package dynscript 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | var numberCategortierRule = ` 11 | plugins: 12 | - name: number_categorier 13 | define: | 14 | m := map[string][]*regexp.Regexp { 15 | "AAA": []*regexp.Regexp{ 16 | regexp.MustCompile("^AAA-.*$"), 17 | }, 18 | "JVR": []*regexp.Regexp{ 19 | regexp.MustCompile("^BBB-.*$"), 20 | }, 21 | } 22 | function: | 23 | func(ctx context.Context, number string) (string, bool, error) { 24 | for category, reList := range m { 25 | for _, re := range reList { 26 | if re.MatchString(number) { 27 | return category, true, nil 28 | } 29 | } 30 | } 31 | return "", false, nil 32 | } 33 | import: 34 | - regexp 35 | ` 36 | 37 | func TestNumberCategortier(t *testing.T) { 38 | ctr, err := NewNumberCategorier(numberCategortierRule) 39 | assert.NoError(t, err) 40 | m := map[string]string{ 41 | "AAA-123": "AAA", 42 | "BBB-456": "JVR", 43 | "CCC-789": "", 44 | } 45 | for k, v := range m { 46 | res, matched, err := ctr.Category(context.Background(), k) 47 | assert.NoError(t, err) 48 | if matched { 49 | assert.Equal(t, v, res) 50 | } else { 51 | assert.Equal(t, "", res) 52 | } 53 | } 54 | } 55 | 56 | var liveNumberCategorierRule = ` 57 | plugins: 58 | - name: basic_categorier 59 | define: | 60 | cats := map[string][]*regexp.Regexp{ 61 | "FC2": []*regexp.Regexp{ 62 | regexp.MustCompile("(?i)^FC2.*$"), 63 | }, 64 | "JVR": []*regexp.Regexp{ 65 | regexp.MustCompile("(?i)^JVR.*$"), 66 | }, 67 | "COSPURI": []*regexp.Regexp{ 68 | regexp.MustCompile("(?i)^COSPURI.*$"), 69 | }, 70 | "MD": []*regexp.Regexp{ 71 | regexp.MustCompile("(?i)^MADOU[-|_].*$"), 72 | }, 73 | } 74 | function: | 75 | func(ctx context.Context, number string) (string, bool, error) { 76 | for cat, ruleList := range cats { 77 | for _, rule := range ruleList { 78 | if rule.MatchString(number) { 79 | return cat, true, nil 80 | } 81 | } 82 | } 83 | return "", false, nil 84 | } 85 | import: 86 | - strings 87 | - regexp 88 | ` 89 | 90 | func TestLiveNumberCategorier(t *testing.T) { 91 | ctr, err := NewNumberCategorier(liveNumberCategorierRule) 92 | assert.NoError(t, err) 93 | m := map[string]string{ 94 | "fc2-ppv-1234": "FC2", 95 | "jvr-12345": "JVR", 96 | "qqqq": "", 97 | "HEYZO-12345": "", 98 | "COSPURI-Emiri-Momota-0548": "COSPURI", 99 | "COSPURI-123456": "COSPURI", 100 | "cospuri-123456": "COSPURI", 101 | "MADOU-123456": "MD", 102 | "MADOU_aaaa": "MD", 103 | "MADOU_bbbb": "MD", 104 | } 105 | for k, v := range m { 106 | res, matched, err := ctr.Category(context.Background(), k) 107 | assert.NoError(t, err) 108 | if matched { 109 | assert.Equal(t, v, res) 110 | } else { 111 | assert.Equal(t, "", res) 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /dynscript/number_rewriter.go: -------------------------------------------------------------------------------- 1 | package dynscript 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/xxxsen/common/logutil" 7 | "github.com/xxxsen/picker" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | type NumberRewriteFunc = func(ctx context.Context, number string) (string, error) 12 | 13 | type INumberRewriter interface { 14 | Rewrite(ctx context.Context, number string) (string, error) 15 | } 16 | 17 | type numberRewriterImpl struct { 18 | pk picker.IPicker[NumberRewriteFunc] 19 | } 20 | 21 | func (n *numberRewriterImpl) Rewrite(ctx context.Context, number string) (string, error) { 22 | for _, name := range n.pk.List() { 23 | f, err := n.pk.Get(name) 24 | if err != nil { 25 | logutil.GetLogger(ctx).Error("get number rewrite plugin failed", zap.String("name", name), zap.Error(err)) 26 | continue 27 | } 28 | rewrited, err := f(ctx, number) 29 | if err != nil { 30 | logutil.GetLogger(ctx).Error("call number rewrite plugin failed", zap.String("name", name), zap.Error(err)) 31 | continue 32 | } 33 | if len(rewrited) == 0 { 34 | continue 35 | } 36 | number = rewrited 37 | } 38 | return number, nil 39 | } 40 | 41 | func NewNumberRewriter(rule string) (INumberRewriter, error) { 42 | rule = rewriteTabToSpace(rule) 43 | pk, err := picker.ParseData[NumberRewriteFunc]([]byte(rule), picker.YamlDecoder, picker.WithSafeFuncWrap(true)) 44 | if err != nil { 45 | return nil, err 46 | } 47 | return &numberRewriterImpl{pk: pk}, nil 48 | } 49 | -------------------------------------------------------------------------------- /dynscript/number_rewriter_test.go: -------------------------------------------------------------------------------- 1 | package dynscript 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | ) 7 | 8 | var numberRewriterRule = ` 9 | plugins: 10 | - name: rewrite_fc2 11 | define: | 12 | re := regexp.MustCompile("(?i)^fc2[-|_]?(ppv)?[-|_]?(\\d+)([-|_].*)?$") 13 | function: | 14 | func(ctx context.Context, number string) (string, error) { 15 | number = re.ReplaceAllString(number, "FC2-PPV-$2$3") 16 | return number, nil 17 | } 18 | 19 | import: 20 | - regexp 21 | ` 22 | 23 | func TestNumberRewrite(t *testing.T) { 24 | rewriter, err := NewNumberRewriter(numberRewriterRule) 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | res, err := rewriter.Rewrite(context.Background(), "fc2ppv12345-CD1") 29 | if err != nil { 30 | t.Fatal(err) 31 | } 32 | if res != "FC2-PPV-12345-CD1" { 33 | t.Fatalf("expected FC2-PPV-12345-CD1, got %s", res) 34 | } 35 | } 36 | 37 | const ( 38 | defaultLiveRewriterRule = ` 39 | plugins: 40 | - name: to_upper_string 41 | function: | 42 | func(ctx context.Context, number string) (string, error) { 43 | return strings.ToUpper(number), nil 44 | } 45 | - name: basic_number_rewriter 46 | define: | 47 | sts := []struct{ 48 | Name string 49 | Rule *regexp.Regexp 50 | Rewrite string 51 | }{ 52 | { 53 | Name: "format fc2", 54 | Rule: regexp.MustCompile("(?i)^fc2[-|_]?(ppv)?[-|_]?(\\d+)([-|_].*)?$"), 55 | Rewrite: "FC2-PPV-$2$3", 56 | }, 57 | { 58 | Name: "format number like '234abc-123' to 'abc-123'", 59 | Rule: regexp.MustCompile("^\\d{3,5}([a-zA-Z]+[-|_]\\d+)([-|_].*)?$"), 60 | Rewrite: "$1$2", 61 | }, 62 | { 63 | Name: "rewrite 1pon or carib", 64 | Rule: regexp.MustCompile("(?i)(1pondo|1pon|carib)[-|_]?(.*)"), 65 | Rewrite: "$2", 66 | }, 67 | } 68 | function: | 69 | func(ctx context.Context, number string) (string, error) { 70 | for _, item := range sts { 71 | newNumber := item.Rule.ReplaceAllString(number, item.Rewrite) 72 | number = newNumber 73 | } 74 | return number, nil 75 | } 76 | import: 77 | - strings 78 | - regexp 79 | ` 80 | ) 81 | 82 | func TestLiveRewriterRule(t *testing.T) { 83 | rewriter, err := NewNumberRewriter(defaultLiveRewriterRule) 84 | if err != nil { 85 | t.Fatal(err) 86 | } 87 | 88 | m := map[string]string{ 89 | "fc2ppv12345-CD1": "FC2-PPV-12345-CD1", 90 | "123ABC-456-CD1": "ABC-456-CD1", 91 | "fc2ppv_1234": "FC2-PPV-1234", 92 | "fc2_ppv_1234": "FC2-PPV-1234", 93 | "fc2ppv-123": "FC2-PPV-123", 94 | "fc2-123445-cd1": "FC2-PPV-123445-CD1", 95 | "fc2-12345": "FC2-PPV-12345", 96 | "aaa": "AAA", 97 | "fc2": "FC2", 98 | "fc2ppv-123-asdasqwe2": "FC2-PPV-123-ASDASQWE2", 99 | "fc2ppv-12345-C-CD1": "FC2-PPV-12345-C-CD1", 100 | "fc2ppv-12345-CD1": "FC2-PPV-12345-CD1", 101 | "123abc_123aaa": "123ABC_123AAA", 102 | "123abc_1234": "ABC_1234", 103 | "222aaa-22222_helloworld": "AAA-22222_HELLOWORLD", 104 | "aaa-1234-CD1": "AAA-1234-CD1", 105 | "carib-1234-222": "1234-222", 106 | "1pon-2344-222": "2344-222", 107 | "1pondo-1234-222": "1234-222", 108 | } 109 | for k, v := range m { 110 | res, err := rewriter.Rewrite(context.Background(), k) 111 | if err != nil { 112 | t.Fatal(err) 113 | } 114 | if res != v { 115 | t.Fatalf("expected %s for %s, got %s", v, k, res) 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /dynscript/number_uncensor_checker.go: -------------------------------------------------------------------------------- 1 | package dynscript 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/xxxsen/common/logutil" 7 | "github.com/xxxsen/picker" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | type NumberUncensorCheckFunc = func(ctx context.Context, number string) (bool, error) 12 | 13 | type INumberUncensorChecker interface { 14 | IsMatch(ctx context.Context, number string) (bool, error) 15 | } 16 | 17 | type uncensorCheckerImpl struct { 18 | pk picker.IPicker[NumberUncensorCheckFunc] 19 | } 20 | 21 | func (u *uncensorCheckerImpl) IsMatch(ctx context.Context, number string) (bool, error) { 22 | for _, name := range u.pk.List() { 23 | f, err := u.pk.Get(name) 24 | if err != nil { 25 | logutil.GetLogger(ctx).Error("get uncensor check plugin failed", zap.String("name", name), zap.Error(err)) 26 | continue 27 | } 28 | matched, err := f(ctx, number) 29 | if err != nil { 30 | logutil.GetLogger(ctx).Error("call uncensor check plugin failed", zap.String("name", name), zap.Error(err)) 31 | continue 32 | } 33 | if matched { 34 | return true, nil 35 | } 36 | } 37 | return false, nil 38 | } 39 | 40 | func NewNumberUncensorChecker(rule string) (INumberUncensorChecker, error) { 41 | rule = rewriteTabToSpace(rule) 42 | pk, err := picker.ParseData[NumberUncensorCheckFunc]([]byte(rule), picker.YamlDecoder, picker.WithSafeFuncWrap(true)) 43 | if err != nil { 44 | return nil, err 45 | } 46 | return &uncensorCheckerImpl{ 47 | pk: pk, 48 | }, nil 49 | } 50 | -------------------------------------------------------------------------------- /enum/movie_meta.go: -------------------------------------------------------------------------------- 1 | package enum 2 | 3 | const ( 4 | MetaLangZH = "zh-cn" //简体 5 | MetaLangJa = "ja" //日语 6 | MetaLangZHTW = "zh-tw" //繁体 7 | MetaLangEn = "en" //英语 8 | ) 9 | -------------------------------------------------------------------------------- /face/constant.go: -------------------------------------------------------------------------------- 1 | package face 2 | 3 | const ( 4 | NameGoFace = "goface" 5 | NamePigo = "pigo" 6 | ) 7 | -------------------------------------------------------------------------------- /face/face_rec.go: -------------------------------------------------------------------------------- 1 | package face 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "image" 7 | ) 8 | 9 | var defaultInst IFaceRec 10 | 11 | func SetFaceRec(impl IFaceRec) { 12 | defaultInst = impl 13 | } 14 | 15 | type IFaceRec interface { 16 | Name() string 17 | SearchFaces(ctx context.Context, data []byte) ([]image.Rectangle, error) 18 | } 19 | 20 | func SearchFaces(ctx context.Context, data []byte) ([]image.Rectangle, error) { 21 | if defaultInst == nil { 22 | return nil, fmt.Errorf("not impl") 23 | } 24 | return defaultInst.SearchFaces(ctx, data) 25 | } 26 | 27 | func FindMaxFace(fs []image.Rectangle) image.Rectangle { 28 | var maxArea int 29 | var m image.Rectangle 30 | for _, f := range fs { 31 | p := f.Size() 32 | if area := p.X * p.Y; area > maxArea { 33 | m = f 34 | maxArea = area 35 | } 36 | } 37 | return m 38 | } 39 | 40 | func IsFaceRecognizeEnabled() bool { 41 | return defaultInst != nil 42 | } 43 | -------------------------------------------------------------------------------- /face/goface/go_face_linux.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package goface 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "image" 10 | "yamdc/face" 11 | 12 | goface "github.com/Kagami/go-face" 13 | "golang.org/x/sys/cpu" 14 | ) 15 | 16 | type goFace struct { 17 | recInst *goface.Recognizer 18 | } 19 | 20 | func (f *goFace) SearchFaces(ctx context.Context, data []byte) ([]image.Rectangle, error) { 21 | inst := f.recInst 22 | fces, err := inst.RecognizeCNN(data) 23 | if err != nil { 24 | return nil, err 25 | } 26 | if len(fces) == 0 { 27 | fces, err = inst.Recognize(data) 28 | } 29 | if err != nil { 30 | return nil, err 31 | } 32 | rs := make([]image.Rectangle, 0, len(fces)) 33 | for _, fce := range fces { 34 | rs = append(rs, fce.Rectangle) 35 | } 36 | return rs, nil 37 | } 38 | 39 | func (f *goFace) Name() string { 40 | return face.NameGoFace 41 | } 42 | 43 | func NewGoFace(modelDir string) (face.IFaceRec, error) { 44 | if !cpu.X86.HasAVX { 45 | return nil, fmt.Errorf("no AVX support, skip init goface recognizer") 46 | } 47 | 48 | inst, err := goface.NewRecognizer(modelDir) 49 | if err != nil { 50 | return nil, err 51 | } 52 | return &goFace{recInst: inst}, nil 53 | } 54 | -------------------------------------------------------------------------------- /face/goface/go_face_other.go: -------------------------------------------------------------------------------- 1 | //go:build !linux 2 | // +build !linux 3 | 4 | package goface 5 | 6 | import ( 7 | "errors" 8 | "yamdc/face" 9 | ) 10 | 11 | var errFeatureNotSupport = errors.New("feature not support") 12 | 13 | func NewGoFace(modelDir string) (face.IFaceRec, error) { 14 | return nil, errFeatureNotSupport 15 | } 16 | -------------------------------------------------------------------------------- /face/group.go: -------------------------------------------------------------------------------- 1 | package face 2 | 3 | import ( 4 | "context" 5 | "image" 6 | 7 | "github.com/xxxsen/common/logutil" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | type group struct { 12 | impls []IFaceRec 13 | } 14 | 15 | func NewGroup(impls []IFaceRec) IFaceRec { 16 | return &group{impls: impls} 17 | } 18 | 19 | func (g *group) Name() string { 20 | return "group" 21 | } 22 | 23 | func (g *group) SearchFaces(ctx context.Context, data []byte) ([]image.Rectangle, error) { 24 | var retErr error 25 | for _, impl := range g.impls { 26 | recs, err := impl.SearchFaces(ctx, data) 27 | if err == nil { 28 | logutil.GetLogger(ctx).Debug("search face succ", zap.String("face_rec_impl", impl.Name())) 29 | return recs, nil 30 | } 31 | retErr = err 32 | } 33 | return nil, retErr 34 | } 35 | -------------------------------------------------------------------------------- /face/pigo/pigo.go: -------------------------------------------------------------------------------- 1 | package pigo 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "image" 7 | "os" 8 | "path/filepath" 9 | "yamdc/face" 10 | 11 | pigo "github.com/esimov/pigo/core" 12 | ) 13 | 14 | const ( 15 | defaultFaceFinderCascade = "facefinder" 16 | ) 17 | 18 | type pigoWrap struct { 19 | inst *pigo.Pigo 20 | } 21 | 22 | func NewPigo(models string) (face.IFaceRec, error) { 23 | csFileData, err := os.ReadFile(filepath.Join(models, defaultFaceFinderCascade)) 24 | if err != nil { 25 | return nil, err 26 | } 27 | pigo := pigo.NewPigo() 28 | classifier, err := pigo.Unpack(csFileData) 29 | if err != nil { 30 | return nil, err 31 | } 32 | return &pigoWrap{inst: classifier}, nil 33 | } 34 | 35 | func (w *pigoWrap) Name() string { 36 | return face.NamePigo 37 | } 38 | 39 | func (w *pigoWrap) SearchFaces(ctx context.Context, data []byte) ([]image.Rectangle, error) { 40 | img, err := pigo.DecodeImage(bytes.NewReader(data)) 41 | if err != nil { 42 | return nil, err 43 | } 44 | pixels := pigo.RgbToGrayscale(img) 45 | cols, rows := img.Bounds().Max.X, img.Bounds().Max.Y 46 | cParams := pigo.CascadeParams{ 47 | MinSize: 20, 48 | MaxSize: 1000, 49 | ShiftFactor: 0.1, 50 | ScaleFactor: 1.1, 51 | 52 | ImageParams: pigo.ImageParams{ 53 | Pixels: pixels, 54 | Rows: rows, 55 | Cols: cols, 56 | Dim: cols, 57 | }, 58 | } 59 | angle := 0.0 60 | dets := w.inst.RunCascade(cParams, angle) 61 | dets = w.inst.ClusterDetections(dets, 0.2) 62 | rs := make([]image.Rectangle, 0, len(dets)) 63 | for _, det := range dets { 64 | if det.Q < 0.5 { 65 | continue 66 | } 67 | x1 := det.Col - det.Scale/2 68 | y1 := det.Row - det.Scale/2 69 | x2 := det.Col + det.Scale/2 70 | y2 := det.Row + det.Scale/2 71 | rs = append(rs, image.Rect(x1, y1, x2, y2)) 72 | } 73 | return rs, nil 74 | } 75 | -------------------------------------------------------------------------------- /ffmpeg/ffmpeg.go: -------------------------------------------------------------------------------- 1 | package ffmpeg 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | 11 | "github.com/google/uuid" 12 | ) 13 | 14 | var defaultFFMpeg *FFMpeg 15 | 16 | func init() { 17 | inst, err := NewFFMpeg() 18 | if err != nil { 19 | return 20 | } 21 | defaultFFMpeg = inst 22 | } 23 | 24 | func IsFFMpegEnabled() bool { 25 | return defaultFFMpeg != nil 26 | } 27 | 28 | type FFMpeg struct { 29 | cmd string 30 | } 31 | 32 | func NewFFMpeg() (*FFMpeg, error) { 33 | location, err := exec.LookPath("ffmpeg") 34 | if err != nil { 35 | return nil, err 36 | } 37 | return &FFMpeg{cmd: location}, nil 38 | } 39 | 40 | func (p *FFMpeg) ConvertToYuv420pJpegFromBytes(ctx context.Context, data []byte) ([]byte, error) { 41 | dstFile := filepath.Join(os.TempDir(), "image-conv-dst-"+uuid.New().String()) 42 | defer func() { 43 | _ = os.Remove(dstFile) 44 | }() 45 | cmd := exec.Command(p.cmd, "-i", "pipe:0", "-vf", "format=yuv420p", "-f", "image2", dstFile) 46 | cmd.Stdin = bytes.NewReader(data) 47 | err := cmd.Run() 48 | if err != nil { 49 | return nil, fmt.Errorf("call ffmpeg to conv failed, err:%w", err) 50 | } 51 | data, err = os.ReadFile(dstFile) 52 | if err != nil { 53 | return nil, fmt.Errorf("unable to read converted data, err:%w", err) 54 | } 55 | return data, nil 56 | } 57 | 58 | func ConvertToYuv420pJpegFromBytes(ctx context.Context, data []byte) ([]byte, error) { 59 | return defaultFFMpeg.ConvertToYuv420pJpegFromBytes(ctx, data) 60 | } 61 | -------------------------------------------------------------------------------- /ffmpeg/ffprobe.go: -------------------------------------------------------------------------------- 1 | package ffmpeg 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os/exec" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | var defaultFFProbe *FFProbe 12 | 13 | func init() { 14 | inst, err := NewFFProbe() 15 | if err != nil { 16 | return 17 | } 18 | defaultFFProbe = inst 19 | } 20 | 21 | func IsFFProbeEnabled() bool { 22 | return defaultFFProbe != nil 23 | } 24 | 25 | type FFProbe struct { 26 | cmd string 27 | } 28 | 29 | func NewFFProbe() (*FFProbe, error) { 30 | location, err := exec.LookPath("ffprobe") 31 | if err != nil { 32 | return nil, fmt.Errorf("search ffprobe command failed, err:%w", err) 33 | } 34 | return &FFProbe{cmd: location}, nil 35 | } 36 | 37 | func (p *FFProbe) ReadDuration(ctx context.Context, file string) (float64, error) { 38 | cmd := exec.CommandContext(ctx, p.cmd, []string{"-i", file, "-show_entries", "format=duration", "-v", "quiet", "-of", "csv=p=0"}...) 39 | output, err := cmd.Output() 40 | if err != nil { 41 | return 0, fmt.Errorf("call ffprobe to detect video duration failed, err:%w", err) 42 | } 43 | durationStr := strings.TrimSpace(string(output)) 44 | duration, err := strconv.ParseFloat(durationStr, 64) 45 | if err != nil { 46 | return 0, fmt.Errorf("parse video duration failed, duration:%s, err:%w", durationStr, err) 47 | } 48 | return duration, nil 49 | } 50 | 51 | func ReadDuration(ctx context.Context, file string) (float64, error) { 52 | return defaultFFProbe.ReadDuration(ctx, file) 53 | } 54 | -------------------------------------------------------------------------------- /flarerr/client_test.go: -------------------------------------------------------------------------------- 1 | package flarerr 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | "time" 7 | "yamdc/client" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestByPas(t *testing.T) { 13 | c, err := New(&http.Client{}, "http://127.0.0.1:8191") 14 | assert.NoError(t, err) 15 | MustAddToSolverList(c, "www.javlibrary.com") 16 | req, err := http.NewRequest(http.MethodGet, "https://www.javlibrary.com/cn/vl_searchbyid.php?keyword=ZMAR-134", nil) 17 | assert.NoError(t, err) 18 | start := time.Now() 19 | rsp, err := c.Do(req) 20 | assert.NoError(t, err) 21 | raw, err := client.ReadHTTPData(rsp) 22 | assert.NoError(t, err) 23 | t.Logf("cost:%dms", time.Since(start).Milliseconds()) 24 | t.Logf("read data:%s", string(raw)) 25 | } 26 | -------------------------------------------------------------------------------- /flarerr/model.go: -------------------------------------------------------------------------------- 1 | package flarerr 2 | 3 | type flareRequest struct { 4 | Cmd string `json:"cmd"` 5 | Url string `json:"url"` 6 | MaxTimeout int `json:"maxTimeout"` 7 | } 8 | 9 | type flareResponse struct { 10 | Status string `json:"status"` 11 | Message string `json:"message"` 12 | Solution flareSolution `json:"solution"` 13 | StartTimestamp int64 `json:"startTimestamp"` 14 | EndTimestamp int64 `json:"endTimestamp"` 15 | Version string `json:"version"` 16 | } 17 | 18 | type flareCookie struct { 19 | Domain string `json:"domain"` 20 | Expiry int64 `json:"expiry"` 21 | HttpOnly bool `json:"httpOnly"` 22 | Name string `json:"name"` 23 | Path string `json:"path"` 24 | SameSite string `json:"sameSite"` 25 | Secure bool `json:"secure"` 26 | Value string `json:"value"` 27 | Size int `json:"size"` 28 | Session bool `json:"session"` 29 | Expires int64 `json:"expires"` 30 | } 31 | 32 | type flareSolution struct { 33 | Url string `json:"url"` 34 | Status int `json:"status"` 35 | Cookies []flareCookie `json:"cookies"` 36 | UserAgent string `json:"userAgent"` 37 | Response string `json:"response"` 38 | } 39 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module yamdc 2 | 3 | go 1.24.0 4 | 5 | require ( 6 | github.com/Conight/go-googletrans v0.2.4 7 | github.com/Kagami/go-face v0.0.0-20210630145111-0c14797b4d0e 8 | github.com/antchfx/htmlquery v1.3.1 9 | github.com/esimov/pigo v1.4.6 10 | github.com/glebarez/go-sqlite v1.22.0 11 | github.com/google/uuid v1.6.0 12 | github.com/imroc/req/v3 v3.48.0 13 | github.com/stretchr/testify v1.10.0 14 | github.com/tailscale/hujson v0.0.0-20241010212012-29efb4a0184b 15 | github.com/xxxsen/common v0.1.22 16 | github.com/xxxsen/picker v0.0.3-beta.2 17 | go.uber.org/zap v1.23.0 18 | golang.org/x/image v0.18.0 19 | golang.org/x/net v0.38.0 20 | golang.org/x/text v0.23.0 21 | ) 22 | 23 | require ( 24 | github.com/thoas/go-funk v0.9.3 // indirect 25 | github.com/traefik/yaegi v0.16.1 // indirect 26 | ) 27 | 28 | require ( 29 | github.com/andybalholm/brotli v1.1.0 // indirect 30 | github.com/antchfx/xpath v1.3.0 // indirect 31 | github.com/cloudflare/circl v1.5.0 // indirect 32 | github.com/davecgh/go-spew v1.1.1 // indirect 33 | github.com/dustin/go-humanize v1.0.1 // indirect 34 | github.com/go-task/slim-sprig/v3 v3.0.0 // indirect 35 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 36 | github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134 // indirect 37 | github.com/hashicorp/errwrap v1.1.0 // indirect 38 | github.com/hashicorp/go-multierror v1.1.1 // indirect 39 | github.com/klauspost/compress v1.18.0 40 | github.com/mattn/go-isatty v0.0.20 // indirect 41 | github.com/mitchellh/mapstructure v1.5.0 // indirect 42 | github.com/onsi/ginkgo/v2 v2.20.2 // indirect 43 | github.com/pmezard/go-difflib v1.0.0 // indirect 44 | github.com/quic-go/qpack v0.5.1 // indirect 45 | github.com/quic-go/quic-go v0.48.2 // indirect 46 | github.com/refraction-networking/utls v1.7.0 // indirect 47 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect 48 | github.com/samber/lo v1.50.0 49 | go.uber.org/atomic v1.7.0 // indirect 50 | go.uber.org/mock v0.4.0 // indirect 51 | go.uber.org/multierr v1.6.0 // indirect 52 | golang.org/x/crypto v0.36.0 // indirect 53 | golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect 54 | golang.org/x/mod v0.21.0 // indirect 55 | golang.org/x/sync v0.12.0 56 | golang.org/x/sys v0.31.0 57 | golang.org/x/tools v0.25.0 // indirect 58 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect 59 | gopkg.in/yaml.v3 v3.0.1 // indirect 60 | modernc.org/libc v1.37.6 // indirect 61 | modernc.org/mathutil v1.6.0 // indirect 62 | modernc.org/memory v1.7.2 // indirect 63 | modernc.org/sqlite v1.28.0 // indirect 64 | ) 65 | 66 | replace github.com/Kagami/go-face => github.com/xxxsen/go-face v0.0.1 67 | -------------------------------------------------------------------------------- /hasher/hasher.go: -------------------------------------------------------------------------------- 1 | package hasher 2 | 3 | import ( 4 | "crypto/md5" 5 | "crypto/sha1" 6 | "encoding/hex" 7 | ) 8 | 9 | func ToMD5(in string) string { 10 | return ToMD5Bytes([]byte(in)) 11 | } 12 | 13 | func ToMD5Bytes(in []byte) string { 14 | h := md5.New() 15 | _, _ = h.Write(in) 16 | return hex.EncodeToString(h.Sum(nil)) 17 | } 18 | 19 | func ToSha1(in string) string { 20 | return ToSha1Bytes([]byte(in)) 21 | } 22 | 23 | func ToSha1Bytes(in []byte) string { 24 | h := sha1.New() 25 | _, _ = h.Write(in) 26 | return hex.EncodeToString(h.Sum(nil)) 27 | } 28 | -------------------------------------------------------------------------------- /image/.gitignore: -------------------------------------------------------------------------------- 1 | /testdata -------------------------------------------------------------------------------- /image/image.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "image" 7 | "image/color" 8 | _ "image/gif" 9 | "image/jpeg" 10 | _ "image/png" 11 | "os" 12 | 13 | _ "golang.org/x/image/bmp" 14 | "golang.org/x/image/draw" 15 | _ "golang.org/x/image/webp" 16 | ) 17 | 18 | func TranscodeToJpeg(data []byte) ([]byte, error) { 19 | img, err := LoadImage(data) 20 | if err != nil { 21 | return nil, err 22 | } 23 | return toJpegData(img) 24 | } 25 | 26 | func LoadImage(data []byte) (image.Image, error) { 27 | img, _, err := image.Decode(bytes.NewReader(data)) 28 | if err != nil { 29 | return nil, err 30 | } 31 | return img, nil 32 | } 33 | 34 | func toJpegData(img image.Image) ([]byte, error) { 35 | buf := bytes.Buffer{} 36 | if err := jpeg.Encode(&buf, img, &jpeg.Options{ 37 | Quality: 100, 38 | }); err != nil { 39 | return nil, fmt.Errorf("unable to convert img to jpg, err:%w", err) 40 | } 41 | return buf.Bytes(), nil 42 | } 43 | 44 | func WriteImageToBytes(img image.Image) ([]byte, error) { 45 | return toJpegData(img) 46 | } 47 | 48 | func fillImage(img *image.RGBA, c color.RGBA) { 49 | bounds := img.Bounds() 50 | for y := bounds.Min.Y; y < bounds.Max.Y; y++ { 51 | for x := bounds.Min.X; x < bounds.Max.X; x++ { 52 | img.Set(x, y, c) 53 | } 54 | } 55 | } 56 | 57 | func MakeColorImage(rect image.Rectangle, rgb color.RGBA) image.Image { 58 | img := image.NewRGBA(rect) 59 | fillImage(img, rgb) 60 | return img 61 | } 62 | 63 | func MakeColorImageData(rect image.Rectangle, rgb color.RGBA) ([]byte, error) { 64 | img := MakeColorImage(rect, rgb) 65 | buf := bytes.Buffer{} 66 | err := jpeg.Encode(&buf, img, nil) 67 | if err != nil { 68 | return nil, err 69 | } 70 | return buf.Bytes(), nil 71 | } 72 | 73 | func Scale(src image.Image, frame image.Rectangle) image.Image { 74 | dst := image.NewRGBA(frame) 75 | draw.NearestNeighbor.Scale(dst, dst.Bounds(), src, src.Bounds(), draw.Over, nil) 76 | return dst 77 | } 78 | 79 | func WriteImageToFile(dst string, img image.Image) error { 80 | raw, err := WriteImageToBytes(img) 81 | if err != nil { 82 | return err 83 | } 84 | return os.WriteFile(dst, raw, 0644) 85 | } 86 | -------------------------------------------------------------------------------- /image/image_cutter_test.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "context" 5 | "image" 6 | "io/fs" 7 | "os" 8 | "path/filepath" 9 | "sync" 10 | "testing" 11 | "yamdc/face" 12 | "yamdc/face/goface" 13 | "yamdc/face/pigo" 14 | 15 | "github.com/stretchr/testify/assert" 16 | ) 17 | 18 | type testPair struct { 19 | //input 20 | dx, dy int 21 | dxCenter, dyCenter int 22 | //output 23 | rect image.Rectangle 24 | gotErr bool 25 | } 26 | 27 | func TestCutFrame(t *testing.T) { 28 | tests := []testPair{ 29 | //宽高比>70%的场景, 此时会使用高度反推宽度 30 | {dx: 71, dy: 100, dxCenter: 71, dyCenter: 0, rect: image.Rectangle{Min: image.Point{1, 0}, Max: image.Point{71, 100}}, gotErr: false}, 31 | {dx: 100, dy: 100, dxCenter: 100, dyCenter: 0, rect: image.Rectangle{Min: image.Point{30, 0}, Max: image.Point{100, 100}}, gotErr: false}, 32 | {dx: 100, dy: 100, dxCenter: 50, dyCenter: 0, rect: image.Rectangle{Min: image.Point{15, 0}, Max: image.Point{85, 100}}, gotErr: false}, 33 | {dx: 100, dy: 100, dxCenter: 0, dyCenter: 0, rect: image.Rectangle{Min: image.Point{0, 0}, Max: image.Point{70, 100}}, gotErr: false}, 34 | {dx: 1000, dy: 100, dxCenter: 1000, dyCenter: 0, rect: image.Rectangle{Min: image.Point{930, 0}, Max: image.Point{1000, 100}}, gotErr: false}, 35 | //宽高比小于70%的场景, 则是使用宽度计算高度 36 | {dx: 70, dy: 120, dxCenter: 70, dyCenter: 0, rect: image.Rectangle{Min: image.Point{0, 0}, Max: image.Point{70, 100}}, gotErr: false}, 37 | {dx: 70, dy: 1000, dxCenter: 0, dyCenter: 0, rect: image.Rectangle{Min: image.Point{0, 0}, Max: image.Point{70, 100}}, gotErr: false}, 38 | {dx: 70, dy: 1000, dxCenter: 0, dyCenter: 100, rect: image.Rectangle{Min: image.Point{0, 50}, Max: image.Point{70, 150}}, gotErr: false}, 39 | //出错场景 40 | {dx: 0, dy: 123, gotErr: true}, 41 | {dx: 123, dy: 0, gotErr: true}, 42 | } 43 | 44 | for _, tst := range tests { 45 | rect, err := DetermineCutFrame(tst.dx, tst.dy, tst.dxCenter, tst.dyCenter, defaultAspectRatio) 46 | gotErr := err != nil 47 | assert.Equal(t, tst.gotErr, gotErr) 48 | assert.Equal(t, tst.rect, rect) 49 | } 50 | } 51 | 52 | func TestPigoRec(t *testing.T) { 53 | os.RemoveAll("./testdata/output_pigo/") 54 | os.MkdirAll("./testdata/output_pigo/", 0755) 55 | pg, err := pigo.NewPigo("../.vscode/tests/models") 56 | assert.NoError(t, err) 57 | face.SetFaceRec(pg) 58 | total := 0 59 | count := 0 60 | filepath.Walk("./testdata/input", func(path string, info fs.FileInfo, err error) error { 61 | if info.IsDir() { 62 | return nil 63 | } 64 | if err != nil { 65 | return nil 66 | } 67 | raw, err := os.ReadFile(path) 68 | assert.NoError(t, err) 69 | total++ 70 | out, err := CutImageWithFaceRecFromBytes(context.Background(), raw) 71 | if err != nil { 72 | return nil 73 | } 74 | count++ 75 | assert.NoError(t, err) 76 | err = os.WriteFile("./testdata/output_pigo/"+filepath.Base(path), out, 0644) 77 | assert.NoError(t, err) 78 | return nil 79 | }) 80 | t.Logf("total:%d, rec:%d", total, count) 81 | //total:17, rec:15 82 | } 83 | 84 | func TestGoFaceRec(t *testing.T) { 85 | os.RemoveAll("./testdata/output_goface/") 86 | os.MkdirAll("./testdata/output_goface/", 0755) 87 | pg, err := goface.NewGoFace("../.vscode/tests/models") 88 | assert.NoError(t, err) 89 | face.SetFaceRec(pg) 90 | total := 0 91 | count := 0 92 | wg := sync.WaitGroup{} 93 | filepath.Walk("./testdata/input", func(path string, info fs.FileInfo, err error) error { 94 | if info.IsDir() { 95 | return nil 96 | } 97 | if err != nil { 98 | return nil 99 | } 100 | wg.Add(1) 101 | go func() { 102 | defer wg.Done() 103 | raw, err := os.ReadFile(path) 104 | assert.NoError(t, err) 105 | total++ 106 | out, err := CutImageWithFaceRecFromBytes(context.Background(), raw) 107 | if err != nil { 108 | return 109 | } 110 | count++ 111 | assert.NoError(t, err) 112 | err = os.WriteFile("./testdata/output_goface/"+filepath.Base(path), out, 0644) 113 | assert.NoError(t, err) 114 | return 115 | }() 116 | return nil 117 | }) 118 | wg.Wait() 119 | t.Logf("total:%d, rec:%d", total, count) 120 | //total:17, rec:14 121 | } 122 | -------------------------------------------------------------------------------- /image/watermark.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "image" 7 | "image/draw" 8 | "yamdc/resource" 9 | ) 10 | 11 | type Watermark int 12 | 13 | const ( 14 | WMChineseSubtitle Watermark = 1 15 | WMUncensored Watermark = 2 16 | WM4K Watermark = 3 17 | WMLeak Watermark = 4 18 | WM8K Watermark = 5 19 | WMVR Watermark = 6 20 | WMHack Watermark = 7 21 | ) 22 | 23 | var resMap = make(map[Watermark][]byte) 24 | 25 | func registerResource() { 26 | resMap[WMChineseSubtitle] = resource.ResIMGSubtitle 27 | resMap[WM4K] = resource.ResIMG4K 28 | resMap[WMUncensored] = resource.ResIMGUncensored 29 | resMap[WMLeak] = resource.ResIMGLeak 30 | resMap[WM8K] = resource.ResIMG8K 31 | resMap[WMVR] = resource.ResIMGVR 32 | resMap[WMHack] = resource.ResIMGHack 33 | } 34 | 35 | func init() { 36 | registerResource() 37 | } 38 | 39 | const ( 40 | defaultMaxWaterMarkCount = 6 //最大的水印个数 41 | defaultWaterMarkWidthToImageWidthRatio = float64(31.58) / 100 //水印与整张图片的宽度比, W(watermark)/W(image) = 0.3158 42 | defaultWaterMarkWithToHeightRatio = 2 //水印本身的宽高比, W(watermark)/H(watermark) = 2 43 | defaultWatermarkGapSize = 10 //2个水印之间的间隔 44 | ) 45 | 46 | func addWatermarkToImage(img image.Image, wms []image.Image) (image.Image, error) { 47 | if len(wms) > defaultMaxWaterMarkCount { 48 | return nil, fmt.Errorf("water mark count out of limit, size:%d", len(wms)) 49 | } 50 | if len(wms) == 0 { 51 | return nil, fmt.Errorf("no watermark found") 52 | } 53 | mainBounds := img.Bounds() 54 | newImg := image.NewRGBA(mainBounds) 55 | draw.Draw(newImg, mainBounds, img, image.Point{0, 0}, draw.Src) 56 | watermarkWidth := int(float64(img.Bounds().Dx()) * defaultWaterMarkWidthToImageWidthRatio) 57 | watermarkHeight := watermarkWidth / 2 58 | for i := 0; i < len(wms); i++ { 59 | wm := Scale(wms[len(wms)-i-1], image.Rect(0, 0, watermarkWidth, watermarkHeight)) 60 | rect := image.Rectangle{ 61 | Min: image.Point{ 62 | X: img.Bounds().Dx() - watermarkWidth, 63 | Y: img.Bounds().Dy() - (i+1)*watermarkHeight - i*defaultWatermarkGapSize, 64 | }, 65 | Max: image.Point{ 66 | X: img.Bounds().Dx(), 67 | Y: img.Bounds().Dy() - i*watermarkHeight - i*defaultWatermarkGapSize, 68 | }, 69 | } 70 | if rect.Min.Y < 0 || rect.Max.Y < 0 { 71 | return nil, fmt.Errorf("image height too smart to contains all watermark") 72 | } 73 | draw.Draw(newImg, rect, wm, image.Point{0, 0}, draw.Over) 74 | } 75 | return newImg, nil 76 | } 77 | 78 | func selectWatermarkResource(w Watermark) ([]byte, bool) { 79 | out, ok := resMap[w] 80 | if !ok { 81 | return nil, false 82 | } 83 | rs := make([]byte, len(out)) 84 | copy(rs, out) 85 | return rs, true 86 | } 87 | 88 | func AddWatermark(img image.Image, wmTags []Watermark) (image.Image, error) { 89 | wms := make([]image.Image, 0, len(wmTags)) 90 | for _, tag := range wmTags { 91 | res, ok := selectWatermarkResource(tag) 92 | if !ok { 93 | return nil, fmt.Errorf("watermark:%d not found", tag) 94 | } 95 | wm, _, err := image.Decode(bytes.NewReader(res)) 96 | if err != nil { 97 | return nil, err 98 | } 99 | wms = append(wms, wm) 100 | } 101 | output, err := addWatermarkToImage(img, wms) 102 | if err != nil { 103 | return nil, fmt.Errorf("add water mark failed, err:%w", err) 104 | } 105 | return output, nil 106 | } 107 | 108 | func AddWatermarkFromBytes(data []byte, wmTags []Watermark) ([]byte, error) { 109 | img, err := LoadImage(data) 110 | if err != nil { 111 | return nil, err 112 | } 113 | newImg, err := AddWatermark(img, wmTags) 114 | if err != nil { 115 | return nil, err 116 | } 117 | return WriteImageToBytes(newImg) 118 | } 119 | -------------------------------------------------------------------------------- /image/watermark_test.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "image" 5 | "image/color" 6 | "image/jpeg" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestSmallWatermark(t *testing.T) { 15 | watermark := MakeColorImage(image.Rect(0, 0, 768, 374), color.RGBA{255, 0, 0, 0}) 16 | f, err := os.OpenFile(filepath.Join(os.TempDir(), "watermark.jpeg"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 17 | assert.NoError(t, err) 18 | defer f.Close() 19 | err = jpeg.Encode(f, watermark, nil) 20 | assert.NoError(t, err) 21 | } 22 | 23 | func TestWatermark(t *testing.T) { 24 | frame := MakeColorImage(image.Rect(0, 0, 380, 540), color.RGBA{0, 0, 0, 255}) 25 | wms := make([]image.Image, 0, 5) 26 | for i := 0; i < 4; i++ { 27 | watermark := MakeColorImage(image.Rect(0, 0, 768, 374), color.RGBA{255, 0, 0, 0}) 28 | wms = append(wms, watermark) 29 | } 30 | img, err := addWatermarkToImage(frame, wms) 31 | assert.NoError(t, err) 32 | f, err := os.OpenFile(filepath.Join(os.TempDir(), "fill_watermark.jpeg"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 33 | assert.NoError(t, err) 34 | defer f.Close() 35 | err = jpeg.Encode(f, img, nil) 36 | assert.NoError(t, err) 37 | } 38 | 39 | func TestWatermarkWithRes(t *testing.T) { 40 | data, err := MakeColorImageData(image.Rect(0, 0, 380, 540), color.RGBA{0, 0, 0, 255}) 41 | assert.NoError(t, err) 42 | raw, err := AddWatermarkFromBytes(data, []Watermark{ 43 | WMChineseSubtitle, 44 | WMUncensored, 45 | WM4K, 46 | }) 47 | assert.NoError(t, err) 48 | err = os.WriteFile(filepath.Join(os.TempDir(), "fill_watermark_with_res.jpeg"), raw, 0644) 49 | assert.NoError(t, err) 50 | } 51 | -------------------------------------------------------------------------------- /model/model.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "yamdc/number" 4 | 5 | type MovieMeta struct { 6 | Number string `json:"number"` //番号 7 | Title string `json:"title"` //标题 8 | TitleLang string `json:"title_lang"` //标题语言 9 | TitleTranslated string `json:"title_translated"` //翻译后的title 10 | Plot string `json:"plot"` //简介 11 | PlotLang string `json:"plot_lang"` //简介语言 12 | PlotTranslated string `json:"plot_translated"` //翻译后的plot 13 | Actors []string `json:"actors"` //演员, 如果产生翻译, 翻译结果会直接替换原始列表 14 | ActorsLang string `json:"actors_lang"` //演员语言 15 | ReleaseDate int64 `json:"release_date"` //发行时间, unix时间戳, 精确到秒 16 | Duration int64 `json:"duration"` //影片时长, 单位为秒 17 | Studio string `json:"studio"` //制作商 18 | Label string `json:"label"` //发行商 19 | Series string `json:"series"` //系列 20 | Genres []string `json:"genres"` //分类, tag 21 | GenresTranslated []string `json:"genres_translated"` //翻译后的genres 22 | GenresLang string `json:"genres_lang"` //tag语言 23 | Cover *File `json:"cover"` //封面 24 | Poster *File `json:"poster"` //海报 25 | SampleImages []*File `json:"sample_images"` //样品图 26 | Director string `json:"director"` //导演 27 | //非抓取的信息 28 | SwithConfig SwitchConfig `json:"switch_config"` //开关配置 29 | ExtInfo ExtInfo `json:"ext_info"` //扩展信息 30 | } 31 | 32 | type ScrapeInfo struct { 33 | Source string `json:"source"` 34 | DateTs int64 `json:"date_ts"` 35 | } 36 | 37 | type SwitchConfig struct { 38 | DisableNumberReplace bool `json:"disable_number_replace"` 39 | DisableReleaseDateCheck bool `json:"disable_release_date_check"` 40 | } 41 | 42 | type ExtInfo struct { 43 | ScrapeInfo ScrapeInfo `json:"scrape_info"` 44 | } 45 | 46 | type File struct { 47 | Name string `json:"name"` 48 | Key string `json:"key"` 49 | } 50 | 51 | type FileContext struct { 52 | FullFilePath string 53 | FileName string 54 | FileExt string 55 | SaveFileBase string 56 | SaveDir string 57 | Meta *MovieMeta 58 | Number *number.Number 59 | } 60 | -------------------------------------------------------------------------------- /nfo/model.go: -------------------------------------------------------------------------------- 1 | package nfo 2 | 3 | import "encoding/xml" 4 | 5 | type Movie struct { 6 | XMLName xml.Name `xml:"movie,omitempty"` 7 | Plot string `xml:"plot,omitempty"` //剧情简介? 8 | Dateadded string `xml:"dateadded,omitempty"` //example: 2022-08-18 06:01:03 9 | Title string `xml:"title,omitempty"` //标题 10 | OriginalTitle string `xml:"originaltitle,omitempty"` //原始标题, 与Title一致 11 | SortTitle string `xml:"sorttitle,omitempty"` //与Title一致即可 12 | Set string `xml:"set,omitempty"` //合集名? 13 | Rating float64 `xml:"rating,omitempty"` //评级, 貌似没用 14 | Release string `xml:"release,omitempty"` //与releaseDate一致即可 15 | ReleaseDate string `xml:"releasedate,omitempty"` //example: 2022-08-15 16 | Premiered string `xml:"premiered,omitempty"` //与ReleaseDate保持一致即可 17 | Runtime uint64 `xml:"runtime,omitempty"` //分钟数 18 | Year int `xml:"year,omitempty"` //发行年份, example: 2022 19 | Tags []string `xml:"tag,omitempty"` //标签信息 20 | Studio string `xml:"studio,omitempty"` //发行商 21 | Maker string `xml:"maker,omitempty"` //与Studio一致即可 22 | Genres []string `xml:"genre,omitempty"` //与标签保持一致即可 23 | Art Art `xml:"art,omitempty"` //图片列表 24 | Mpaa string `xml:"mpaa,omitempty"` //分级信息, 例如JP-18+ 25 | Director string `xml:"director,omitempty"` //导演 26 | Actors []Actor `xml:"actor,omitempty"` //演员 27 | Poster string `xml:"poster,omitempty"` //海报 28 | Thumb string `xml:"thumb,omitempty"` //缩略图 29 | Label string `xml:"label,omitempty"` //发行商 30 | ID string `xml:"id,omitempty"` //番号 31 | Cover string `xml:"cover,omitempty"` //封面 32 | Fanart string `xml:"fanart,omitempty"` //跟封面一致就好了 33 | ScrapeInfo ScrapeInfo `xml:"scrape_info"` //抓取信息 34 | } 35 | 36 | type ScrapeInfo struct { 37 | Source string `xml:"source"` 38 | Date string `xml:"date"` 39 | } 40 | 41 | type Actor struct { 42 | Name string `xml:"name,omitempty"` 43 | Role string `xml:"role,omitempty"` 44 | Thumb string `xml:"thumb,omitempty"` 45 | } 46 | 47 | type Art struct { 48 | Poster string `xml:"poster,omitempty"` 49 | Fanart []string `xml:"fanart,omitempty"` 50 | } 51 | -------------------------------------------------------------------------------- /nfo/nfo.go: -------------------------------------------------------------------------------- 1 | package nfo 2 | 3 | import ( 4 | "encoding/xml" 5 | "io" 6 | "os" 7 | ) 8 | 9 | func ParseMovie(f string) (*Movie, error) { 10 | raw, err := os.ReadFile(f) 11 | if err != nil { 12 | return nil, err 13 | } 14 | return ParseMovieWithData(raw) 15 | } 16 | 17 | func ParseMovieWithData(data []byte) (*Movie, error) { 18 | movie := &Movie{} 19 | if err := xml.Unmarshal(data, movie); err != nil { 20 | return nil, err 21 | } 22 | return movie, nil 23 | } 24 | 25 | func WriteMovie(w io.Writer, movie *Movie) error { 26 | xmlData, err := xml.MarshalIndent(movie, "", " ") 27 | if err != nil { 28 | return err 29 | } 30 | 31 | xmlWithHeader := []byte(xml.Header + string(xmlData)) 32 | if _, err := w.Write(xmlWithHeader); err != nil { 33 | return err 34 | } 35 | return nil 36 | } 37 | 38 | func WriteMovieToFile(f string, m *Movie) error { 39 | file, err := os.OpenFile(f, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 40 | if err != nil { 41 | return err 42 | } 43 | defer file.Close() 44 | return WriteMovie(file, m) 45 | } 46 | -------------------------------------------------------------------------------- /nfo/nfo_test.go: -------------------------------------------------------------------------------- 1 | package nfo 2 | 3 | import ( 4 | "bytes" 5 | "encoding/xml" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestReadWrite(t *testing.T) { 12 | m := &Movie{ 13 | XMLName: xml.Name{}, 14 | Plot: "hello world, this is a test", 15 | Dateadded: "2022-01-02", 16 | Title: "hello world", 17 | OriginalTitle: "hello world", 18 | SortTitle: "hello world", 19 | Set: "aaaa", 20 | Rating: 111, 21 | Release: "2021-01-05", 22 | ReleaseDate: "2021-01-05", 23 | Premiered: "2021-01-05", 24 | Runtime: 60, 25 | Year: 2021, 26 | Tags: []string{"t_a", "t_b", "t_c", "t_d"}, 27 | Studio: "hello_studio", 28 | Maker: "hello_maker", 29 | Genres: []string{"t_x", "t_y", "t_z"}, 30 | Art: Art{Poster: "art_poster.jpg", Fanart: []string{"art_fanart_1", "art_fanart_2", "art_fanart_3"}}, 31 | Mpaa: "JP-18+", 32 | Director: "hello_director", 33 | Actors: []Actor{{Name: "act_a", Role: "main", Thumb: "act_a.jpg"}}, 34 | Poster: "poster.jpg", 35 | Thumb: "thumb.jpg", 36 | Label: "hello_label", 37 | ID: "2022-01111", 38 | Cover: "cover.jpg", 39 | Fanart: "fanart.jpg", 40 | ScrapeInfo: ScrapeInfo{ 41 | Source: "abc", 42 | Date: "2021-03-05", 43 | }, 44 | } 45 | buf := bytes.NewBuffer(nil) 46 | err := WriteMovie(buf, m) 47 | assert.NoError(t, err) 48 | newM, err := ParseMovieWithData(buf.Bytes()) 49 | assert.NoError(t, err) 50 | newM.XMLName = m.XMLName 51 | assert.Equal(t, m, newM) 52 | } 53 | -------------------------------------------------------------------------------- /number/constant.go: -------------------------------------------------------------------------------- 1 | package number 2 | 3 | const ( 4 | defaultSuffixLeak = "LEAK" 5 | defaultSuffixChineseSubtitle = "C" 6 | defaultSuffix4K = "4K" 7 | defaultSuffix4KV2 = "2160P" 8 | defaultSuffix8K = "8K" 9 | defaultSuffixVR = "VR" 10 | defaultSuffixMultiCD = "CD" 11 | defaultSuffixHack1 = "U" 12 | defaultSuffixHack2 = "UC" 13 | ) 14 | 15 | const ( 16 | defaultTagUncensored = "无码" 17 | defaultTagChineseSubtitle = "中文字幕" 18 | defaultTag4K = "4K" 19 | defaultTagLeak = "无码流出" 20 | defaultTagHack = "破解" 21 | defaultTag8K = "8K" 22 | defaultTagVR = "VR" 23 | ) 24 | -------------------------------------------------------------------------------- /number/model.go: -------------------------------------------------------------------------------- 1 | package number 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | type externalField struct { 8 | isUncensor bool 9 | cat string 10 | } 11 | 12 | type Number struct { 13 | numberId string 14 | isChineseSubtitle bool 15 | isMultiCD bool 16 | multiCDIndex int 17 | is4k bool 18 | is8k bool 19 | isVr bool 20 | isLeak bool 21 | isHack bool 22 | extField externalField 23 | } 24 | 25 | func (n *Number) SetExternalFieldUncensor(v bool) { 26 | n.extField.isUncensor = v 27 | } 28 | 29 | func (n *Number) GetExternalFieldUncensor() bool { 30 | return n.extField.isUncensor 31 | } 32 | 33 | func (n *Number) SetExternalFieldCategory(cat string) { 34 | n.extField.cat = cat 35 | } 36 | 37 | func (n *Number) GetExternalFieldCategory() string { 38 | return n.extField.cat 39 | } 40 | 41 | func (n *Number) GetNumberID() string { 42 | return n.numberId 43 | } 44 | 45 | func (n *Number) GetIsChineseSubtitle() bool { 46 | return n.isChineseSubtitle 47 | } 48 | 49 | func (n *Number) GetIsMultiCD() bool { 50 | return n.isMultiCD 51 | } 52 | 53 | func (n *Number) GetMultiCDIndex() int { 54 | return n.multiCDIndex 55 | } 56 | 57 | func (n *Number) GetIs4K() bool { 58 | return n.is4k 59 | } 60 | 61 | func (n *Number) GetIs8K() bool { 62 | return n.is8k 63 | } 64 | 65 | func (n *Number) GetIsVR() bool { 66 | return n.isVr 67 | } 68 | 69 | func (n *Number) GetIsLeak() bool { 70 | return n.isLeak 71 | } 72 | 73 | func (n *Number) GetIsHack() bool { 74 | return n.isHack 75 | } 76 | 77 | func (n *Number) GenerateSuffix(base string) string { 78 | if n.GetIs4K() { 79 | base += "-" + defaultSuffix4K 80 | } 81 | if n.GetIs8K() { 82 | base += "-" + defaultSuffix8K 83 | } 84 | if n.GetIsVR() { 85 | base += "-" + defaultSuffixVR 86 | } 87 | if n.GetIsChineseSubtitle() { 88 | base += "-" + defaultSuffixChineseSubtitle 89 | } 90 | if n.GetIsLeak() { 91 | base += "-" + defaultSuffixLeak 92 | } 93 | if n.GetIsHack() { 94 | base += "-" + defaultSuffixHack2 95 | } 96 | if n.GetIsMultiCD() { 97 | base += "-" + defaultSuffixMultiCD + strconv.FormatInt(int64(n.GetMultiCDIndex()), 10) 98 | } 99 | return base 100 | } 101 | 102 | func (n *Number) GenerateTags() []string { 103 | rs := make([]string, 0, 5) 104 | if n.GetExternalFieldUncensor() { 105 | rs = append(rs, defaultTagUncensored) 106 | } 107 | if n.GetIsChineseSubtitle() { 108 | rs = append(rs, defaultTagChineseSubtitle) 109 | } 110 | if n.GetIs4K() { 111 | rs = append(rs, defaultTag4K) 112 | } 113 | if n.GetIs8K() { 114 | rs = append(rs, defaultTag8K) 115 | } 116 | if n.GetIsVR() { 117 | rs = append(rs, defaultTagVR) 118 | } 119 | if n.GetIsLeak() { 120 | rs = append(rs, defaultTagLeak) 121 | } 122 | if n.GetIsHack() { 123 | rs = append(rs, defaultTagHack) 124 | } 125 | return rs 126 | } 127 | 128 | func (n *Number) GenerateFileName() string { 129 | return n.GenerateSuffix(n.GetNumberID()) 130 | } 131 | -------------------------------------------------------------------------------- /number/number.go: -------------------------------------------------------------------------------- 1 | package number 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | type suffixInfoResolveFunc func(info *Number, normalizedSuffix string) bool 11 | 12 | var defaultSuffixResolverList = []suffixInfoResolveFunc{ 13 | resolveIsChineseSubTitle, 14 | resolveCDInfo, 15 | resolve4K, 16 | resolve8K, 17 | resolveVr, 18 | resolveLeak, 19 | resolveHack, 20 | } 21 | 22 | func extractSuffix(str string) (string, bool) { 23 | for i := len(str) - 1; i >= 0; i-- { 24 | if str[i] == '_' || str[i] == '-' { 25 | return str[i:], true 26 | } 27 | } 28 | return "", false 29 | } 30 | 31 | func tryResolveSuffix(info *Number, suffix string) bool { 32 | normalizedSuffix := strings.ToUpper(suffix[1:]) 33 | for _, resolver := range defaultSuffixResolverList { 34 | if resolver(info, normalizedSuffix) { 35 | return true 36 | } 37 | } 38 | return false 39 | } 40 | 41 | func resolveSuffixInfo(info *Number, str string) string { 42 | for { 43 | suffix, ok := extractSuffix(str) 44 | if !ok { 45 | return str 46 | } 47 | if !tryResolveSuffix(info, suffix) { 48 | return str 49 | } 50 | str = str[:len(str)-len(suffix)] 51 | } 52 | } 53 | 54 | func resolveCDInfo(info *Number, str string) bool { 55 | if !strings.HasPrefix(str, defaultSuffixMultiCD) { 56 | return false 57 | } 58 | strNum := str[2:] 59 | num, err := strconv.ParseInt(strNum, 10, 64) 60 | if err != nil { 61 | return false 62 | } 63 | info.isMultiCD = true 64 | info.multiCDIndex = int(num) 65 | return true 66 | } 67 | 68 | func resolveLeak(info *Number, str string) bool { 69 | if str != defaultSuffixLeak { 70 | return false 71 | } 72 | info.isLeak = true 73 | return true 74 | } 75 | 76 | func resolveHack(info *Number, str string) bool { 77 | if str != defaultSuffixHack1 && str != defaultSuffixHack2 { 78 | return false 79 | } 80 | info.isHack = true 81 | return true 82 | } 83 | 84 | func resolve4K(info *Number, str string) bool { 85 | if str != defaultSuffix4K && str != defaultSuffix4KV2 { 86 | return false 87 | } 88 | info.is4k = true 89 | return true 90 | } 91 | 92 | func resolve8K(info *Number, str string) bool { 93 | if str != defaultSuffix8K { 94 | return false 95 | } 96 | info.is8k = true 97 | return true 98 | } 99 | 100 | func resolveVr(info *Number, str string) bool { 101 | if str != defaultSuffixVR { 102 | return false 103 | } 104 | info.isVr = true 105 | return true 106 | } 107 | 108 | func resolveIsChineseSubTitle(info *Number, str string) bool { 109 | if str != defaultSuffixChineseSubtitle { 110 | return false 111 | } 112 | info.isChineseSubtitle = true 113 | return true 114 | } 115 | 116 | func ParseWithFileName(f string) (*Number, error) { 117 | filename := filepath.Base(f) 118 | fileext := filepath.Ext(f) 119 | filenoext := filename[:len(filename)-len(fileext)] 120 | return Parse(filenoext) 121 | } 122 | 123 | func Parse(str string) (*Number, error) { 124 | if len(str) == 0 { 125 | return nil, fmt.Errorf("empty number str") 126 | } 127 | if strings.Contains(str, ".") { 128 | return nil, fmt.Errorf("should not contain extname, str:%s", str) 129 | } 130 | number := strings.ToUpper(str) //默认所有的番号都是大写的 131 | rs := &Number{ 132 | numberId: "", 133 | isChineseSubtitle: false, 134 | isMultiCD: false, 135 | multiCDIndex: 0, 136 | } 137 | //部分番号需要进行改写, 改写逻辑提到外面去, number只做解析用 138 | 139 | //提取后缀信息并对番号进行裁剪 140 | number = resolveSuffixInfo(rs, number) 141 | rs.numberId = number 142 | return rs, nil 143 | } 144 | 145 | // GetCleanID 将番号中`-`, `_` 进行移除 146 | func GetCleanID(str string) string { 147 | sb := strings.Builder{} 148 | for _, c := range str { 149 | if c == '-' || c == '_' { 150 | continue 151 | } 152 | sb.WriteRune(c) 153 | } 154 | return sb.String() 155 | } 156 | -------------------------------------------------------------------------------- /number/number_test.go: -------------------------------------------------------------------------------- 1 | package number 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestNumber(t *testing.T) { 10 | checkList := map[string]*Number{ 11 | "HEYZO-3332.mp4": { 12 | numberId: "HEYZO-3332", 13 | }, 14 | "052624_01.mp4": { 15 | numberId: "052624_01", 16 | }, 17 | "052624_01-C.mp4": { 18 | numberId: "052624_01", 19 | isChineseSubtitle: true, 20 | }, 21 | "052624_01-CD2.mp4": { 22 | numberId: "052624_01", 23 | isMultiCD: true, 24 | multiCDIndex: 2, 25 | }, 26 | "052624_01-CD3-C.mp4": { 27 | numberId: "052624_01", 28 | isMultiCD: true, 29 | multiCDIndex: 3, 30 | isChineseSubtitle: true, 31 | }, 32 | "052624_01_cd3_c.mp4": { 33 | numberId: "052624_01", 34 | isMultiCD: true, 35 | multiCDIndex: 3, 36 | isChineseSubtitle: true, 37 | }, 38 | "k0009-c_cd1-4k.mp4": { 39 | numberId: "K0009", 40 | isMultiCD: true, 41 | multiCDIndex: 1, 42 | isChineseSubtitle: true, 43 | is4k: true, 44 | }, 45 | "n001-Cd1-4k.mp4": { 46 | numberId: "N001", 47 | isMultiCD: true, 48 | multiCDIndex: 1, 49 | is4k: true, 50 | }, 51 | "c-4k.mp4": { 52 | numberId: "C", 53 | isChineseSubtitle: false, 54 | is4k: true, 55 | }, 56 | "-c-4k.mp4": { 57 | numberId: "", 58 | isChineseSubtitle: true, 59 | is4k: true, 60 | }, 61 | "abc-leak-c.mp4": { 62 | numberId: "ABC", 63 | isLeak: true, 64 | isChineseSubtitle: true, 65 | }, 66 | } 67 | for file, info := range checkList { 68 | rs, err := ParseWithFileName(file) 69 | assert.NoError(t, err) 70 | assert.Equal(t, info.GetNumberID(), rs.GetNumberID()) 71 | assert.Equal(t, info.GetIsChineseSubtitle(), rs.GetIsChineseSubtitle()) 72 | assert.Equal(t, info.GetIsMultiCD(), rs.GetIsMultiCD()) 73 | assert.Equal(t, info.GetMultiCDIndex(), rs.GetMultiCDIndex()) 74 | //assert.Equal(t, info.GetExternalFieldUncensor(), rs.GetExternalFieldUncensor()) 75 | assert.Equal(t, info.GetIs4K(), rs.GetIs4K()) 76 | } 77 | } 78 | 79 | func TestAlnumber(t *testing.T) { 80 | assert.Equal(t, "fc2ppv12345", GetCleanID("fc2-ppv_12345")) 81 | } 82 | 83 | func TestSetFiledByExternal(t *testing.T) { 84 | n, err := Parse("abc-123") 85 | assert.NoError(t, err) 86 | n.SetExternalFieldUncensor(true) 87 | n.SetExternalFieldCategory("abc") 88 | assert.Equal(t, "abc", n.GetExternalFieldCategory()) 89 | assert.True(t, n.GetExternalFieldUncensor()) 90 | } 91 | -------------------------------------------------------------------------------- /numberkit/numberkit.go: -------------------------------------------------------------------------------- 1 | package numberkit 2 | 3 | import "strings" 4 | 5 | func IsFc2(number string) bool { 6 | number = strings.ToUpper(number) 7 | return strings.HasPrefix(number, "FC2") 8 | } 9 | 10 | func DecodeFc2ValID(n string) (string, bool) { 11 | if !IsFc2(n) { 12 | return "", false 13 | } 14 | idx := strings.LastIndex(n, "-") 15 | if idx < 0 { 16 | return "", false 17 | } 18 | return n[idx+1:], true 19 | } 20 | -------------------------------------------------------------------------------- /processor/default.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "context" 5 | "yamdc/model" 6 | "yamdc/processor/handler" 7 | ) 8 | 9 | var DefaultProcessor IProcessor = &defaultProcessor{} 10 | 11 | type defaultProcessor struct { 12 | } 13 | 14 | func (p *defaultProcessor) Name() string { 15 | return "default" 16 | } 17 | 18 | func (p *defaultProcessor) Process(ctx context.Context, fc *model.FileContext) error { 19 | return nil 20 | } 21 | 22 | type processorImpl struct { 23 | name string 24 | h handler.IHandler 25 | } 26 | 27 | func NewProcessor(name string, h handler.IHandler) IProcessor { 28 | return &processorImpl{name: name, h: h} 29 | } 30 | 31 | func (p *processorImpl) Name() string { 32 | return p.name 33 | } 34 | 35 | func (p *processorImpl) Process(ctx context.Context, meta *model.FileContext) error { 36 | return p.h.Handle(ctx, meta) 37 | } 38 | -------------------------------------------------------------------------------- /processor/group.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "context" 5 | "yamdc/model" 6 | 7 | "github.com/xxxsen/common/logutil" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | type group struct { 12 | ps []IProcessor 13 | } 14 | 15 | func NewGroup(ps []IProcessor) IProcessor { 16 | return &group{ps: ps} 17 | } 18 | 19 | func (g *group) Name() string { 20 | return "group" 21 | } 22 | 23 | func (g *group) Process(ctx context.Context, fc *model.FileContext) error { 24 | var lastErr error 25 | for _, p := range g.ps { 26 | err := p.Process(ctx, fc) 27 | if err == nil { 28 | continue 29 | } 30 | logutil.GetLogger(ctx).Error("process failed", zap.Error(err), zap.String("name", p.Name())) 31 | lastErr = err 32 | } 33 | return lastErr 34 | } 35 | -------------------------------------------------------------------------------- /processor/handler/actor_split_handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "regexp" 6 | "strings" 7 | "yamdc/model" 8 | ) 9 | 10 | var ( 11 | defaultExtractActorRegexp = regexp.MustCompile(`\s*(.+?)\s*\(\s*(.+?)\s*\)`) 12 | ) 13 | 14 | type actorSplitHandler struct { 15 | } 16 | 17 | func (h *actorSplitHandler) cleanActor(actor string) string { 18 | actor = strings.TrimSpace(actor) 19 | actor = strings.ReplaceAll(actor, "（", "(") 20 | actor = strings.ReplaceAll(actor, "）", ")") 21 | return actor 22 | } 23 | 24 | func (h *actorSplitHandler) tryExtractActor(actor string) ([]string, bool) { 25 | // 查找所有匹配的内容 26 | matches := defaultExtractActorRegexp.FindAllStringSubmatch(actor, -1) 27 | 28 | if len(matches) == 0 { 29 | return nil, false 30 | } 31 | rs := make([]string, 0, 2) 32 | for _, match := range matches { 33 | if len(match) == 3 { // match[0] 是整个匹配的字符串，match[1] 和 match[2] 是捕获组 34 | rs = append(rs, strings.TrimSpace(match[1]), strings.TrimSpace(match[2])) 35 | } 36 | } 37 | return rs, true 38 | } 39 | 40 | func (h *actorSplitHandler) Handle(ctx context.Context, fc *model.FileContext) error { 41 | //如果女优有括号, 尝试将其从括号中提取出来, example: 永野司 (永野つかさ) 42 | actorlist := make([]string, 0, len(fc.Meta.Actors)) 43 | for _, actor := range fc.Meta.Actors { 44 | actor = h.cleanActor(actor) 45 | splited, ok := h.tryExtractActor(actor) 46 | if !ok { 47 | actorlist = append(actorlist, actor) 48 | continue 49 | } 50 | actorlist = append(actorlist, splited...) 51 | } 52 | fc.Meta.Actors = actorlist 53 | return nil 54 | } 55 | 56 | func init() { 57 | Register(HActorSpliter, HandlerToCreator(&actorSplitHandler{})) 58 | } 59 | -------------------------------------------------------------------------------- /processor/handler/actor_split_handler_test.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "yamdc/model" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestSplitActor(t *testing.T) { 12 | tests := []string{ 13 | "永野司 (永野つかさ)", 14 | "萨达（AA萨达）", 15 | } 16 | h := &actorSplitHandler{} 17 | in := &model.FileContext{ 18 | Meta: &model.MovieMeta{ 19 | Actors: tests, 20 | }, 21 | } 22 | err := h.Handle(context.Background(), in) 23 | assert.NoError(t, err) 24 | t.Logf("read actor list:%+v", in.Meta.Actors) 25 | } 26 | -------------------------------------------------------------------------------- /processor/handler/ai_tagger_handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "unicode/utf8" 8 | "yamdc/aiengine" 9 | "yamdc/model" 10 | 11 | "github.com/xxxsen/common/logutil" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | const ( 16 | defaultAITaggerPrompt = ` 17 | You are an expert in tagging adult video content. The input is a title or description written in Chinese, Japanese, or English. Your task is to extract up to 5 keywords that are explicitly mentioned or directly implied by the text. Do not guess or invent. 18 | 19 | Each keyword must: 20 | - Be in Simplified Chinese 21 | - Be 2 to 3 Chinese characters long (no 4-character or longer phrases) 22 | - Be directly supported by the text 23 | 24 | Only output the keywords, separated by commas. No explanation, no extra text. 25 | 26 | Input: 27 | TITLE: "{TITLE}" 28 | PLOT: "{PLOT}" 29 | ` 30 | ) 31 | 32 | const ( 33 | defaultMaxAllowAITagLength = 4 34 | defaultMinTitleLengthForAITagging = 20 35 | defualtMinPlotLengthForAITagging = 60 36 | ) 37 | 38 | type aiTaggerHandler struct { 39 | } 40 | 41 | func (a *aiTaggerHandler) Handle(ctx context.Context, fc *model.FileContext) error { 42 | if !aiengine.IsAIEngineEnabled() { 43 | return nil 44 | } 45 | title := fc.Meta.Title 46 | if len(fc.Meta.TitleTranslated) > 0 { 47 | title = fc.Meta.TitleTranslated 48 | } 49 | plot := fc.Meta.Plot 50 | if len(fc.Meta.PlotTranslated) > 0 { 51 | plot = fc.Meta.PlotTranslated 52 | } 53 | if utf8.RuneCountInString(title) < defaultMinTitleLengthForAITagging && utf8.RuneCountInString(plot) < defualtMinPlotLengthForAITagging { 54 | return nil 55 | } 56 | res, err := aiengine.Complete(ctx, defaultAITaggerPrompt, map[string]interface{}{ 57 | "TITLE": title, 58 | "PLOT": plot, 59 | }) 60 | if err != nil { 61 | return fmt.Errorf("call ai engine for tagging failed, err:%w", err) 62 | } 63 | taglist := strings.Split(res, ",") 64 | for _, tag := range taglist { 65 | if utf8.RuneCountInString(tag) > defaultMaxAllowAITagLength { 66 | logutil.GetLogger(ctx).Warn("warning: tag is too long, may has err in ai engine, ignore it", zap.String("tag", tag)) 67 | continue 68 | } 69 | fc.Meta.Genres = append(fc.Meta.Genres, "AI-"+strings.TrimSpace(tag)) 70 | } 71 | return nil 72 | } 73 | 74 | func init() { 75 | Register(HAITagger, HandlerToCreator(&aiTaggerHandler{})) 76 | } 77 | -------------------------------------------------------------------------------- /processor/handler/ai_tagger_handler_test.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "os" 7 | "testing" 8 | "yamdc/aiengine" 9 | "yamdc/aiengine/gemini" 10 | "yamdc/model" 11 | 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func init() { 16 | raw, err := os.ReadFile("../../.vscode/keys.json") 17 | if err != nil { 18 | panic(err) 19 | } 20 | keys := make(map[string]string) 21 | if err := json.Unmarshal(raw, &keys); err != nil { 22 | panic(err) 23 | } 24 | for k, v := range keys { 25 | _ = os.Setenv(k, v) 26 | } 27 | } 28 | 29 | func TestAITagger(t *testing.T) { 30 | eng, err := gemini.New(gemini.WithKey(os.Getenv("GEMINI_KEY")), gemini.WithModel("gemini-2.0-flash")) 31 | assert.NoError(t, err) 32 | aiengine.SetAIEngine(eng) 33 | titles := []string{ 34 | "控制不住的戴绿帽冲动，得到了老公的认可！一个表情严肃，受虐狂的变态老婆，在镜头前暴露一切。她拥有美丽的乳房，乳头溢出胸罩，臀部丰满而美丽。她的阴部被她丈夫以外的人弄乱，使她潮吹。当她舔着沾满自己爱液的手指时，脸上的表情真是太色情了！这敏感到无比的小穴，只要被插入就能达到高潮吗？ ！随着另一个男人的阴茎一次又一次地出汗和射精！ [首拍] 线上应聘AV→AV体验拍摄2326", 35 | "MTALL-148 うるちゅるリップで締めつけるジュル音高めの極上スローフェラと淫語性交 雫月心桜", 36 | "MKMP-626 顔面レベル100 小那海あやの4コス＆4シチュで男性を最高の快感に導く ささやき淫語射精サポート", 37 | "JUR-036 超大型新人 専属第2章 中出し解禁！！ 夫と子作りSEXをした後はいつも義父に中出しされ続けています…。 新妻ゆうか", 38 | "上司の目を盗んで同僚とこっそり…社内不倫の背徳感に溺れて", 39 | "风骚女秘书深夜主动诱惑上司，在办公室掀起欲望风暴", 40 | } 41 | for idx, title := range titles { 42 | h := &aiTaggerHandler{} 43 | fctx := &model.FileContext{ 44 | Meta: &model.MovieMeta{ 45 | Title: title, 46 | Plot: "", 47 | }, 48 | } 49 | err = h.Handle(context.Background(), fctx) 50 | assert.NoError(t, err) 51 | t.Logf("index:%d, tags:%+v", idx, fctx.Meta.Genres) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /processor/handler/chinese_title_translate_optimizer_test.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "net/url" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestCnumber(t *testing.T) { 12 | h := &chineseTitleTranslateOptimizer{} 13 | ctx := context.Background() 14 | { 15 | title, ok, err := h.readTitleFromCNumber(ctx, "012413-001") 16 | assert.NoError(t, err) 17 | assert.True(t, ok) 18 | assert.Equal(t, "波多野结衣超豪华风俗服务", title) 19 | } 20 | { 21 | _, ok, err := h.readTitleFromCNumber(ctx, "111111") 22 | assert.NoError(t, err) 23 | assert.False(t, ok) 24 | } 25 | } 26 | 27 | func TestYesJav100(t *testing.T) { 28 | h := &chineseTitleTranslateOptimizer{} 29 | ctx := context.Background() 30 | { 31 | title, ok, err := h.readTitleFromYesJav(ctx, "jur-036") 32 | assert.NoError(t, err) 33 | assert.True(t, ok) 34 | assert.Equal(t, "JUR-036 和老公造人运动 但一直被公公内射 新妻优香", title) 35 | } 36 | { 37 | title, ok, err := h.readTitleFromYesJav(ctx, "DASS-541") 38 | assert.NoError(t, err) 39 | assert.True(t, ok) 40 | assert.Equal(t, "DASS-541 人妻外卖妹与中年男性外遇 橘玛丽", title) 41 | } 42 | { 43 | title, ok, err := h.readTitleFromYesJav(ctx, "111111") 44 | assert.NoError(t, err) 45 | assert.False(t, ok) 46 | assert.Equal(t, "", title) 47 | } 48 | } 49 | 50 | func TestEscape(t *testing.T) { 51 | origin := "jur-036" 52 | e1 := url.QueryEscape(origin) 53 | e2 := url.PathEscape(origin) 54 | t.Logf("e1:%s, e2:%s", e1, e2) 55 | } 56 | 57 | func TestTitleExtract(t *testing.T) { 58 | tests := []struct { 59 | input string 60 | expected string 61 | }{ 62 | {"[AI無碼] JUR-036 和老公造人运动 但一直被公公内射(中文字幕)", "JUR-036 和老公造人运动 但一直被公公内射"}, 63 | {"JUR-224 『從周三開始，和妻子做愛』的自豪朋友，讓我每週五，每次射3-4發，總共射18發中出的那個妻子。 市來真尋 (中文字幕)", "JUR-224 『從周三開始，和妻子做愛』的自豪朋友，讓我每週五，每次射3-4發，總共射18發中出的那個妻子。 市來真尋"}, 64 | {"SONE-669 鄰居団地妻在陽台晾襪的午後的訊息，是丈夫不在的信號 黑島玲衣 (中文字幕)", "SONE-669 鄰居団地妻在陽台晾襪的午後的訊息，是丈夫不在的信號 黑島玲衣"}, 65 | {"[AI無碼] DASS-541 人妻外卖妹与中年男性外遇 橘玛丽 (中文字幕)", "DASS-541 人妻外卖妹与中年男性外遇 橘玛丽"}, 66 | } 67 | for _, tst := range tests { 68 | t.Run(tst.input, func(t *testing.T) { 69 | h := chineseTitleTranslateOptimizer{} 70 | result := h.cleanSearchTitle(tst.input) 71 | assert.Equal(t, tst.expected, result) 72 | }) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /processor/handler/constant.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | const ( 4 | HPosterCropper = "poster_cropper" 5 | HDurationFixer = "duration_fixer" 6 | HImageTranscoder = "image_transcoder" 7 | HTranslater = "translater" 8 | HWatermakrMaker = "watermark_maker" 9 | HTagPadder = "tag_padder" 10 | HNumberTitle = "number_title" 11 | HActorSpliter = "actor_spliter" 12 | HAITagger = "ai_tagger" 13 | HHDCoverHandler = "hd_cover" 14 | HChineseTitleTranslateOptimizer = "chinese_title_translate_optimizer" 15 | ) 16 | -------------------------------------------------------------------------------- /processor/handler/duration_fixer_handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "yamdc/ffmpeg" 6 | "yamdc/model" 7 | 8 | "github.com/xxxsen/common/logutil" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | type durationFixerHandler struct { 13 | } 14 | 15 | func (h *durationFixerHandler) Handle(ctx context.Context, fc *model.FileContext) error { 16 | if fc.Meta.Duration > 0 { 17 | return nil 18 | } 19 | if !ffmpeg.IsFFProbeEnabled() { 20 | return nil 21 | } 22 | duration, err := ffmpeg.ReadDuration(ctx, fc.FullFilePath) 23 | if err != nil { 24 | return err 25 | } 26 | fc.Meta.Duration = int64(duration) 27 | logutil.GetLogger(ctx).Debug("rewrite video duration succ", zap.Float64("duration", duration)) 28 | return nil 29 | } 30 | 31 | func init() { 32 | Register(HDurationFixer, HandlerToCreator(&durationFixerHandler{})) 33 | } 34 | -------------------------------------------------------------------------------- /processor/handler/duration_fixer_handler_test.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/exec" 7 | "path/filepath" 8 | "testing" 9 | "yamdc/model" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestDurationFixer(t *testing.T) { 15 | tmpVideo := filepath.Join(os.TempDir(), "test_video.mp4") 16 | defer func() { 17 | _ = os.RemoveAll(tmpVideo) 18 | }() 19 | cmd := exec.Command("ffmpeg", []string{"-f", "lavfi", "-i", "color=c=black:s=320x240:d=10", "-an", "-vcodec", "libx264", tmpVideo, "-y"}...) 20 | err := cmd.Run() 21 | assert.NoError(t, err) 22 | defer filepath.Join(os.TempDir(), "temp_video.mp4") 23 | fc := &model.FileContext{ 24 | FullFilePath: tmpVideo, 25 | Meta: &model.MovieMeta{}, 26 | } 27 | h, err := CreateHandler(HDurationFixer, nil) 28 | assert.NoError(t, err) 29 | err = h.Handle(context.Background(), fc) 30 | assert.NoError(t, err) 31 | assert.Equal(t, int64(10), fc.Meta.Duration) 32 | } 33 | -------------------------------------------------------------------------------- /processor/handler/handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sort" 7 | "yamdc/model" 8 | ) 9 | 10 | type IHandler interface { 11 | Handle(ctx context.Context, fc *model.FileContext) error 12 | } 13 | 14 | var mp = make(map[string]CreatorFunc) 15 | 16 | type CreatorFunc func(args interface{}) (IHandler, error) 17 | 18 | func Register(name string, fn CreatorFunc) { 19 | mp[name] = fn 20 | } 21 | 22 | func CreateHandler(name string, args interface{}) (IHandler, error) { 23 | cr, ok := mp[name] 24 | if !ok { 25 | return nil, fmt.Errorf("handler:%s not found", name) 26 | } 27 | return cr(args) 28 | } 29 | 30 | func HandlerToCreator(h IHandler) CreatorFunc { 31 | return func(args interface{}) (IHandler, error) { 32 | return h, nil 33 | } 34 | } 35 | 36 | func Handlers() []string { 37 | rs := make([]string, 0, len(mp)) 38 | for k := range mp { 39 | rs = append(rs, k) 40 | } 41 | return sort.StringSlice(rs) 42 | } 43 | -------------------------------------------------------------------------------- /processor/handler/hd_cover_handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "strings" 9 | "yamdc/client" 10 | "yamdc/image" 11 | "yamdc/model" 12 | "yamdc/store" 13 | ) 14 | 15 | const ( 16 | defaultHDCoverLinkTemplate = "https://awsimgsrc.dmm.co.jp/pics_dig/digital/video/%s/%spl.jpg" 17 | defaultMinCoverSize = 20 * 1024 //20k 18 | ) 19 | 20 | type highQualityCoverHandler struct { 21 | } 22 | 23 | func (h *highQualityCoverHandler) Handle(ctx context.Context, fc *model.FileContext) error { 24 | res := strings.Split(fc.Number.GetNumberID(), "-") 25 | if len(res) != 2 { //仅存在2个part的情况下才需要处理, 否则直接跳过 26 | return nil 27 | } 28 | num := strings.ToLower(strings.ReplaceAll(fc.Number.GetNumberID(), "-", "00")) 29 | link := fmt.Sprintf(defaultHDCoverLinkTemplate, num, num) 30 | req, err := http.NewRequest(http.MethodGet, link, nil) 31 | if err != nil { 32 | return fmt.Errorf("build hd cover link failed, err:%w", err) 33 | } 34 | rsp, err := client.DefaultClient().Do(req) 35 | if err != nil { 36 | return fmt.Errorf("request hd cover failed, err:%w", err) 37 | } 38 | defer rsp.Body.Close() 39 | if rsp.StatusCode != http.StatusOK { 40 | return fmt.Errorf("hd cover response not ok, code:%d", rsp.StatusCode) 41 | } 42 | raw, err := io.ReadAll(rsp.Body) 43 | if err != nil { 44 | return fmt.Errorf("read hd cover data failed, err:%w", err) 45 | } 46 | if len(raw) < defaultMinCoverSize { 47 | return fmt.Errorf("skip hd cover, too small, size:%d", len(raw)) 48 | } 49 | if _, err := image.LoadImage(raw); err != nil { 50 | return fmt.Errorf("hd cover server return non-image data, err:%w", err) 51 | } 52 | key, err := store.AnonymousPutData(ctx, raw) 53 | if err != nil { 54 | return fmt.Errorf("write hd cover data failed, err:%w", err) 55 | } 56 | fc.Meta.Cover.Key = key 57 | return nil 58 | } 59 | 60 | func init() { 61 | Register(HHDCoverHandler, HandlerToCreator(&highQualityCoverHandler{})) 62 | } 63 | -------------------------------------------------------------------------------- /processor/handler/image_transcode_handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "yamdc/ffmpeg" 8 | "yamdc/image" 9 | "yamdc/model" 10 | "yamdc/store" 11 | 12 | "github.com/xxxsen/common/logutil" 13 | "go.uber.org/zap" 14 | ) 15 | 16 | type imageTranscodeHandler struct { 17 | } 18 | 19 | func (p *imageTranscodeHandler) Name() string { 20 | return HImageTranscoder 21 | } 22 | 23 | func (p *imageTranscodeHandler) Handle(ctx context.Context, meta *model.FileContext) error { 24 | meta.Meta.Cover = p.transcode(ctx, "cover", meta.Meta.Cover) 25 | meta.Meta.Poster = p.transcode(ctx, "poster", meta.Meta.Poster) 26 | rebuildSampleList := make([]*model.File, 0, len(meta.Meta.SampleImages)) 27 | for idx, item := range meta.Meta.SampleImages { 28 | if v := p.transcode(ctx, fmt.Sprintf("sample_%d", idx), item); v != nil { 29 | rebuildSampleList = append(rebuildSampleList, v) 30 | } 31 | } 32 | meta.Meta.SampleImages = rebuildSampleList 33 | return nil 34 | } 35 | 36 | func (p *imageTranscodeHandler) transcode(ctx context.Context, name string, f *model.File) *model.File { 37 | logger := logutil.GetLogger(ctx).With(zap.String("name", name)) 38 | if f == nil || len(f.Key) == 0 { 39 | logger.Debug("no image found, skip transcode to jpeg logic") 40 | return nil 41 | } 42 | logger = logger.With(zap.String("key", f.Key)) 43 | 44 | key, err := store.AnonymousDataRewrite(ctx, f.Key, func(ctx context.Context, data []byte) ([]byte, error) { 45 | raw, err := image.TranscodeToJpeg(data) 46 | if err != nil && strings.Contains(err.Error(), "luma/chroma subsampling ratio") && ffmpeg.IsFFMpegEnabled() { 47 | data, err = ffmpeg.ConvertToYuv420pJpegFromBytes(ctx, data) 48 | if err != nil { 49 | logger.Error("use ffmpeg to correct invalid image data failed", zap.Error(err)) 50 | return nil, err 51 | } 52 | raw, err = image.TranscodeToJpeg(data) 53 | } 54 | return raw, err 55 | }) 56 | if err != nil { 57 | logger.Error("transcoded image data failed", zap.Error(err)) 58 | return nil // 59 | } 60 | logger.Debug("transcode image succ", zap.String("new_key", key)) 61 | f.Key = key 62 | return f 63 | } 64 | 65 | func init() { 66 | Register(HImageTranscoder, HandlerToCreator(&imageTranscodeHandler{})) 67 | } 68 | -------------------------------------------------------------------------------- /processor/handler/number_title_handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "yamdc/model" 7 | "yamdc/number" 8 | ) 9 | 10 | type numberTitleHandler struct { 11 | } 12 | 13 | func (h *numberTitleHandler) Handle(ctx context.Context, fc *model.FileContext) error { 14 | title := number.GetCleanID(fc.Meta.Title) 15 | num := number.GetCleanID(fc.Number.GetNumberID()) 16 | if strings.Contains(title, num) { 17 | return nil 18 | } 19 | fc.Meta.Title = fc.Number.GetNumberID() + " " + fc.Meta.Title 20 | return nil 21 | } 22 | 23 | func init() { 24 | Register(HNumberTitle, HandlerToCreator(&numberTitleHandler{})) 25 | } 26 | -------------------------------------------------------------------------------- /processor/handler/poster_crop_handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "yamdc/face" 7 | "yamdc/image" 8 | "yamdc/model" 9 | "yamdc/store" 10 | 11 | "github.com/xxxsen/common/logutil" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | type imageCutter func(data []byte) ([]byte, error) 16 | 17 | type posterCropHandler struct { 18 | } 19 | 20 | func (c *posterCropHandler) Name() string { 21 | return HPosterCropper 22 | } 23 | 24 | func (c *posterCropHandler) censorCutter(ctx context.Context) imageCutter { 25 | //如果没有开启人脸识别, 那么直接使用原始的骑兵裁剪方案 26 | if !face.IsFaceRecognizeEnabled() { 27 | return image.CutCensoredImageFromBytes 28 | } 29 | return func(data []byte) ([]byte, error) { 30 | raw, err := image.CutCensoredImageFromBytes(data) 31 | if err != nil { 32 | return nil, err 33 | } 34 | //出错或者已经存在人脸, 直接返回 35 | rects, err := face.SearchFaces(ctx, raw) 36 | if err != nil || len(rects) > 0 { 37 | return raw, nil 38 | } 39 | //在原始图中, 存在人脸, 那么尝试从原始图中进行人脸识别并裁剪 40 | //主要优化 SIRO 之类的番号无法截取正常带人脸poster的问题 41 | rects, err = face.SearchFaces(ctx, data) 42 | if err != nil || len(rects) != 1 { //仅有一个人脸的场景下才执行人脸识别, 避免截到奇奇怪怪的地方 43 | return raw, nil 44 | } 45 | logutil.GetLogger(ctx).Info("enhance poster crop with face rec for censored number") 46 | rec, err := image.CutImageWithFaceRecFromBytes(ctx, data) 47 | if err != nil { 48 | return raw, nil 49 | } 50 | return rec, nil 51 | } 52 | } 53 | 54 | func (c *posterCropHandler) uncensorCutter(ctx context.Context) imageCutter { 55 | if !face.IsFaceRecognizeEnabled() { 56 | return image.CutCensoredImageFromBytes 57 | } 58 | return func(data []byte) ([]byte, error) { 59 | rec, err := image.CutImageWithFaceRecFromBytes(ctx, data) 60 | if err == nil { 61 | return rec, nil 62 | } 63 | logutil.GetLogger(ctx).Warn("cut image with face rec failed, try use other cut method", zap.Error(err)) 64 | return image.CutCensoredImageFromBytes(data) 65 | } 66 | } 67 | 68 | func (c *posterCropHandler) Handle(ctx context.Context, fc *model.FileContext) error { 69 | logger := logutil.GetLogger(ctx).With(zap.String("number", fc.Meta.Number)) 70 | if fc.Meta.Poster != nil { //仅处理没有海报的元数据 71 | logger.Debug("poster exist, skip generate") 72 | return nil 73 | } 74 | if fc.Meta.Cover == nil { //无封面, 处理无意义 75 | logger.Error("no cover found, skip process poster") 76 | return nil 77 | } 78 | var cutter imageCutter = c.censorCutter(ctx) //默认情况下, 都按骑兵进行封面处理 79 | if fc.Number.GetExternalFieldUncensor() { //如果为步兵, 则使用人脸识别(当然, 只有该特性能用的情况下才启用) 80 | cutter = c.uncensorCutter(ctx) 81 | } 82 | key, err := store.AnonymousDataRewrite(ctx, fc.Meta.Cover.Key, func(ctx context.Context, data []byte) ([]byte, error) { 83 | return cutter(data) 84 | }) 85 | if err != nil { 86 | return fmt.Errorf("save cutted poster data failed, err:%w", err) 87 | } 88 | fc.Meta.Poster = &model.File{ 89 | Name: "./poster.jpg", 90 | Key: key, 91 | } 92 | return nil 93 | } 94 | 95 | func init() { 96 | Register(HPosterCropper, HandlerToCreator(&posterCropHandler{})) 97 | } 98 | -------------------------------------------------------------------------------- /processor/handler/tag_padder_handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "unicode" 7 | "yamdc/model" 8 | "yamdc/utils" 9 | ) 10 | 11 | type tagPadderHandler struct{} 12 | 13 | func (h *tagPadderHandler) generateNumberPrefixTag(fc *model.FileContext) (string, bool) { 14 | //将番号的前版本部分提取, 作为分类的一部分, 方便从一个影片看到这个系列相关的全部影片 15 | sb := strings.Builder{} 16 | isPureNumber := true 17 | for _, c := range fc.Number.GetNumberID() { 18 | if c == '-' || c == '_' { 19 | break 20 | } 21 | if unicode.IsLetter(c) { 22 | isPureNumber = false 23 | } 24 | sb.WriteRune(c) 25 | } 26 | if isPureNumber { 27 | return "", false 28 | } 29 | return sb.String(), true 30 | } 31 | 32 | func (h *tagPadderHandler) rewriteOrAppendTag(fc *model.MovieMeta, tagname string) { 33 | isContained := false 34 | for idx, item := range fc.Genres { 35 | if strings.EqualFold(item, tagname) { 36 | fc.Genres[idx] = tagname 37 | isContained = true 38 | } 39 | } 40 | if isContained { 41 | return 42 | } 43 | fc.Genres = append(fc.Genres, tagname) 44 | } 45 | 46 | func (h *tagPadderHandler) Handle(ctx context.Context, fc *model.FileContext) error { 47 | //提取番号特有的tag 48 | fc.Meta.Genres = append(fc.Meta.Genres, fc.Number.GenerateTags()...) 49 | //提取番号前缀作为tag 50 | if tag, ok := h.generateNumberPrefixTag(fc); ok { 51 | h.rewriteOrAppendTag(fc.Meta, tag) 52 | } 53 | fc.Meta.Genres = utils.DedupStringList(fc.Meta.Genres) 54 | return nil 55 | } 56 | 57 | func init() { 58 | Register(HTagPadder, HandlerToCreator(&tagPadderHandler{})) 59 | } 60 | -------------------------------------------------------------------------------- /processor/handler/tag_padder_handler_test.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "yamdc/model" 7 | "yamdc/number" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | type testPair struct { 13 | in string 14 | tagCount int 15 | existTags []string 16 | } 17 | 18 | func TestTagPadde(t *testing.T) { 19 | tsts := []testPair{ 20 | { 21 | in: "fc2-1234-c-4k", 22 | tagCount: 4, 23 | existTags: []string{}, 24 | }, 25 | { 26 | in: "fc2-ppv-123", 27 | tagCount: 2, 28 | existTags: []string{"FC2"}, 29 | }, 30 | { 31 | in: "heyzo-123", 32 | tagCount: 2, 33 | existTags: []string{"HEYZO"}, 34 | }, 35 | { 36 | in: "111111-11", 37 | tagCount: 1, 38 | existTags: []string{}, 39 | }, 40 | } 41 | for _, item := range tsts { 42 | num, err := number.Parse(item.in) 43 | assert.NoError(t, err) 44 | padder := &tagPadderHandler{} 45 | fc := &model.FileContext{ 46 | Number: num, 47 | Meta: &model.MovieMeta{}, 48 | } 49 | padder.Handle(context.Background(), fc) 50 | assert.Equal(t, item.tagCount, len(fc.Meta.Genres)) 51 | for _, existTag := range item.existTags { 52 | assert.Contains(t, fc.Meta.Genres, existTag) 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /processor/handler/translate_handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | "yamdc/enum" 8 | "yamdc/hasher" 9 | "yamdc/model" 10 | "yamdc/store" 11 | "yamdc/translator" 12 | 13 | "github.com/xxxsen/common/logutil" 14 | "go.uber.org/zap" 15 | ) 16 | 17 | const ( 18 | defaultTranslateDataSaveTime = 30 * 24 * time.Hour 19 | ) 20 | 21 | type translaterHandler struct { 22 | } 23 | 24 | func (p *translaterHandler) Name() string { 25 | return HTranslater 26 | } 27 | 28 | func (p *translaterHandler) buildKey(data string) string { 29 | return fmt.Sprintf("yamdc:translate:%s", hasher.ToMD5(data)) 30 | } 31 | 32 | func (p *translaterHandler) translateSingle(ctx context.Context, name string, in string, lang string, out *string) error { 33 | if len(in) == 0 { 34 | return nil 35 | } 36 | if !p.isNeedTranslate(lang) { 37 | return nil 38 | } 39 | res, err := store.LoadData(ctx, p.buildKey(in), defaultTranslateDataSaveTime, func() ([]byte, error) { 40 | res, err := translator.Translate(ctx, in, "auto", "zh") 41 | if err != nil { 42 | return nil, err 43 | } 44 | return []byte(res), nil 45 | }) 46 | 47 | if err != nil { 48 | return fmt.Errorf("translate failed, name:%s, data:%s, err:%w", name, in, err) 49 | } 50 | *out = string(res) 51 | return nil 52 | } 53 | 54 | func (p *translaterHandler) isNeedTranslate(lang string) bool { 55 | if len(lang) == 0 || lang == enum.MetaLangZHTW || lang == enum.MetaLangZH { 56 | return false 57 | } 58 | return true 59 | } 60 | 61 | func (p *translaterHandler) translateArray(ctx context.Context, name string, in []string, lang string, out *[]string) error { 62 | if !p.isNeedTranslate(lang) { 63 | return nil 64 | } 65 | rs := make([]string, 0, len(in)*2) 66 | rs = append(rs, in...) 67 | for _, item := range in { 68 | var res string 69 | if err := p.translateSingle(ctx, "dispatch-"+name+"-translate", item, lang, &res); err != nil { 70 | logutil.GetLogger(ctx).Error("translate array failed", zap.Error(err), zap.String("name", name), zap.String("translate_item", item)) 71 | continue 72 | } 73 | rs = append(rs, res) 74 | } 75 | *out = rs 76 | return nil 77 | } 78 | 79 | func (p *translaterHandler) Handle(ctx context.Context, fc *model.FileContext) error { 80 | if !translator.IsTranslatorEnabled() { 81 | return nil 82 | } 83 | var errs []error 84 | errs = append(errs, p.translateSingle(ctx, "title", fc.Meta.Title, fc.Meta.TitleLang, &fc.Meta.TitleTranslated)) 85 | errs = append(errs, p.translateSingle(ctx, "plot", fc.Meta.Plot, fc.Meta.PlotLang, &fc.Meta.PlotTranslated)) 86 | errs = append(errs, p.translateArray(ctx, "genere", fc.Meta.Genres, fc.Meta.GenresLang, &fc.Meta.Genres)) 87 | errs = append(errs, p.translateArray(ctx, "actor", fc.Meta.Actors, fc.Meta.ActorsLang, &fc.Meta.Actors)) 88 | 89 | for _, err := range errs { 90 | if err != nil { 91 | return fmt.Errorf("translate part failed, err:%w", err) 92 | } 93 | } 94 | return nil 95 | } 96 | 97 | func init() { 98 | Register(HTranslater, HandlerToCreator(&translaterHandler{})) 99 | } 100 | -------------------------------------------------------------------------------- /processor/handler/watermark_handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "yamdc/image" 7 | "yamdc/model" 8 | "yamdc/store" 9 | 10 | "github.com/xxxsen/common/logutil" 11 | ) 12 | 13 | type watermark struct { 14 | } 15 | 16 | func (h *watermark) Handle(ctx context.Context, fc *model.FileContext) error { 17 | if fc.Meta.Poster == nil || len(fc.Meta.Poster.Key) == 0 { 18 | return nil 19 | } 20 | tags := make([]image.Watermark, 0, 5) 21 | if fc.Number.GetIs4K() { 22 | tags = append(tags, image.WM4K) 23 | } 24 | if fc.Number.GetIs8K() { 25 | tags = append(tags, image.WM8K) 26 | } 27 | if fc.Number.GetIsVR() { 28 | tags = append(tags, image.WMVR) 29 | } 30 | if fc.Number.GetExternalFieldUncensor() { 31 | tags = append(tags, image.WMUncensored) 32 | } 33 | if fc.Number.GetIsChineseSubtitle() { 34 | tags = append(tags, image.WMChineseSubtitle) 35 | } 36 | if fc.Number.GetIsLeak() { 37 | tags = append(tags, image.WMLeak) 38 | } 39 | if fc.Number.GetIsHack() { 40 | tags = append(tags, image.WMHack) 41 | } 42 | if len(tags) == 0 { 43 | logutil.GetLogger(ctx).Debug("no watermark tag found, skip watermark proc") 44 | return nil 45 | } 46 | key, err := store.AnonymousDataRewrite(ctx, fc.Meta.Poster.Key, func(ctx context.Context, data []byte) ([]byte, error) { 47 | return image.AddWatermarkFromBytes(data, tags) 48 | }) 49 | if err != nil { 50 | return fmt.Errorf("save watermarked image failed, err:%w", err) 51 | } 52 | fc.Meta.Poster.Key = key 53 | return nil 54 | } 55 | 56 | func init() { 57 | Register(HWatermakrMaker, HandlerToCreator(&watermark{})) 58 | } 59 | -------------------------------------------------------------------------------- /processor/processor.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "context" 5 | "yamdc/model" 6 | ) 7 | 8 | type IProcessor interface { 9 | Name() string 10 | Process(ctx context.Context, meta *model.FileContext) error 11 | } 12 | -------------------------------------------------------------------------------- /resource/image/4k.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxxsen/yamdc/439ef7bc33b835f3bd926283250c9a0dafae98ab/resource/image/4k.png -------------------------------------------------------------------------------- /resource/image/8k.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxxsen/yamdc/439ef7bc33b835f3bd926283250c9a0dafae98ab/resource/image/8k.png -------------------------------------------------------------------------------- /resource/image/README.md: -------------------------------------------------------------------------------- 1 | resource 2 | === 3 | 4 | 所有的水印图都使用相同大小, 800x400(圆角大小:180像素), 方便后续处理。 5 | 6 | 重建分辨率命令(尽量保持宽高比为2:1, 该命令不会填黑边): 7 | 8 | ```shell 9 | ffmpeg -i ./leak.png -s "800x400" leak_convert.png 10 | ``` -------------------------------------------------------------------------------- /resource/image/hack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxxsen/yamdc/439ef7bc33b835f3bd926283250c9a0dafae98ab/resource/image/hack.png -------------------------------------------------------------------------------- /resource/image/leak.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxxsen/yamdc/439ef7bc33b835f3bd926283250c9a0dafae98ab/resource/image/leak.png -------------------------------------------------------------------------------- /resource/image/subtitle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxxsen/yamdc/439ef7bc33b835f3bd926283250c9a0dafae98ab/resource/image/subtitle.png -------------------------------------------------------------------------------- /resource/image/uncensored.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxxsen/yamdc/439ef7bc33b835f3bd926283250c9a0dafae98ab/resource/image/uncensored.png -------------------------------------------------------------------------------- /resource/image/vr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxxsen/yamdc/439ef7bc33b835f3bd926283250c9a0dafae98ab/resource/image/vr.png -------------------------------------------------------------------------------- /resource/json/c_number.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxxsen/yamdc/439ef7bc33b835f3bd926283250c9a0dafae98ab/resource/json/c_number.json.gz -------------------------------------------------------------------------------- /resource/resource.go: -------------------------------------------------------------------------------- 1 | package resource 2 | 3 | import ( 4 | _ "embed" 5 | ) 6 | 7 | //go:embed image/subtitle.png 8 | var ResIMGSubtitle []byte 9 | 10 | //go:embed image/uncensored.png 11 | var ResIMGUncensored []byte 12 | 13 | //go:embed image/4k.png 14 | var ResIMG4K []byte 15 | 16 | //go:embed image/leak.png 17 | var ResIMGLeak []byte 18 | 19 | //go:embed image/8k.png 20 | var ResIMG8K []byte 21 | 22 | //go:embed image/vr.png 23 | var ResIMGVR []byte 24 | 25 | //go:embed image/hack.png 26 | var ResIMGHack []byte 27 | 28 | //go:embed json/c_number.json.gz 29 | var ResCNumber []byte //数据来源为mdcx 30 | -------------------------------------------------------------------------------- /scripts/build_archive.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$#" != "3" ]; then 4 | echo "$0"' $os $arch $filename' 5 | echo "example:" 6 | echo "-- $0 windows amd64 yamdc" 7 | echo "-- $0 linux amd64 yamdc" 8 | exit 1 9 | fi 10 | 11 | os="$1" 12 | arch="$2" 13 | filename="$3" 14 | output="${filename}-${os}-${arch}" 15 | bname="$output" 16 | if [ "$os" == "windows" ]; then 17 | bname="$bname.exe" 18 | fi 19 | 20 | CGO_LDFLAGS="-static" CGO_ENABLED=1 GOOS=${os} GOARCH=${arch} go build -a -tags netgo -ldflags '-w' -o ${bname} ./ 21 | tar -czf "$output.tar.gz" "$bname" 22 | rm $bname -------------------------------------------------------------------------------- /scripts/download_models.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | SAVEDIR="./models" 6 | 7 | if [ "$#" != "0" ]; then 8 | SAVEDIR="$1" 9 | fi 10 | 11 | echo "downloading model files to dir:$SAVEDIR..." 12 | 13 | if [ -d "$SAVEDIR" ]; then 14 | echo "DIR: $SAVEDIR exist, remove it first!" 15 | exit 0 16 | fi 17 | 18 | mkdir "$SAVEDIR" -p 19 | curl https://github.com/Kagami/go-face-testdata/raw/master/models/shape_predictor_5_face_landmarks.dat -L -o "$SAVEDIR/shape_predictor_5_face_landmarks.dat" 20 | curl https://github.com/Kagami/go-face-testdata/raw/master/models/dlib_face_recognition_resnet_model_v1.dat -L -o "$SAVEDIR/dlib_face_recognition_resnet_model_v1.dat" 21 | curl https://github.com/Kagami/go-face-testdata/raw/master/models/mmod_human_face_detector.dat -L -o "$SAVEDIR/mmod_human_face_detector.dat" 22 | 23 | echo "model files download succ" 24 | -------------------------------------------------------------------------------- /scripts/install_deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$#" != "1" ]; then 4 | echo "$0"' $os' 5 | echo "example:" 6 | echo "-- $0 linux" 7 | exit 1 8 | fi 9 | 10 | os="$1" 11 | 12 | if [ "$os" != "linux" ]; then 13 | exit 0 14 | fi 15 | 16 | sudo apt update 17 | sudo apt-get install libdlib-dev libblas-dev libatlas-base-dev liblapack-dev libjpeg62-turbo-dev gfortran -y 18 | 19 | -------------------------------------------------------------------------------- /searcher/category_searcher.go: -------------------------------------------------------------------------------- 1 | package searcher 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "yamdc/model" 7 | "yamdc/number" 8 | 9 | "github.com/xxxsen/common/logutil" 10 | "go.uber.org/zap" 11 | ) 12 | 13 | type categorySearcher struct { 14 | defSearcher []ISearcher 15 | catSearchers map[string][]ISearcher 16 | } 17 | 18 | func NewCategorySearcher(def []ISearcher, cats map[string][]ISearcher) ISearcher { 19 | return &categorySearcher{defSearcher: def, catSearchers: cats} 20 | } 21 | 22 | func (s *categorySearcher) Name() string { 23 | return "category" 24 | } 25 | 26 | func (s *categorySearcher) Check(ctx context.Context) error { 27 | return fmt.Errorf("unable to perform check on category searcher") 28 | } 29 | 30 | func (s *categorySearcher) Search(ctx context.Context, n *number.Number) (*model.MovieMeta, bool, error) { 31 | cat := n.GetExternalFieldCategory() 32 | //没分类, 那么使用主链进行查询 33 | //存在分类, 但是分类对应的链没有配置, 则使用主链进行查询 34 | //如果已经存在分类链, 则不再进行降级 35 | logger := logutil.GetLogger(ctx).With(zap.String("cat", string(cat))) 36 | chain := s.defSearcher 37 | if len(cat) > 0 { 38 | if c, ok := s.catSearchers[cat]; ok { 39 | chain = c 40 | logger.Debug("use cat chain for search") 41 | } else { 42 | logger.Error("no cat chain found, use default plugin chain for search") 43 | } 44 | } 45 | 46 | return performGroupSearch(ctx, n, chain) 47 | } 48 | -------------------------------------------------------------------------------- /searcher/config.go: -------------------------------------------------------------------------------- 1 | package searcher 2 | 3 | import "yamdc/client" 4 | 5 | type config struct { 6 | cli client.IHTTPClient 7 | searchCache bool 8 | } 9 | 10 | type Option func(c *config) 11 | 12 | func WithHTTPClient(cli client.IHTTPClient) Option { 13 | return func(c *config) { 14 | c.cli = cli 15 | } 16 | } 17 | 18 | func WithSearchCache(v bool) Option { 19 | return func(c *config) { 20 | c.searchCache = v 21 | } 22 | } 23 | 24 | func applyOpts(opts ...Option) *config { 25 | c := &config{ 26 | cli: client.DefaultClient(), 27 | } 28 | for _, opt := range opts { 29 | opt(c) 30 | } 31 | return c 32 | } 33 | -------------------------------------------------------------------------------- /searcher/decoder/config.go: -------------------------------------------------------------------------------- 1 | package decoder 2 | 3 | import "strconv" 4 | 5 | type StringParseFunc func(v string) string 6 | type StringListParseFunc func(v []string) []string 7 | type NumberParseFunc func(v string) int64 8 | 9 | type config struct { 10 | OnNumberParse StringParseFunc 11 | OnTitleParse StringParseFunc 12 | OnPlotParse StringParseFunc 13 | OnActorListParse StringListParseFunc 14 | OnReleaseDateParse NumberParseFunc 15 | OnDurationParse NumberParseFunc 16 | OnStudioParse StringParseFunc 17 | OnLabelParse StringParseFunc 18 | OnSeriesParse StringParseFunc 19 | OnGenreListParse StringListParseFunc 20 | OnCoverParse StringParseFunc 21 | OnDirectorParse StringParseFunc 22 | OnPosterParse StringParseFunc 23 | OnSampleImageListParse StringListParseFunc 24 | DefaultStringProcessor StringParseFunc 25 | DefaultStringListProcessor StringListParseFunc 26 | } 27 | 28 | func defaultStringParser(v string) string { 29 | return v 30 | } 31 | 32 | func defaultStringListParser(vs []string) []string { 33 | return vs 34 | } 35 | 36 | func defaultNumberParser(v string) int64 { 37 | res, _ := strconv.ParseInt(v, 10, 64) 38 | return res 39 | } 40 | 41 | func defaultStringProcessor(v string) string { 42 | return v 43 | } 44 | 45 | func defaultStringListProcessor(vs []string) []string { 46 | return vs 47 | } 48 | 49 | type Option func(c *config) 50 | 51 | func WithDefaultStringProcessor(p StringParseFunc) Option { 52 | return func(c *config) { 53 | c.DefaultStringProcessor = p 54 | } 55 | } 56 | 57 | func WithDefaultStringListProcessor(p StringListParseFunc) Option { 58 | return func(c *config) { 59 | c.DefaultStringListProcessor = p 60 | } 61 | } 62 | 63 | func WithSampleImageListParser(p StringListParseFunc) Option { 64 | return func(c *config) { 65 | c.OnSampleImageListParse = p 66 | } 67 | } 68 | 69 | func WithPosterParser(p StringParseFunc) Option { 70 | return func(c *config) { 71 | c.OnPosterParse = p 72 | } 73 | } 74 | 75 | func WithCoverParser(p StringParseFunc) Option { 76 | return func(c *config) { 77 | c.OnCoverParse = p 78 | } 79 | } 80 | 81 | func WithGenreListParser(p StringListParseFunc) Option { 82 | return func(c *config) { 83 | c.OnGenreListParse = p 84 | } 85 | } 86 | 87 | func WithSeriesParser(p StringParseFunc) Option { 88 | return func(c *config) { 89 | c.OnSeriesParse = p 90 | } 91 | } 92 | 93 | func WithLabelParser(p StringParseFunc) Option { 94 | return func(c *config) { 95 | c.OnLabelParse = p 96 | } 97 | } 98 | 99 | func WithStudioParser(p StringParseFunc) Option { 100 | return func(c *config) { 101 | c.OnStudioParse = p 102 | } 103 | } 104 | 105 | func WithDurationParser(p NumberParseFunc) Option { 106 | return func(c *config) { 107 | c.OnDurationParse = p 108 | } 109 | } 110 | 111 | func WithReleaseDateParser(p NumberParseFunc) Option { 112 | return func(c *config) { 113 | c.OnReleaseDateParse = p 114 | } 115 | } 116 | 117 | func WithActorListParser(p StringListParseFunc) Option { 118 | return func(c *config) { 119 | c.OnActorListParse = p 120 | } 121 | } 122 | 123 | func WithPlotParser(p StringParseFunc) Option { 124 | return func(c *config) { 125 | c.OnPlotParse = p 126 | } 127 | } 128 | 129 | func WithTitleParser(p StringParseFunc) Option { 130 | return func(c *config) { 131 | c.OnTitleParse = p 132 | } 133 | } 134 | 135 | func WithNumberParser(p StringParseFunc) Option { 136 | return func(c *config) { 137 | c.OnNumberParse = p 138 | } 139 | } 140 | 141 | func WithDirectorParser(p StringParseFunc) Option { 142 | return func(c *config) { 143 | c.OnDirectorParse = p 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /searcher/group_searcher.go: -------------------------------------------------------------------------------- 1 | package searcher 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "yamdc/model" 7 | "yamdc/number" 8 | 9 | "github.com/xxxsen/common/logutil" 10 | "go.uber.org/zap" 11 | ) 12 | 13 | type group struct { 14 | ss []ISearcher 15 | } 16 | 17 | func NewGroup(ss []ISearcher) ISearcher { 18 | return &group{ss: ss} 19 | } 20 | func (g *group) Name() string { 21 | return "group" 22 | } 23 | 24 | func (g *group) Search(ctx context.Context, number *number.Number) (*model.MovieMeta, bool, error) { 25 | return performGroupSearch(ctx, number, g.ss) 26 | } 27 | 28 | func (g *group) Check(ctx context.Context) error { 29 | return fmt.Errorf("unable to perform check on group searcher") 30 | } 31 | 32 | func performGroupSearch(ctx context.Context, number *number.Number, ss []ISearcher) (*model.MovieMeta, bool, error) { 33 | var lastErr error 34 | for _, s := range ss { 35 | logutil.GetLogger(ctx).Debug("search number", zap.String("plugin", s.Name())) 36 | meta, found, err := s.Search(ctx, number) 37 | if err != nil { 38 | lastErr = err 39 | continue 40 | } 41 | if !found { 42 | continue 43 | } 44 | return meta, true, nil 45 | } 46 | if lastErr != nil { 47 | return nil, false, lastErr 48 | } 49 | return nil, false, nil 50 | } 51 | -------------------------------------------------------------------------------- /searcher/parser/date_parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "context" 5 | "time" 6 | "yamdc/searcher/decoder" 7 | 8 | "github.com/xxxsen/common/logutil" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | func DateOnlyReleaseDateParser(ctx context.Context) decoder.NumberParseFunc { 13 | return func(v string) int64 { 14 | t, err := time.Parse(time.DateOnly, v) 15 | if err != nil { 16 | logutil.GetLogger(ctx).Error("decode release date failed", zap.Error(err), zap.String("data", v)) 17 | return 0 18 | } 19 | return t.UnixMilli() 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /searcher/parser/duration_parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "math" 7 | "regexp" 8 | "strconv" 9 | "strings" 10 | "yamdc/searcher/decoder" 11 | 12 | "github.com/xxxsen/common/logutil" 13 | "go.uber.org/zap" 14 | ) 15 | 16 | var ( 17 | defaultDurationRegexp = regexp.MustCompile(`\s*(\d+)\s*.+`) 18 | ) 19 | 20 | func cleanTimeSequence(res string) []string { 21 | list := strings.Split(res, ":") 22 | rs := make([]string, 0, len(list)) 23 | for _, item := range list { 24 | rs = append(rs, strings.TrimSpace(item)) 25 | } 26 | return rs 27 | } 28 | 29 | func DefaultMMDurationParser(ctx context.Context) decoder.NumberParseFunc { 30 | return func(v string) int64 { 31 | res, _ := strconv.ParseInt(v, 10, 64) 32 | return res * 60 // convert minutes to seconds 33 | } 34 | } 35 | 36 | func DefaultHHMMSSDurationParser(ctx context.Context) decoder.NumberParseFunc { 37 | return func(v string) int64 { 38 | res := cleanTimeSequence(v) 39 | if len(res) > 3 { 40 | logutil.GetLogger(ctx).Error("invalid time format", zap.String("data", v)) 41 | return 0 42 | } 43 | var sec int64 44 | for i := 0; i < len(res); i++ { 45 | item := strings.TrimSpace(res[len(res)-i-1]) 46 | val, err := strconv.ParseInt(item, 10, 60) 47 | if err != nil { 48 | logutil.GetLogger(ctx).Error("invalid time format", zap.String("data", v)) 49 | return 0 50 | } 51 | sec += val * int64(math.Pow(60, float64(i))) 52 | } 53 | return sec 54 | } 55 | } 56 | 57 | func DefaultDurationParser(ctx context.Context) decoder.NumberParseFunc { 58 | return func(v string) int64 { 59 | val, err := toDuration(v) 60 | if err != nil { 61 | logutil.GetLogger(ctx).Error("decode duration failed", zap.Error(err), zap.String("data", v)) 62 | return 0 63 | } 64 | return val 65 | } 66 | } 67 | 68 | func MinuteOnlyDurationParser(ctx context.Context) decoder.NumberParseFunc { 69 | return func(v string) int64 { 70 | intv, err := strconv.ParseInt(v, 10, 64) 71 | if err != nil { 72 | logutil.GetLogger(ctx).Error("decode minute only duration failed", zap.Error(err), zap.String("data", v)) 73 | return 0 74 | } 75 | return intv * 60 // convert minutes to seconds 76 | } 77 | } 78 | 79 | func toDuration(timeStr string) (int64, error) { 80 | re := defaultDurationRegexp 81 | matches := re.FindStringSubmatch(timeStr) 82 | if len(matches) <= 1 { 83 | return 0, errors.New("invalid time format") 84 | } 85 | 86 | number, err := strconv.Atoi(matches[1]) 87 | if err != nil { 88 | return 0, err 89 | } 90 | seconds := number * 60 91 | 92 | return int64(seconds), nil 93 | } 94 | -------------------------------------------------------------------------------- /searcher/parser/duration_parser_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | type testPair struct { 11 | in string 12 | sec int64 13 | } 14 | 15 | func TestHHMMSS(t *testing.T) { 16 | tests := []testPair{ 17 | {in: "01 :01: 01", sec: 1*3600 + 60 + 1}, 18 | {in: "02: 05", sec: 2*60 + 5}, 19 | } 20 | for _, tst := range tests { 21 | out := DefaultHHMMSSDurationParser(context.Background())(tst.in) 22 | assert.Equal(t, tst.sec, out) 23 | } 24 | } 25 | 26 | func TestConv(t *testing.T) { 27 | sts := []struct { 28 | in string 29 | out int64 30 | }{ 31 | {"47分钟", 47 * 60}, 32 | {" 10分钟", 600}, 33 | {"140分", 140 * 60}, 34 | {"117分鐘", 117 * 60}, 35 | } 36 | for _, st := range sts { 37 | out, err := toDuration(st.in) 38 | assert.NoError(t, err) 39 | assert.Equal(t, st.out, out) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /searcher/plugin/api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "yamdc/model" 7 | ) 8 | 9 | type HTTPInvoker func(ctx context.Context, req *http.Request) (*http.Response, error) 10 | 11 | type IPlugin interface { 12 | OnGetHosts(ctx context.Context) []string 13 | OnPrecheckRequest(ctx context.Context, number string) (bool, error) 14 | OnMakeHTTPRequest(ctx context.Context, number string) (*http.Request, error) 15 | OnDecorateRequest(ctx context.Context, req *http.Request) error 16 | OnHandleHTTPRequest(ctx context.Context, invoker HTTPInvoker, req *http.Request) (*http.Response, error) 17 | OnPrecheckResponse(ctx context.Context, req *http.Request, rsp *http.Response) (bool, error) 18 | OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) 19 | OnDecorateMediaRequest(ctx context.Context, req *http.Request) error 20 | } 21 | -------------------------------------------------------------------------------- /searcher/plugin/api/container.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "maps" 7 | ) 8 | 9 | type container struct { 10 | m map[string]string 11 | } 12 | 13 | type containerType struct{} 14 | 15 | var ( 16 | defaultContainerTypeKey = containerType{} 17 | ) 18 | 19 | func InitContainer(ctx context.Context) context.Context { 20 | c := &container{ 21 | m: make(map[string]string), 22 | } 23 | return context.WithValue(ctx, defaultContainerTypeKey, c) 24 | } 25 | 26 | func SetKeyValue(ctx context.Context, key string, value string) { 27 | c := ctx.Value(defaultContainerTypeKey).(*container) 28 | c.m[key] = value 29 | } 30 | 31 | func GetKeyValue(ctx context.Context, key string) (string, bool) { 32 | c := ctx.Value(defaultContainerTypeKey).(*container) 33 | v, ok := c.m[key] 34 | return v, ok 35 | } 36 | 37 | func MustGetKeyValue(ctx context.Context, key string) string { 38 | c := ctx.Value(defaultContainerTypeKey).(*container) 39 | v, ok := c.m[key] 40 | if !ok { 41 | panic(fmt.Errorf("key:%s not found", key)) 42 | } 43 | return v 44 | } 45 | 46 | func ExportContainerData(ctx context.Context) map[string]string { 47 | c := ctx.Value(defaultContainerTypeKey).(*container) 48 | m := make(map[string]string) 49 | maps.Copy(m, c.m) 50 | return m 51 | } 52 | 53 | func ImportContainerData(ctx context.Context, m map[string]string) { 54 | c := ctx.Value(defaultContainerTypeKey).(*container) 55 | maps.Copy(c.m, m) 56 | } 57 | -------------------------------------------------------------------------------- /searcher/plugin/api/defaults.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "yamdc/model" 8 | ) 9 | 10 | type DefaultPlugin struct { 11 | } 12 | 13 | func (p *DefaultPlugin) OnGetHosts(ctx context.Context) []string { 14 | return []string{} 15 | } 16 | 17 | func (p *DefaultPlugin) OnPrecheckRequest(ctx context.Context, number string) (bool, error) { 18 | return true, nil 19 | } 20 | 21 | func (p *DefaultPlugin) OnMakeHTTPRequest(ctx context.Context, number string) (*http.Request, error) { 22 | return nil, fmt.Errorf("no impl") 23 | } 24 | 25 | func (p *DefaultPlugin) OnDecorateRequest(ctx context.Context, req *http.Request) error { 26 | return nil 27 | } 28 | 29 | func (p *DefaultPlugin) OnPrecheckResponse(ctx context.Context, req *http.Request, rsp *http.Response) (bool, error) { 30 | if rsp.StatusCode == http.StatusNotFound { 31 | return false, nil 32 | } 33 | return true, nil 34 | } 35 | 36 | func (p *DefaultPlugin) OnHandleHTTPRequest(ctx context.Context, invoker HTTPInvoker, req *http.Request) (*http.Response, error) { 37 | return invoker(ctx, req) 38 | } 39 | 40 | func (p *DefaultPlugin) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 41 | return nil, false, fmt.Errorf("no impl") 42 | } 43 | 44 | func (p *DefaultPlugin) OnDecorateMediaRequest(ctx context.Context, req *http.Request) error { 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /searcher/plugin/api/domain_selector.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import "math/rand" 4 | 5 | func SelectDomain(in []string) (string, bool) { 6 | if len(in) == 0 { 7 | return "", false 8 | } 9 | if len(in) == 1 { 10 | return in[0], true 11 | } 12 | return in[rand.Int()%len(in)], true 13 | } 14 | 15 | func MustSelectDomain(in []string) string { 16 | res, ok := SelectDomain(in) 17 | if !ok { 18 | panic("unable to select domain") 19 | } 20 | return res 21 | } 22 | -------------------------------------------------------------------------------- /searcher/plugin/constant/constant.go: -------------------------------------------------------------------------------- 1 | package constant 2 | 3 | const ( 4 | SSJavBus = "javbus" 5 | SSJav321 = "jav321" 6 | SSFc2 = "fc2" 7 | SSCaribpr = "caribpr" 8 | SSJavhoo = "javhoo" 9 | SSAvsox = "avsox" 10 | SSAirav = "airav" 11 | SSFreeJavBt = "freejavbt" 12 | SSJavDB = "javdb" 13 | SS18AV = "18av" 14 | SSTKTube = "tktube" 15 | SSNJav = "njav" 16 | SSFc2PPVDB = "fc2ppvdb" 17 | SSMissav = "missav" 18 | SSJvrPorn = "jvrporn" 19 | SSJavLibrary = "javlibrary" 20 | SSCospuri = "cospuri" 21 | SSMadouqu = "madouqu" 22 | ) 23 | -------------------------------------------------------------------------------- /searcher/plugin/factory/factory.go: -------------------------------------------------------------------------------- 1 | package factory 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "yamdc/searcher/plugin/api" 7 | ) 8 | 9 | type CreatorFunc func(args interface{}) (api.IPlugin, error) 10 | 11 | var mp = make(map[string]CreatorFunc) 12 | 13 | func Register(name string, fn CreatorFunc) { 14 | mp[name] = fn 15 | } 16 | 17 | func CreatePlugin(name string, args interface{}) (api.IPlugin, error) { 18 | cr, ok := mp[name] 19 | if !ok { 20 | return nil, fmt.Errorf("plugin:%s not found", name) 21 | } 22 | return cr(args) 23 | } 24 | 25 | func PluginToCreator(plg api.IPlugin) CreatorFunc { 26 | return func(args interface{}) (api.IPlugin, error) { 27 | return plg, nil 28 | } 29 | } 30 | 31 | func Plugins() []string { 32 | rs := make([]string, 0, len(mp)) 33 | for k := range mp { 34 | rs = append(rs, k) 35 | } 36 | return sort.StringSlice(rs) 37 | } 38 | -------------------------------------------------------------------------------- /searcher/plugin/impl/18av.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "strings" 8 | "yamdc/model" 9 | "yamdc/searcher/decoder" 10 | "yamdc/searcher/parser" 11 | "yamdc/searcher/plugin/api" 12 | "yamdc/searcher/plugin/constant" 13 | "yamdc/searcher/plugin/factory" 14 | "yamdc/searcher/plugin/meta" 15 | "yamdc/searcher/plugin/twostep" 16 | ) 17 | 18 | var default18AvHostList = []string{ 19 | "https://18av.me", 20 | } 21 | 22 | type av18 struct { 23 | api.DefaultPlugin 24 | } 25 | 26 | func (p *av18) OnGetHosts(ctx context.Context) []string { 27 | return default18AvHostList 28 | } 29 | 30 | func (p *av18) OnMakeHTTPRequest(ctx context.Context, number string) (*http.Request, error) { 31 | host := api.MustSelectDomain(default18AvHostList) 32 | uri := fmt.Sprintf("%s/cn/search.php?kw_type=key&kw=%s", host, number) 33 | return http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) 34 | } 35 | 36 | func (p *av18) OnHandleHTTPRequest(ctx context.Context, invoker api.HTTPInvoker, req *http.Request) (*http.Response, error) { 37 | xctx := &twostep.XPathTwoStepContext{ 38 | Ps: []*twostep.XPathPair{ 39 | { 40 | Name: "read-link", 41 | XPath: `//div[@class="content flex-columns small px-2"]/span[@class="title"]/a/@href`, 42 | }, 43 | { 44 | Name: "read-title", 45 | XPath: `//div[@class="content flex-columns small px-2"]/span[@class="title"]/a/text()`, 46 | }, 47 | }, 48 | LinkSelector: func(ps []*twostep.XPathPair) (string, bool, error) { 49 | number := strings.ToUpper(meta.GetNumberId(ctx)) 50 | linkList := ps[0].Result 51 | titleList := ps[1].Result 52 | for idx, link := range linkList { 53 | title := titleList[idx] 54 | if strings.Contains(strings.ToUpper(title), number) { 55 | return link, true, nil 56 | } 57 | } 58 | return "", false, nil 59 | }, 60 | ValidStatusCode: []int{http.StatusOK}, 61 | CheckResultCountMatch: true, 62 | LinkPrefix: fmt.Sprintf("%s://%s/cn", req.URL.Scheme, req.URL.Host), 63 | } 64 | return twostep.HandleXPathTwoStepSearch(ctx, invoker, req, xctx) 65 | } 66 | 67 | func (p *av18) coverParser(in string) string { 68 | return strings.ReplaceAll(in, " ", "") 69 | } 70 | 71 | func (p *av18) plotParser(in string) string { 72 | return strings.TrimSpace(strings.TrimLeft(in, "简介：")) 73 | } 74 | 75 | func (p *av18) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 76 | dec := decoder.XPathHtmlDecoder{ 77 | NumberExpr: `//div[@class="px-0 flex-columns"]/div[@class="number"]/text()`, 78 | TitleExpr: `//div[@class="d-flex px-3 py-2 name col bg-w"]/h1[@class="h4 b"]/text()`, 79 | PlotExpr: `//div[@class="intro bd-light w-100 mt-1"]/p[contains(text(), '简介：')]/text()`, 80 | ActorListExpr: `//div[@class="d-flex col px-0 tag-info flex-wrap mt-2 pt-2 bd-top bd-primary"]/a/span[@itemprop="name"]/text()`, 81 | ReleaseDateExpr: `//div[@class="date"]/text()`, 82 | DurationExpr: "", 83 | StudioExpr: "", 84 | LabelExpr: "", 85 | DirectorExpr: "", 86 | SeriesExpr: `//div[@class="bd-top my-1 align-items-center"]/a[@class="btn btn-ripple border-pill px-3 mr-2 my-1 bg-primary"]`, 87 | GenreListExpr: `//div[@class="d-flex col px-0 tag-info flex-wrap mt-2 pt-2 bd-top bd-primary"]/a[contains(@href, "s_type=tag")]/text()`, 88 | CoverExpr: `//meta[@property="og:image"]/@content`, 89 | PosterExpr: "", 90 | SampleImageListExpr: `//div[@class="cover"]/a/img/@data-src`, 91 | } 92 | meta, err := dec.DecodeHTML(data, 93 | decoder.WithCoverParser(p.coverParser), 94 | decoder.WithPlotParser(p.plotParser), 95 | decoder.WithDurationParser(parser.DefaultDurationParser(ctx)), 96 | decoder.WithReleaseDateParser(parser.DateOnlyReleaseDateParser(ctx)), 97 | ) 98 | if err != nil { 99 | return nil, false, err 100 | } 101 | if len(meta.Number) == 0 { 102 | return nil, false, nil 103 | } 104 | return meta, true, nil 105 | } 106 | 107 | func init() { 108 | factory.Register(constant.SS18AV, factory.PluginToCreator(&av18{})) 109 | } 110 | -------------------------------------------------------------------------------- /searcher/plugin/impl/airav/airav.go: -------------------------------------------------------------------------------- 1 | package airav 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "strings" 9 | "yamdc/model" 10 | "yamdc/searcher/parser" 11 | "yamdc/searcher/plugin/api" 12 | "yamdc/searcher/plugin/constant" 13 | "yamdc/searcher/plugin/factory" 14 | 15 | "github.com/xxxsen/common/logutil" 16 | "go.uber.org/zap" 17 | ) 18 | 19 | var defaultAirAvHostList = []string{ 20 | "https://www.airav.wiki", 21 | } 22 | 23 | type airav struct { 24 | api.DefaultPlugin 25 | } 26 | 27 | func (p *airav) OnGetHosts(ctx context.Context) []string { 28 | return defaultAirAvHostList 29 | } 30 | 31 | func (p *airav) OnMakeHTTPRequest(ctx context.Context, number string) (*http.Request, error) { 32 | domain := api.MustSelectDomain(defaultAirAvHostList) 33 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/api/video/barcode/%s?lng=zh-TW", domain, number), nil) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return req, nil 38 | } 39 | 40 | func (p *airav) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 41 | vdata := &VideoData{} 42 | if err := json.Unmarshal(data, vdata); err != nil { 43 | return nil, false, fmt.Errorf("decode json data failed, err:%w", err) 44 | } 45 | if !strings.EqualFold(vdata.Status, "ok") { 46 | return nil, false, fmt.Errorf("search result:`%s`, not ok", vdata.Status) 47 | } 48 | if vdata.Count == 0 { 49 | return nil, false, nil 50 | } 51 | if vdata.Count > 1 { 52 | logutil.GetLogger(ctx).Warn("more than one result, may cause data mismatch", zap.Int("count", vdata.Count)) 53 | } 54 | result := vdata.Result 55 | avdata := &model.MovieMeta{ 56 | Number: result.Barcode, 57 | Title: result.Name, 58 | Plot: result.Description, 59 | Actors: p.readActors(&result), 60 | ReleaseDate: parser.DateOnlyReleaseDateParser(ctx)(result.PublishDate), 61 | Studio: p.readStudio(&result), 62 | Genres: p.readGenres(&result), 63 | Cover: &model.File{ 64 | Name: result.ImgURL, 65 | }, 66 | SampleImages: p.readSampleImages(&result), 67 | } 68 | return avdata, true, nil 69 | } 70 | 71 | func (p *airav) readSampleImages(result *Result) []*model.File { 72 | rs := make([]*model.File, 0, len(result.Images)) 73 | for _, item := range result.Images { 74 | rs = append(rs, &model.File{ 75 | Name: item, 76 | }) 77 | } 78 | return rs 79 | } 80 | 81 | func (p *airav) readGenres(result *Result) []string { 82 | rs := make([]string, 0, len(result.Tags)) 83 | for _, item := range result.Tags { 84 | rs = append(rs, item.Name) 85 | } 86 | return rs 87 | } 88 | 89 | func (p *airav) readStudio(result *Result) string { 90 | if len(result.Factories) > 0 { 91 | return result.Factories[0].Name 92 | } 93 | return "" 94 | } 95 | 96 | func (p *airav) readActors(result *Result) []string { 97 | rs := make([]string, 0, len(result.Actors)) 98 | for _, item := range result.Actors { 99 | rs = append(rs, item.Name) 100 | } 101 | return rs 102 | } 103 | 104 | func init() { 105 | factory.Register(constant.SSAirav, factory.PluginToCreator(&airav{})) 106 | } 107 | -------------------------------------------------------------------------------- /searcher/plugin/impl/airav/model.go: -------------------------------------------------------------------------------- 1 | package airav 2 | 3 | type VideoData struct { 4 | Count int `json:"count"` 5 | Result Result `json:"result"` 6 | Status string `json:"status"` 7 | } 8 | 9 | type Result struct { 10 | ID int `json:"id"` 11 | Vid string `json:"vid"` 12 | Slug interface{} `json:"slug"` 13 | Barcode string `json:"barcode"` 14 | ActorsName string `json:"actors_name"` 15 | Name string `json:"name"` 16 | ImgURL string `json:"img_url"` 17 | OtherImages []string `json:"other_images"` 18 | Photo interface{} `json:"photo"` 19 | PublishDate string `json:"publish_date"` 20 | Description string `json:"description"` 21 | Actors []Actor `json:"actors"` 22 | Images []string `json:"images"` 23 | Tags []Tag `json:"tags"` 24 | Factories []Factory `json:"factories"` 25 | MaybeLike []MaybeLike `json:"maybe_like_videos"` 26 | QCURL string `json:"qc_url"` 27 | View int `json:"view"` 28 | OtherDesc interface{} `json:"other_desc"` 29 | VideoURL VideoURL `json:"video_url"` 30 | } 31 | 32 | type Actor struct { 33 | Name string `json:"name"` 34 | NameCn string `json:"name_cn"` 35 | NameJp string `json:"name_jp"` 36 | NameEn string `json:"name_en"` 37 | ID string `json:"id"` 38 | } 39 | 40 | type Tag struct { 41 | Name string `json:"name"` 42 | } 43 | 44 | type Factory struct { 45 | Name string `json:"name"` 46 | } 47 | 48 | type MaybeLike struct { 49 | Vid string `json:"vid"` 50 | Slug string `json:"slug"` 51 | Name string `json:"name"` 52 | URL string `json:"url"` 53 | ImgURL string `json:"img_url"` 54 | Barcode string `json:"barcode"` 55 | } 56 | 57 | type VideoURL struct { 58 | URLCdn string `json:"url_cdn"` 59 | URLHlsCdn string `json:"url_hls_cdn"` 60 | } 61 | -------------------------------------------------------------------------------- /searcher/plugin/impl/caribpr.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "strings" 9 | "time" 10 | "yamdc/enum" 11 | "yamdc/model" 12 | "yamdc/searcher/decoder" 13 | "yamdc/searcher/plugin/api" 14 | "yamdc/searcher/plugin/constant" 15 | "yamdc/searcher/plugin/factory" 16 | "yamdc/searcher/plugin/meta" 17 | "yamdc/utils" 18 | 19 | "github.com/xxxsen/common/logutil" 20 | "go.uber.org/zap" 21 | "golang.org/x/text/encoding/japanese" 22 | "golang.org/x/text/transform" 23 | ) 24 | 25 | var defaultCaribprHostList = []string{ 26 | "https://www.caribbeancompr.com", 27 | } 28 | 29 | type caribpr struct { 30 | api.DefaultPlugin 31 | } 32 | 33 | func (p *caribpr) OnGetHosts(ctx context.Context) []string { 34 | return defaultCaribprHostList 35 | } 36 | 37 | func (p *caribpr) OnMakeHTTPRequest(ctx context.Context, number string) (*http.Request, error) { 38 | uri := fmt.Sprintf("%s/moviepages/%s/index.html", api.MustSelectDomain(defaultCaribprHostList), number) 39 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) 40 | return req, err 41 | } 42 | 43 | func (p *caribpr) decodeDuration(ctx context.Context) decoder.NumberParseFunc { 44 | return func(v string) int64 { 45 | ts, err := utils.TimeStrToSecond(v) 46 | if err != nil { 47 | logutil.GetLogger(ctx).Error("parse duration failed", zap.String("duration", v), zap.Error(err)) 48 | return 0 49 | } 50 | return ts 51 | } 52 | } 53 | 54 | func (p *caribpr) decodeReleaseDate(ctx context.Context) decoder.NumberParseFunc { 55 | return func(v string) int64 { 56 | t, err := time.Parse(time.DateOnly, v) 57 | if err != nil { 58 | logutil.GetLogger(ctx).Error("parse release date failed", zap.String("release_date", v), zap.Error(err)) 59 | return 0 60 | } 61 | return t.UnixMilli() 62 | } 63 | } 64 | 65 | func (p *caribpr) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 66 | reader := transform.NewReader(strings.NewReader(string(data)), japanese.EUCJP.NewDecoder()) 67 | data, err := io.ReadAll(reader) 68 | if err != nil { 69 | return nil, false, fmt.Errorf("unable to decode with eucjp charset, err:%w", err) 70 | } 71 | dec := decoder.XPathHtmlDecoder{ 72 | TitleExpr: `//div[@class='movie-info']/div[@class='section is-wide']/div[@class='heading']/h1/text()`, 73 | PlotExpr: `//meta[@name="description"]/@content`, 74 | ActorListExpr: `//li[span[contains(text(), "出演")]]/span[@class="spec-content"]/a[@class="spec-item"]/text()`, 75 | ReleaseDateExpr: `//li[span[contains(text(), "販売日")]]/span[@class="spec-content"]/text()`, 76 | DurationExpr: `//li[span[contains(text(), "再生時間")]]/span[@class="spec-content"]/text()`, 77 | StudioExpr: `//li[span[contains(text(), "スタジオ")]]/span[@class="spec-content"]/a/text()`, 78 | LabelExpr: ``, 79 | SeriesExpr: `//li[span[contains(text(), "シリーズ")]]/span[@class="spec-content"]/a/text()`, 80 | GenreListExpr: `//li[span[contains(text(), "タグ")]]/span[@class="spec-content"]/a/text()`, 81 | CoverExpr: ``, 82 | PosterExpr: ``, 83 | SampleImageListExpr: `//div[@class='movie-gallery']/div[@class='section is-wide']/div[2]/div[@class='grid-item']/div/a/@href`, 84 | } 85 | metadata, err := dec.DecodeHTML(data, 86 | decoder.WithDurationParser(p.decodeDuration(ctx)), 87 | decoder.WithReleaseDateParser(p.decodeReleaseDate(ctx)), 88 | ) 89 | if err != nil { 90 | return nil, false, err 91 | } 92 | metadata.Number = meta.GetNumberId(ctx) 93 | metadata.Cover.Name = fmt.Sprintf("https://www.caribbeancompr.com/moviepages/%s/images/l_l.jpg", metadata.Number) //TODO: 看看能不能直接从元数据提取而不是直接拼链接 94 | metadata.TitleLang = enum.MetaLangJa 95 | metadata.PlotLang = enum.MetaLangJa 96 | return metadata, true, nil 97 | } 98 | 99 | func init() { 100 | factory.Register(constant.SSCaribpr, factory.PluginToCreator(&caribpr{})) 101 | } 102 | -------------------------------------------------------------------------------- /searcher/plugin/impl/cospuri_test.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestExtractModelAndID(t *testing.T) { 8 | plg := &cospuri{} 9 | 10 | tests := []struct { 11 | number string 12 | expectedModel string 13 | expectedID string 14 | expectError bool 15 | }{ 16 | {"COSPURI-Emiri-Momota-0548cpar", "Emiri-Momota", "0548cpar", false}, 17 | {"COSPURI-Emiri-Momota-0548", "Emiri-Momota", "0548", false}, 18 | {"COSPURI-Invalid-Format", "", "", true}, 19 | {"cospuri-Emiri-Momota-0548", "Emiri-Momota", "0548", false}, 20 | {"cospuri-AAAA-BBBBB-CCCCC-DDDDD-22223aaa", "Aaaa-Bbbbb-Ccccc-Ddddd", "22223aaa", false}, 21 | {"cospuri-AAAA-BBBBB-CCCCC-DDDDD-22223", "Aaaa-Bbbbb-Ccccc-Ddddd", "22223", false}, 22 | {"cospuri-12345abc", "", "12345abc", false}, 23 | } 24 | 25 | for _, test := range tests { 26 | model, id, err := plg.extractModelAndID(test.number) 27 | if (err != nil) != test.expectError { 28 | t.Errorf("extractModelAndID(%q) error = %v; want error = %v", test.number, err != nil, test.expectError) 29 | } 30 | if model != test.expectedModel || id != test.expectedID { 31 | t.Errorf("extractModelAndID(%q) = (%q, %q); want (%q, %q)", test.number, model, id, test.expectedModel, test.expectedID) 32 | } 33 | } 34 | } 35 | 36 | func TestNormalizeModel(t *testing.T) { 37 | plg := &cospuri{} 38 | 39 | tests := []struct { 40 | input string 41 | expected string 42 | }{ 43 | {"emiri-momota", "Emiri-Momota"}, 44 | {"EMIRI-MOMOTA", "Emiri-Momota"}, 45 | {"eMiRi-MoMoTa", "Emiri-Momota"}, 46 | {"singleword", "Singleword"}, 47 | } 48 | 49 | for _, test := range tests { 50 | result := plg.normalizeModel(test.input) 51 | if result != test.expected { 52 | t.Errorf("normalizeModel(%q) = %q; want %q", test.input, result, test.expected) 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /searcher/plugin/impl/fc2.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "strconv" 8 | "strings" 9 | "time" 10 | "yamdc/enum" 11 | "yamdc/model" 12 | "yamdc/numberkit" 13 | "yamdc/searcher/decoder" 14 | "yamdc/searcher/plugin/api" 15 | "yamdc/searcher/plugin/constant" 16 | "yamdc/searcher/plugin/factory" 17 | "yamdc/searcher/plugin/meta" 18 | 19 | "github.com/xxxsen/common/logutil" 20 | "go.uber.org/zap" 21 | ) 22 | 23 | var ( 24 | defaultFc2DomainList = []string{ 25 | "https://adult.contents.fc2.com", 26 | } 27 | ) 28 | 29 | type fc2 struct { 30 | api.DefaultPlugin 31 | } 32 | 33 | func (p *fc2) OnGetHosts(ctx context.Context) []string { 34 | return defaultFc2DomainList 35 | } 36 | 37 | func (p *fc2) OnMakeHTTPRequest(ctx context.Context, n string) (*http.Request, error) { 38 | nid, ok := numberkit.DecodeFc2ValID(n) 39 | if !ok { 40 | return nil, fmt.Errorf("unable to decode fc2 number") 41 | } 42 | uri := fmt.Sprintf("%s/article/%s/", api.MustSelectDomain(defaultFc2DomainList), nid) 43 | return http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) 44 | } 45 | 46 | func (p *fc2) decodeDuration(ctx context.Context) decoder.NumberParseFunc { 47 | return func(v string) int64 { 48 | logger := logutil.GetLogger(ctx).With(zap.String("duration", v)) 49 | res := strings.Split(v, ":") 50 | if len(res) != 2 { 51 | logger.Error("invalid duration str") 52 | return 0 53 | } 54 | min, err := strconv.ParseUint(res[0], 10, 64) 55 | if err != nil { 56 | logger.Error("decode miniute failed", zap.Error(err)) 57 | return 0 58 | } 59 | sec, err := strconv.ParseUint(res[1], 10, 64) 60 | if err != nil { 61 | logger.Error("decode second failed", zap.Error(err)) 62 | return 0 63 | } 64 | return int64(min*60 + sec) 65 | } 66 | } 67 | 68 | func (p *fc2) decodeReleaseDate(ctx context.Context) decoder.NumberParseFunc { 69 | return func(v string) int64 { 70 | logger := logutil.GetLogger(ctx).With(zap.String("releasedate", v)) 71 | res := strings.Split(v, ":") 72 | if len(res) != 2 { 73 | logger.Error("invalid release date str") 74 | return 0 75 | } 76 | date := strings.TrimSpace(res[1]) 77 | t, err := time.Parse("2006/01/02", date) 78 | if err != nil { 79 | logger.Error("parse date time failed", zap.Error(err)) 80 | return 0 81 | } 82 | return t.UnixMilli() 83 | } 84 | } 85 | 86 | func (p *fc2) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 87 | dec := decoder.XPathHtmlDecoder{ 88 | NumberExpr: ``, 89 | TitleExpr: `/html/head/title/text()`, 90 | ActorListExpr: `//*[@id="top"]/div[1]/section[1]/div/section/div[2]/ul/li[3]/a/text()`, 91 | ReleaseDateExpr: `//*[@id="top"]/div[1]/section[1]/div/section/div[2]/div[2]/p/text()`, 92 | DurationExpr: `//p[@class='items_article_info']/text()`, 93 | StudioExpr: `//*[@id="top"]/div[1]/section[1]/div/section/div[2]/ul/li[3]/a/text()`, 94 | LabelExpr: ``, 95 | SeriesExpr: ``, 96 | GenreListExpr: `//a[@class='tag tagTag']/text()`, 97 | CoverExpr: `//div[@class='items_article_MainitemThumb']/span/img/@src`, 98 | PosterExpr: `//div[@class='items_article_MainitemThumb']/span/img/@src`, //这东西就一张封面图, 直接当海报得了 99 | SampleImageListExpr: `//ul[@class="items_article_SampleImagesArea"]/li/a/@href`, 100 | } 101 | metadata, err := dec.DecodeHTML(data, 102 | decoder.WithDurationParser(p.decodeDuration(ctx)), 103 | decoder.WithReleaseDateParser(p.decodeReleaseDate(ctx)), 104 | ) 105 | if err != nil { 106 | return nil, false, err 107 | } 108 | metadata.Number = meta.GetNumberId(ctx) 109 | metadata.TitleLang = enum.MetaLangJa 110 | return metadata, true, nil 111 | } 112 | 113 | func init() { 114 | factory.Register(constant.SSFc2, factory.PluginToCreator(&fc2{})) 115 | } 116 | -------------------------------------------------------------------------------- /searcher/plugin/impl/fc2ppvdb.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "yamdc/enum" 8 | "yamdc/model" 9 | "yamdc/numberkit" 10 | "yamdc/searcher/decoder" 11 | "yamdc/searcher/parser" 12 | "yamdc/searcher/plugin/api" 13 | "yamdc/searcher/plugin/constant" 14 | "yamdc/searcher/plugin/factory" 15 | "yamdc/searcher/plugin/meta" 16 | ) 17 | 18 | var defaultFc2PPVDBDomains = []string{ 19 | "https://fc2ppvdb.com", 20 | } 21 | 22 | type fc2ppvdb struct { 23 | api.DefaultPlugin 24 | } 25 | 26 | func (p *fc2ppvdb) OnGetHosts(ctx context.Context) []string { 27 | return defaultFc2PPVDBDomains 28 | } 29 | 30 | func (p *fc2ppvdb) OnMakeHTTPRequest(ctx context.Context, nid string) (*http.Request, error) { 31 | vid, ok := numberkit.DecodeFc2ValID(nid) 32 | if !ok { 33 | return nil, fmt.Errorf("unable to decode fc2 vid") 34 | } 35 | link := fmt.Sprintf("%s/articles/%s", api.MustSelectDomain(defaultFc2PPVDBDomains), vid) 36 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, link, nil) 37 | if err != nil { 38 | return nil, err 39 | } 40 | return req, nil 41 | } 42 | 43 | func (p *fc2ppvdb) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 44 | dec := decoder.XPathHtmlDecoder{ 45 | NumberExpr: `//div[contains(text(), "ID")]/span/text()`, 46 | TitleExpr: `//div[@class="w-full lg:pl-8 px-2 lg:w-3/5"]/h2/a/text()`, 47 | PlotExpr: "", 48 | ActorListExpr: `//div[contains(text(), "女優")]/span/a/text()`, 49 | ReleaseDateExpr: `//div[contains(text(), "販売日")]/span/text()`, 50 | DurationExpr: `//div[contains(text(), "収録時間")]/span/text()`, 51 | StudioExpr: `//div[contains(text(), "販売者")]/span/a/text()`, 52 | LabelExpr: "", 53 | DirectorExpr: `//div[contains(text(), "販売者")]/span/a/text()`, 54 | SeriesExpr: "", 55 | GenreListExpr: `//div[contains(text(), "タグ")]/span/a/text()`, 56 | CoverExpr: `//div[@class="lg:w-2/5 w-full mb-12 md:mb-0"]/a/img/@src`, 57 | PosterExpr: `//div[@class="lg:w-2/5 w-full mb-12 md:mb-0"]/a/img/@src`, 58 | SampleImageListExpr: "", 59 | } 60 | mdata, err := dec.DecodeHTML(data, 61 | decoder.WithReleaseDateParser(parser.DateOnlyReleaseDateParser(ctx)), 62 | decoder.WithDurationParser(parser.DefaultHHMMSSDurationParser(ctx)), 63 | ) 64 | if err != nil { 65 | return nil, false, err 66 | } 67 | if len(mdata.Number) == 0 { 68 | return nil, false, nil 69 | } 70 | mdata.Number = meta.GetNumberId(ctx) 71 | mdata.TitleLang = enum.MetaLangJa 72 | return mdata, true, nil 73 | } 74 | 75 | func init() { 76 | factory.Register(constant.SSFc2PPVDB, factory.PluginToCreator(&fc2ppvdb{})) 77 | } 78 | -------------------------------------------------------------------------------- /searcher/plugin/impl/freejavbt.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "yamdc/enum" 7 | "yamdc/model" 8 | "yamdc/searcher/decoder" 9 | "yamdc/searcher/parser" 10 | "yamdc/searcher/plugin/api" 11 | "yamdc/searcher/plugin/constant" 12 | "yamdc/searcher/plugin/factory" 13 | "yamdc/searcher/plugin/meta" 14 | ) 15 | 16 | var defaultFreeJavBtHostList = []string{ 17 | "https://freejavbt.com", 18 | } 19 | 20 | type freejavbt struct { 21 | api.DefaultPlugin 22 | } 23 | 24 | func (p *freejavbt) OnGetHosts(ctx context.Context) []string { 25 | return defaultFreeJavBtHostList 26 | } 27 | 28 | func (p *freejavbt) OnMakeHTTPRequest(ctx context.Context, number string) (*http.Request, error) { 29 | host := api.MustSelectDomain(defaultFreeJavBtHostList) 30 | uri := host + "/zh/" + number 31 | return http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) 32 | } 33 | 34 | func (p *freejavbt) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 35 | dec := decoder.XPathHtmlDecoder{ 36 | NumberExpr: "", 37 | TitleExpr: `//h1[@class="text-white"]/strong/text()`, 38 | PlotExpr: "", 39 | ActorListExpr: `//div[span[contains(text(), "女优")]]/div/a/text()`, 40 | ReleaseDateExpr: `//div[span[contains(text(), "日期")]]/span[2]`, 41 | DurationExpr: `//div[span[contains(text(), "时长")]]/span[2]`, 42 | StudioExpr: `//div[span[contains(text(), "制作")]]/a`, 43 | LabelExpr: "", 44 | DirectorExpr: `//div[span[contains(text(), "导演")]]/a`, 45 | SeriesExpr: "", 46 | GenreListExpr: `//div[span[contains(text(), "类别")]]/div/a/text()`, 47 | CoverExpr: `//img[@class="video-cover rounded lazyload"]/@data-src`, 48 | PosterExpr: "", 49 | SampleImageListExpr: `//div[@class="preview"]/a/img/@data-src`, 50 | } 51 | res, err := dec.DecodeHTML(data, 52 | decoder.WithDurationParser(parser.DefaultDurationParser(ctx)), 53 | decoder.WithReleaseDateParser(parser.DateOnlyReleaseDateParser(ctx)), 54 | ) 55 | if err != nil { 56 | return nil, false, err 57 | } 58 | res.Number = meta.GetNumberId(ctx) 59 | res.TitleLang = enum.MetaLangJa 60 | return res, true, nil 61 | } 62 | 63 | func init() { 64 | factory.Register(constant.SSFreeJavBt, factory.PluginToCreator(&freejavbt{})) 65 | } 66 | -------------------------------------------------------------------------------- /searcher/plugin/impl/jav321.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | "strconv" 9 | "strings" 10 | "yamdc/enum" 11 | "yamdc/model" 12 | "yamdc/searcher/decoder" 13 | "yamdc/searcher/parser" 14 | "yamdc/searcher/plugin/api" 15 | "yamdc/searcher/plugin/constant" 16 | "yamdc/searcher/plugin/factory" 17 | ) 18 | 19 | var defaultFreeJav321HostList = []string{ 20 | "https://www.jav321.com", 21 | } 22 | 23 | type jav321 struct { 24 | api.DefaultPlugin 25 | } 26 | 27 | func (p *jav321) OnGetHosts(ctx context.Context) []string { 28 | return defaultFreeJav321HostList 29 | } 30 | 31 | func (p *jav321) OnMakeHTTPRequest(ctx context.Context, number string) (*http.Request, error) { 32 | data := url.Values{} 33 | data.Set("sn", number) 34 | body := data.Encode() 35 | host := api.MustSelectDomain(defaultFreeJav321HostList) 36 | req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("%s/search", host), strings.NewReader(body)) 37 | if err != nil { 38 | return nil, err 39 | } 40 | req.Header.Add("Content-Type", "application/x-www-form-urlencoded") 41 | req.Header.Add("Content-Length", strconv.Itoa(len(body))) 42 | return req, nil 43 | } 44 | 45 | func (s *jav321) defaultStringProcessor(v string) string { 46 | v = strings.Trim(v, ": \t") 47 | return strings.TrimSpace(v) 48 | } 49 | 50 | func (p *jav321) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 51 | dec := &decoder.XPathHtmlDecoder{ 52 | NumberExpr: `//b[contains(text(),"品番")]/following-sibling::node()`, 53 | TitleExpr: `/html/body/div[2]/div[1]/div[1]/div[1]/h3/text()`, 54 | PlotExpr: `/html/body/div[2]/div[1]/div[1]/div[2]/div[3]/div/text()`, 55 | ActorListExpr: `//b[contains(text(),"出演者")]/following-sibling::a[starts-with(@href,"/star")]/text()`, 56 | ReleaseDateExpr: `//b[contains(text(),"配信開始日")]/following-sibling::node()`, 57 | DurationExpr: `//b[contains(text(),"収録時間")]/following-sibling::node()`, 58 | StudioExpr: `//b[contains(text(),"メーカー")]/following-sibling::a[starts-with(@href,"/company")]/text()`, 59 | LabelExpr: `//b[contains(text(),"メーカー")]/following-sibling::a[starts-with(@href,"/company")]/text()`, 60 | SeriesExpr: `//b[contains(text(),"シリーズ")]/following-sibling::node()`, 61 | GenreListExpr: `//b[contains(text(),"ジャンル")]/following-sibling::a[starts-with(@href,"/genre")]/text()`, 62 | CoverExpr: `/html/body/div[2]/div[2]/div[1]/p/a/img/@src`, 63 | PosterExpr: "", 64 | SampleImageListExpr: `//div[@class="col-md-3"]/div[@class="col-xs-12 col-md-12"]/p/a/img/@src`, 65 | } 66 | rs, err := dec.DecodeHTML(data, 67 | decoder.WithDefaultStringProcessor(p.defaultStringProcessor), 68 | decoder.WithReleaseDateParser(parser.DateOnlyReleaseDateParser(ctx)), 69 | decoder.WithDurationParser(parser.DefaultDurationParser(ctx)), 70 | ) 71 | if err != nil { 72 | return nil, false, err 73 | } 74 | rs.TitleLang = enum.MetaLangJa 75 | rs.PlotLang = enum.MetaLangJa 76 | return rs, true, nil 77 | } 78 | 79 | func init() { 80 | factory.Register(constant.SSJav321, factory.PluginToCreator(&jav321{})) 81 | } 82 | -------------------------------------------------------------------------------- /searcher/plugin/impl/javbus.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "yamdc/enum" 8 | "yamdc/model" 9 | "yamdc/searcher/decoder" 10 | "yamdc/searcher/parser" 11 | "yamdc/searcher/plugin/api" 12 | "yamdc/searcher/plugin/constant" 13 | "yamdc/searcher/plugin/factory" 14 | ) 15 | 16 | var defaultJavBusDomainList = []string{ 17 | "https://www.javbus.com", 18 | } 19 | 20 | type javbus struct { 21 | api.DefaultPlugin 22 | } 23 | 24 | func (p *javbus) OnGetHosts(ctx context.Context) []string { 25 | return defaultJavBusDomainList 26 | } 27 | 28 | func (p *javbus) OnMakeHTTPRequest(ctx context.Context, number string) (*http.Request, error) { 29 | url := fmt.Sprintf("%s/%s", api.MustSelectDomain(defaultJavBusDomainList), number) 30 | return http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 31 | } 32 | 33 | func (p *javbus) OnDecorateRequest(ctx context.Context, req *http.Request) error { 34 | req.AddCookie(&http.Cookie{ 35 | Name: "existmag", 36 | Value: "mag", 37 | }) 38 | req.AddCookie(&http.Cookie{ 39 | Name: "age", 40 | Value: "verified", 41 | }) 42 | req.AddCookie(&http.Cookie{ 43 | Name: "dv", 44 | Value: "1", 45 | }) 46 | req.Header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") 47 | req.Header.Add("Accept-Language", "en-US,en;q=0.5") 48 | req.Header.Add("Accept-Encoding", "gzip, deflate, br, zstd") 49 | return nil 50 | } 51 | 52 | func (p *javbus) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 53 | dec := decoder.XPathHtmlDecoder{ 54 | NumberExpr: `//div[@class="row movie"]/div[@class="col-md-3 info"]/p[span[contains(text(),'識別碼:')]]/span[2]/text()`, 55 | TitleExpr: `//div[@class="container"]/h3`, 56 | ActorListExpr: `//div[@class="star-name"]/a/text()`, 57 | ReleaseDateExpr: `//div[@class="row movie"]/div[@class="col-md-3 info"]/p[span[contains(text(),'發行日期:')]]/text()[1]`, 58 | DurationExpr: `//div[@class="row movie"]/div[@class="col-md-3 info"]/p[span[contains(text(),'長度:')]]/text()[1]`, 59 | StudioExpr: `//div[@class="row movie"]/div[@class="col-md-3 info"]/p[span[contains(text(),'製作商:')]]/a/text()`, 60 | LabelExpr: `//div[@class="row movie"]/div[@class="col-md-3 info"]/p[span[contains(text(),'發行商:')]]/a/text()`, 61 | SeriesExpr: `//div[@class="row movie"]/div[@class="col-md-3 info"]/p[span[contains(text(),'系列:')]]/a/text()`, 62 | GenreListExpr: `//div[@class="row movie"]/div[@class="col-md-3 info"]/p/span[@class="genre"]/label[input[@name="gr_sel"]]/a/text()`, 63 | CoverExpr: `//div[@class="row movie"]/div[@class="col-md-9 screencap"]/a[@class="bigImage"]/@href`, 64 | PosterExpr: "", 65 | PlotExpr: "", 66 | SampleImageListExpr: `//div[@id="sample-waterfall"]/a[@class="sample-box"]/@href`, 67 | } 68 | rs, err := dec.DecodeHTML(data, 69 | decoder.WithReleaseDateParser(parser.DateOnlyReleaseDateParser(ctx)), 70 | decoder.WithDurationParser(parser.DefaultDurationParser(ctx)), 71 | ) 72 | if err != nil { 73 | return nil, false, err 74 | } 75 | rs.TitleLang = enum.MetaLangJa 76 | return rs, true, nil 77 | } 78 | 79 | func init() { 80 | factory.Register(constant.SSJavBus, factory.PluginToCreator(&javbus{})) 81 | } 82 | -------------------------------------------------------------------------------- /searcher/plugin/impl/javbus_test.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "yamdc/number" 7 | "yamdc/searcher" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestJavbus(t *testing.T) { 13 | ss, err := searcher.NewDefaultSearcher("test", &javbus{}) 14 | assert.NoError(t, err) 15 | ctx := context.Background() 16 | num, err := number.Parse("STZY-015") 17 | assert.NoError(t, err) 18 | meta, ok, err := ss.Search(ctx, num) 19 | assert.NoError(t, err) 20 | assert.True(t, ok) 21 | assert.Equal(t, "STZY-015", meta.Number) 22 | assert.Equal(t, 3900, int(meta.Duration)) 23 | assert.Equal(t, 1742256000000, int(meta.ReleaseDate)) 24 | assert.Equal(t, 1, len(meta.Actors)) 25 | assert.True(t, len(meta.Title) > 0) 26 | assert.True(t, len(meta.Plot) > 0) 27 | assert.True(t, len(meta.Series) > 0) 28 | t.Logf("data:%+v", *meta) 29 | } 30 | -------------------------------------------------------------------------------- /searcher/plugin/impl/javdb.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "yamdc/enum" 8 | "yamdc/model" 9 | "yamdc/number" 10 | "yamdc/searcher/decoder" 11 | "yamdc/searcher/parser" 12 | "yamdc/searcher/plugin/api" 13 | "yamdc/searcher/plugin/constant" 14 | "yamdc/searcher/plugin/factory" 15 | "yamdc/searcher/plugin/meta" 16 | "yamdc/searcher/plugin/twostep" 17 | ) 18 | 19 | var defaultJavDBHostList = []string{ 20 | "https://javdb.com", 21 | } 22 | 23 | type javdb struct { 24 | api.DefaultPlugin 25 | } 26 | 27 | func (p *javdb) OnGetHosts(ctx context.Context) []string { 28 | return defaultJavDBHostList 29 | } 30 | 31 | func (p *javdb) OnMakeHTTPRequest(ctx context.Context, number string) (*http.Request, error) { 32 | link := fmt.Sprintf("%s/search?q=%s&f=all", api.MustSelectDomain(defaultJavDBHostList), number) 33 | return http.NewRequestWithContext(ctx, http.MethodGet, link, nil) 34 | } 35 | 36 | func (p *javdb) OnHandleHTTPRequest(ctx context.Context, invoker api.HTTPInvoker, req *http.Request) (*http.Response, error) { 37 | return twostep.HandleXPathTwoStepSearch(ctx, invoker, req, &twostep.XPathTwoStepContext{ 38 | Ps: []*twostep.XPathPair{ 39 | { 40 | Name: "read-link", 41 | XPath: `//div[@class="movie-list h cols-4 vcols-8"]/div[@class="item"]/a/@href`, 42 | }, 43 | { 44 | Name: "read-number", 45 | XPath: `//div[@class="movie-list h cols-4 vcols-8"]/div[@class="item"]/a/div[@class="video-title"]/strong`, 46 | }, 47 | }, 48 | LinkSelector: func(ps []*twostep.XPathPair) (string, bool, error) { 49 | linklist := ps[0].Result 50 | numberlist := ps[1].Result 51 | num := number.GetCleanID(meta.GetNumberId(ctx)) 52 | for idx, numberItem := range numberlist { 53 | link := linklist[idx] 54 | if number.GetCleanID(numberItem) == num { 55 | return link, true, nil 56 | } 57 | } 58 | return "", false, nil 59 | }, 60 | ValidStatusCode: []int{http.StatusOK}, 61 | CheckResultCountMatch: true, 62 | LinkPrefix: fmt.Sprintf("%s://%s", req.URL.Scheme, req.URL.Host), 63 | }) 64 | } 65 | 66 | func (p *javdb) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 67 | dec := decoder.XPathHtmlDecoder{ 68 | NumberExpr: `//a[@class="button is-white copy-to-clipboard"]/@data-clipboard-text`, 69 | TitleExpr: `//h2[@class="title is-4"]/strong[@class="current-title"]`, 70 | PlotExpr: "", 71 | ActorListExpr: `//div[strong[contains(text(), "演員")]]/span[@class="value"]/a`, 72 | ReleaseDateExpr: `//div[strong[contains(text(), "日期")]]/span[@class="value"]`, 73 | DurationExpr: `//div[strong[contains(text(), "時長")]]/span[@class="value"]`, 74 | StudioExpr: `//div[strong[contains(text(), "片商")]]/span[@class="value"]`, 75 | LabelExpr: "", 76 | DirectorExpr: "", 77 | SeriesExpr: `//div[strong[contains(text(), "系列")]]/span[@class="value"]`, 78 | GenreListExpr: `//div[strong[contains(text(), "類別")]]/span[@class="value"]/a`, 79 | CoverExpr: `//div[@class="column column-video-cover"]/a/img/@src`, 80 | PosterExpr: "", 81 | SampleImageListExpr: `//div[@class="tile-images preview-images"]/a[@class="tile-item"]/@href`, 82 | } 83 | meta, err := dec.DecodeHTML(data, 84 | decoder.WithReleaseDateParser(parser.DateOnlyReleaseDateParser(ctx)), 85 | decoder.WithDurationParser(parser.DefaultDurationParser(ctx)), 86 | ) 87 | if err != nil { 88 | return nil, false, err 89 | } 90 | if len(meta.Number) == 0 { 91 | return nil, false, nil 92 | } 93 | meta.TitleLang = enum.MetaLangJa 94 | return meta, true, nil 95 | } 96 | 97 | func init() { 98 | factory.Register(constant.SSJavDB, factory.PluginToCreator(&javdb{})) 99 | } 100 | -------------------------------------------------------------------------------- /searcher/plugin/impl/javhoo.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "yamdc/enum" 8 | "yamdc/model" 9 | "yamdc/searcher/decoder" 10 | "yamdc/searcher/parser" 11 | "yamdc/searcher/plugin/api" 12 | "yamdc/searcher/plugin/constant" 13 | "yamdc/searcher/plugin/factory" 14 | ) 15 | 16 | var defaultJavHooHostList = []string{ 17 | "https://www.javhoo.com", 18 | } 19 | 20 | type javhoo struct { 21 | api.DefaultPlugin 22 | } 23 | 24 | func (p *javhoo) OnGetHosts(ctx context.Context) []string { 25 | return defaultJavHooHostList 26 | } 27 | 28 | func (p *javhoo) OnMakeHTTPRequest(ctx context.Context, number string) (*http.Request, error) { 29 | uri := fmt.Sprintf("%s/av/%s", api.MustSelectDomain(defaultJavHooHostList), number) 30 | return http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) 31 | } 32 | 33 | func (p *javhoo) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 34 | dec := decoder.XPathHtmlDecoder{ 35 | NumberExpr: `//div[@class="project_info"]/p/span[@class="categories"]/text()`, 36 | TitleExpr: `//header[@class="article-header"]/h1[@class="article-title"]/text()`, 37 | PlotExpr: "", 38 | ActorListExpr: `//p/span[@class="genre"]/a[contains(@href, "star")]/text()`, 39 | ReleaseDateExpr: `//div[@class="project_info"]/p[span[contains(text(), "發行日期")]]/text()[2]`, 40 | DurationExpr: `//div[@class="project_info"]/p[span[contains(text(), "長度")]]/text()[2]`, 41 | StudioExpr: `//div[@class="project_info"]/p[span[contains(text(), "製作商")]]/a/text()`, 42 | LabelExpr: `//div[@class="project_info"]/p[span[contains(text(), "發行商")]]/a/text()`, 43 | DirectorExpr: `//div[@class="project_info"]/p[span[contains(text(), "導演")]]/a/text()`, 44 | SeriesExpr: `//div[@class="project_info"]/p[span[contains(text(), "系列")]]/a/text()`, 45 | GenreListExpr: `//p/span[@class="genre"]/a[contains(@href, "genre")]/text()`, 46 | CoverExpr: `//p/a[@class="dt-single-image"]/@href`, 47 | PosterExpr: "", 48 | SampleImageListExpr: `//div[@id="sample-box"]/div/a/@href`, 49 | } 50 | meta, err := dec.DecodeHTML(data, 51 | decoder.WithReleaseDateParser(parser.DateOnlyReleaseDateParser(ctx)), 52 | decoder.WithDurationParser(parser.DefaultDurationParser(ctx)), 53 | ) 54 | if err != nil { 55 | return nil, false, err 56 | } 57 | if len(meta.Number) == 0 { 58 | return nil, false, nil 59 | } 60 | meta.TitleLang = enum.MetaLangJa 61 | return meta, true, nil 62 | } 63 | 64 | func init() { 65 | factory.Register(constant.SSJavhoo, factory.PluginToCreator(&javhoo{})) 66 | } 67 | -------------------------------------------------------------------------------- /searcher/plugin/impl/javlibrary.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "yamdc/enum" 8 | "yamdc/model" 9 | "yamdc/searcher/decoder" 10 | "yamdc/searcher/parser" 11 | "yamdc/searcher/plugin/api" 12 | "yamdc/searcher/plugin/constant" 13 | "yamdc/searcher/plugin/factory" 14 | ) 15 | 16 | var defaultJavLibraryHostList = []string{ 17 | "https://www.javlibrary.com", 18 | } 19 | 20 | type javlibrary struct { 21 | api.DefaultPlugin 22 | } 23 | 24 | func (j *javlibrary) OnGetHosts(ctx context.Context) []string { 25 | return defaultJavLibraryHostList 26 | } 27 | 28 | func (j *javlibrary) OnMakeHTTPRequest(ctx context.Context, number string) (*http.Request, error) { 29 | uri := fmt.Sprintf("%s/cn/vl_searchbyid.php?keyword=%s", api.MustSelectDomain(defaultJavLibraryHostList), number) 30 | return http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) 31 | } 32 | 33 | func (j *javlibrary) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 34 | dec := decoder.XPathHtmlDecoder{ 35 | NumberExpr: `//tbody/tr[td[contains(text(), "识别码:")]]/td[@class="text"]/text()`, 36 | TitleExpr: `//div[@id="video_title"]/h3[@class="post-title text"]/a[@rel="bookmark"]/text()`, 37 | PlotExpr: "", 38 | ActorListExpr: `//tbody/tr[td[contains(text(), "演员:")]]/td[@class="text"]//span[@class="star"]/a/text()`, 39 | ReleaseDateExpr: `//tbody/tr[td[contains(text(), "发行日期:")]]/td[@class="text"]/text()`, 40 | DurationExpr: `//tbody/tr[td[contains(text(), "长度:")]]/td/span[@class="text"]/text()`, 41 | StudioExpr: `//tbody/tr[td[contains(text(), "制作商:")]]/td[@class="text"]//span[@class="maker"]/a/text()`, 42 | LabelExpr: `//tbody/tr[td[contains(text(), "发行商:")]]/td[@class="text"]//span[@class="label"]/a/text()`, 43 | DirectorExpr: `//tbody/tr[td[contains(text(), "导演:")]]/td[@class="text"]/text()`, 44 | SeriesExpr: "", 45 | GenreListExpr: `//tbody/tr[td[contains(text(), "类别:")]]/td[@class="text"]/span[@class="genre"]/a/text()`, 46 | CoverExpr: `//img[@id="video_jacket_img"]/@src`, 47 | PosterExpr: "", 48 | SampleImageListExpr: `//div[@class="previewthumbs"]/a/@href`, 49 | } 50 | mm, err := dec.DecodeHTML(data, 51 | decoder.WithReleaseDateParser(parser.DateOnlyReleaseDateParser(ctx)), 52 | decoder.WithDurationParser(parser.MinuteOnlyDurationParser(ctx)), 53 | ) 54 | if err != nil { 55 | return nil, false, err 56 | } 57 | if mm.Number == "" { 58 | return nil, false, nil 59 | } 60 | if mm.Director == "----" { 61 | mm.Director = "" 62 | } 63 | mm.TitleLang = enum.MetaLangJa 64 | return mm, true, nil 65 | } 66 | 67 | func init() { 68 | factory.Register(constant.SSJavLibrary, factory.PluginToCreator(&javlibrary{})) 69 | } 70 | -------------------------------------------------------------------------------- /searcher/plugin/impl/jvrporn.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "strings" 8 | "yamdc/enum" 9 | "yamdc/model" 10 | "yamdc/searcher/decoder" 11 | "yamdc/searcher/parser" 12 | "yamdc/searcher/plugin/api" 13 | "yamdc/searcher/plugin/constant" 14 | "yamdc/searcher/plugin/factory" 15 | "yamdc/searcher/plugin/meta" 16 | ) 17 | 18 | var ( 19 | defaultJvrpornHostList = []string{ 20 | "https://jvrporn.com", 21 | } 22 | ) 23 | 24 | type jvrporn struct { 25 | api.DefaultPlugin 26 | } 27 | 28 | func (j *jvrporn) OnGetHosts(ctx context.Context) []string { 29 | return defaultJvrpornHostList 30 | } 31 | 32 | func (j *jvrporn) OnPrecheckRequest(ctx context.Context, number string) (bool, error) { 33 | if !strings.HasPrefix(number, "JVR-") { 34 | return false, nil 35 | } 36 | return true, nil 37 | } 38 | 39 | func (j *jvrporn) OnMakeHTTPRequest(ctx context.Context, number string) (*http.Request, error) { 40 | slices := strings.Split(number, "-") 41 | if len(slices) != 2 { 42 | return nil, fmt.Errorf("invalid number for jvrporn") 43 | } 44 | id := slices[1] 45 | uri := fmt.Sprintf("%s/video/%s/", api.MustSelectDomain(defaultJvrpornHostList), id) 46 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) 47 | if err != nil { 48 | return nil, err 49 | } 50 | return req, nil 51 | } 52 | 53 | func (j *jvrporn) OnDecorateRequest(ctx context.Context, req *http.Request) error { 54 | req.AddCookie(&http.Cookie{ 55 | Name: "adult", 56 | Value: "true", 57 | }) 58 | return nil 59 | } 60 | 61 | func (j *jvrporn) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 62 | dec := decoder.XPathHtmlDecoder{ 63 | NumberExpr: "", 64 | TitleExpr: `//h1`, 65 | PlotExpr: `//pre`, 66 | ActorListExpr: `//div[@class="basic-info"]//td/a[@class="actress"]/span/text()`, 67 | ReleaseDateExpr: "", 68 | DurationExpr: `//tr[td[span[contains(text(), "Duration")]]]/td[span[@class="bold"]]/span/text()`, 69 | StudioExpr: "", 70 | LabelExpr: "", 71 | DirectorExpr: "", 72 | SeriesExpr: "", 73 | GenreListExpr: `//tr[td[span[contains(text(), "Tags")]]]/td/a/span[@class="bold"]/text()`, 74 | CoverExpr: `//div[@class="video-play-container"]/deo-video/@cover-image`, 75 | PosterExpr: "", 76 | SampleImageListExpr: `//div[@class="gallery-wrap"]/div[@id="snapshot-gallery"]/a/@href`, 77 | } 78 | rs, err := dec.DecodeHTML(data, 79 | decoder.WithReleaseDateParser(parser.DateOnlyReleaseDateParser(ctx)), 80 | decoder.WithDurationParser(parser.DefaultHHMMSSDurationParser(ctx)), 81 | ) 82 | if err != nil { 83 | return nil, false, err 84 | } 85 | if len(rs.Title) == 0 { 86 | return nil, false, nil 87 | } 88 | rs.Number = meta.GetNumberId(ctx) 89 | rs.TitleLang = enum.MetaLangEn 90 | rs.PlotLang = enum.MetaLangEn 91 | rs.GenresLang = enum.MetaLangEn 92 | rs.ActorsLang = enum.MetaLangEn 93 | rs.SwithConfig.DisableReleaseDateCheck = true 94 | return rs, true, nil 95 | } 96 | 97 | func init() { 98 | factory.Register(constant.SSJvrPorn, factory.PluginToCreator(&jvrporn{})) 99 | } 100 | -------------------------------------------------------------------------------- /searcher/plugin/impl/missav.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "strings" 8 | "yamdc/model" 9 | "yamdc/searcher/decoder" 10 | "yamdc/searcher/parser" 11 | "yamdc/searcher/plugin/api" 12 | "yamdc/searcher/plugin/constant" 13 | "yamdc/searcher/plugin/factory" 14 | "yamdc/searcher/plugin/meta" 15 | "yamdc/searcher/plugin/twostep" 16 | ) 17 | 18 | var ( 19 | defaultMissavDomains = []string{ 20 | "https://missav.ws", 21 | } 22 | ) 23 | 24 | type missav struct { 25 | api.DefaultPlugin 26 | } 27 | 28 | func (p *missav) OnGetHosts(ctx context.Context) []string { 29 | return defaultMissavDomains 30 | } 31 | 32 | func (p *missav) OnMakeHTTPRequest(ctx context.Context, number string) (*http.Request, error) { 33 | link := fmt.Sprintf("%s/cn/search/%s", api.MustSelectDomain(defaultMissavDomains), number) 34 | return http.NewRequestWithContext(ctx, http.MethodGet, link, nil) 35 | } 36 | 37 | func (p *missav) OnHandleHTTPRequest(ctx context.Context, invoker api.HTTPInvoker, req *http.Request) (*http.Response, error) { 38 | xctx := &twostep.XPathTwoStepContext{ 39 | Ps: []*twostep.XPathPair{ 40 | { 41 | Name: "read-link", 42 | XPath: `//div[@class="my-2 text-sm text-nord4 truncate"]/a[@class="text-secondary group-hover:text-primary"]/@href`, 43 | }, 44 | { 45 | Name: "read-title", 46 | XPath: `//div[@class="my-2 text-sm text-nord4 truncate"]/a[@class="text-secondary group-hover:text-primary"]/text()`, 47 | }, 48 | }, 49 | LinkSelector: func(ps []*twostep.XPathPair) (string, bool, error) { 50 | linkList := ps[0].Result 51 | titleList := ps[1].Result 52 | for i, link := range linkList { 53 | title := titleList[i] 54 | if strings.Contains(title, meta.GetNumberId(ctx)) { 55 | return link, true, nil 56 | } 57 | } 58 | return "", false, nil 59 | }, 60 | ValidStatusCode: []int{http.StatusOK}, 61 | CheckResultCountMatch: true, 62 | LinkPrefix: "", 63 | } 64 | return twostep.HandleXPathTwoStepSearch(ctx, invoker, req, xctx) 65 | } 66 | 67 | func (p *missav) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 68 | dec := decoder.XPathHtmlDecoder{ 69 | NumberExpr: `//div[span[contains(text(), "番号")]]/span[@class="font-medium"]/text()`, 70 | TitleExpr: `//div[@class="mt-4"]/h1[@class="text-base lg:text-lg text-nord6"]/text()`, 71 | PlotExpr: "", 72 | ActorListExpr: `//div[span[contains(text(), "女优")]]/a/text()`, 73 | ReleaseDateExpr: `//div[span[contains(text(), "发行日期")]]/time/text()`, 74 | DurationExpr: "", 75 | StudioExpr: `//div[span[contains(text(), "发行商")]]/a/text()`, 76 | LabelExpr: "", 77 | DirectorExpr: `//div[span[contains(text(), "导演")]]/a/text()`, 78 | SeriesExpr: "", 79 | GenreListExpr: `//div[span[contains(text(), "类型")]]/a/text()`, 80 | CoverExpr: `//link[@rel="preload" and @as="image"]/@href`, 81 | PosterExpr: "", 82 | SampleImageListExpr: "", 83 | } 84 | mdata, err := dec.DecodeHTML(data, decoder.WithReleaseDateParser(parser.DateOnlyReleaseDateParser(ctx))) 85 | if err != nil { 86 | return nil, false, err 87 | } 88 | if len(mdata.Number) == 0 { 89 | return nil, false, err 90 | } 91 | return mdata, true, nil 92 | } 93 | 94 | func init() { 95 | factory.Register(constant.SSMissav, factory.PluginToCreator(&missav{})) 96 | } 97 | -------------------------------------------------------------------------------- /searcher/plugin/impl/njav.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "strings" 8 | "yamdc/model" 9 | "yamdc/number" 10 | "yamdc/searcher/decoder" 11 | "yamdc/searcher/parser" 12 | "yamdc/searcher/plugin/api" 13 | "yamdc/searcher/plugin/constant" 14 | "yamdc/searcher/plugin/factory" 15 | "yamdc/searcher/plugin/meta" 16 | "yamdc/searcher/plugin/twostep" 17 | ) 18 | 19 | var defaultNJavHostList = []string{ 20 | "https://njavtv.com", 21 | } 22 | 23 | type njav struct { 24 | api.DefaultPlugin 25 | } 26 | 27 | func (p *njav) OnGetHosts(ctx context.Context) []string { 28 | return defaultNJavHostList 29 | } 30 | 31 | func (p *njav) OnMakeHTTPRequest(ctx context.Context, number string) (*http.Request, error) { 32 | nid := number 33 | nid = strings.ReplaceAll(nid, "_", "-") //将下划线替换为中划线 34 | uri := fmt.Sprintf("%s/cn/search/%s", api.MustSelectDomain(defaultNJavHostList), nid) 35 | return http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) 36 | } 37 | 38 | func (p *njav) OnHandleHTTPRequest(ctx context.Context, invoker api.HTTPInvoker, req *http.Request) (*http.Response, error) { 39 | cleanNumberId := strings.ToUpper(number.GetCleanID(meta.GetNumberId(ctx))) 40 | return twostep.HandleXPathTwoStepSearch(ctx, invoker, req, &twostep.XPathTwoStepContext{ 41 | Ps: []*twostep.XPathPair{ 42 | { 43 | Name: "links", 44 | XPath: `//div[@class="my-2 text-sm text-nord4 truncate"]/a[@class="text-secondary group-hover:text-primary"]/@href`, 45 | }, 46 | { 47 | Name: "title", 48 | XPath: `//div[@class="my-2 text-sm text-nord4 truncate"]/a[@class="text-secondary group-hover:text-primary"]/text()`, 49 | }, 50 | }, 51 | LinkSelector: func(ps []*twostep.XPathPair) (string, bool, error) { 52 | links := ps[0].Result 53 | titles := ps[1].Result 54 | for i, link := range links { 55 | title := titles[i] 56 | title = strings.ToUpper(number.GetCleanID(title)) 57 | if strings.Contains(title, cleanNumberId) { 58 | return link, true, nil 59 | } 60 | } 61 | return "", false, nil 62 | }, 63 | ValidStatusCode: []int{http.StatusOK}, 64 | CheckResultCountMatch: true, 65 | }) 66 | 67 | } 68 | 69 | func (p *njav) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 70 | dec := decoder.XPathHtmlDecoder{ 71 | NumberExpr: `//div[@class="text-secondary" and contains(span[text()], "番号:")]/span[@class="font-medium"]/text()`, 72 | TitleExpr: `//div[@class="text-secondary" and contains(span[text()], "标题:")]/span[@class="font-medium"]/text()`, 73 | PlotExpr: "", 74 | ActorListExpr: `//meta[@property="og:video:actor"]/@content`, 75 | ReleaseDateExpr: `//div[@class="text-secondary" and contains(span[text()], "发行日期:")]/time[@class="font-medium"]/text()`, 76 | DurationExpr: `//meta[@property="og:video:duration"]/@content`, 77 | StudioExpr: `//div[@class="text-secondary" and contains(span[text()], "发行商:")]/a[@class="text-nord13 font-medium"]/text()`, 78 | LabelExpr: `//div[@class="text-secondary" and contains(span[text()], "标籤:")]/a[@class="text-nord13 font-medium"]/text()`, 79 | DirectorExpr: `//div[@class="text-secondary" and contains(span[text()], "导演:")]/a[@class="text-nord13 font-medium"]/text()`, 80 | SeriesExpr: `//div[@class="text-secondary" and contains(span[text()], "系列:")]/a[@class="text-nord13 font-medium"]/text()`, 81 | GenreListExpr: `//div[@class="text-secondary" and contains(span[text()], "类型:")]/a[@class="text-nord13 font-medium"]/text()`, 82 | CoverExpr: `//link[@rel="preload" and @as="image"]/@href`, 83 | PosterExpr: "", 84 | SampleImageListExpr: "", 85 | } 86 | meta, err := dec.DecodeHTML(data, decoder.WithReleaseDateParser(parser.DateOnlyReleaseDateParser(ctx))) 87 | if err != nil { 88 | return nil, false, err 89 | } 90 | if len(meta.Number) == 0 { 91 | return nil, false, nil 92 | } 93 | return meta, true, nil 94 | } 95 | 96 | func init() { 97 | factory.Register(constant.SSNJav, factory.PluginToCreator(&njav{})) 98 | } 99 | -------------------------------------------------------------------------------- /searcher/plugin/impl/tktube.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "strings" 8 | "yamdc/model" 9 | "yamdc/searcher/decoder" 10 | "yamdc/searcher/parser" 11 | "yamdc/searcher/plugin/api" 12 | "yamdc/searcher/plugin/constant" 13 | "yamdc/searcher/plugin/factory" 14 | "yamdc/searcher/plugin/meta" 15 | "yamdc/searcher/plugin/twostep" 16 | ) 17 | 18 | var defaultTkTubeHostList = []string{ 19 | "https://tktube.com", 20 | } 21 | 22 | type tktube struct { 23 | api.DefaultPlugin 24 | } 25 | 26 | func (p *tktube) OnGetHosts(ctx context.Context) []string { 27 | return defaultTkTubeHostList 28 | } 29 | 30 | func (p *tktube) OnMakeHTTPRequest(ctx context.Context, n string) (*http.Request, error) { 31 | nid := strings.ReplaceAll(n, "-", "--") 32 | uri := fmt.Sprintf("%s/zh/search/%s/", api.MustSelectDomain(defaultTkTubeHostList), nid) 33 | return http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) 34 | } 35 | 36 | func (p *tktube) OnHandleHTTPRequest(ctx context.Context, invoker api.HTTPInvoker, req *http.Request) (*http.Response, error) { 37 | numberId := strings.ToUpper(meta.GetNumberId(ctx)) 38 | return twostep.HandleXPathTwoStepSearch(ctx, invoker, req, &twostep.XPathTwoStepContext{ 39 | Ps: []*twostep.XPathPair{ 40 | { 41 | Name: "links", 42 | XPath: `//div[@id="list_videos_videos_list_search_result_items"]/div/a/@href`, 43 | }, 44 | { 45 | Name: "names", 46 | XPath: `//div[@id="list_videos_videos_list_search_result_items"]/div/a/strong[@class="title"]/text()`, 47 | }, 48 | }, 49 | LinkSelector: func(ps []*twostep.XPathPair) (string, bool, error) { 50 | links := ps[0].Result 51 | names := ps[1].Result 52 | for i := 0; i < len(links); i++ { 53 | if strings.Contains(strings.ToUpper(names[i]), numberId) { 54 | return links[i], true, nil 55 | } 56 | } 57 | return "", false, nil 58 | }, 59 | ValidStatusCode: []int{http.StatusOK}, 60 | CheckResultCountMatch: true, 61 | LinkPrefix: "", 62 | }) 63 | } 64 | 65 | func (p *tktube) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.MovieMeta, bool, error) { 66 | dec := decoder.XPathHtmlDecoder{ 67 | TitleExpr: `//div[@class="headline"]/h1/text()`, 68 | PlotExpr: "", 69 | ActorListExpr: `//div[contains(text(), "女優:")]/a[contains(@href, "models")]/text()`, 70 | ReleaseDateExpr: `//div[@class="item"]/span[contains(text(), "加入日期:")]/em/text()`, 71 | DurationExpr: `//div[@class="item"]/span[contains(text(), "時長:")]/em/text()`, 72 | StudioExpr: "", 73 | LabelExpr: "", 74 | DirectorExpr: "", 75 | SeriesExpr: "", 76 | GenreListExpr: `//div[contains(text(), "標籤:")]/a[contains(@href, "tags")]/text()`, 77 | CoverExpr: `//meta[@property="og:image"]/@content`, 78 | PosterExpr: "", 79 | SampleImageListExpr: "", 80 | } 81 | res, err := dec.DecodeHTML(data, 82 | decoder.WithDurationParser(parser.DefaultHHMMSSDurationParser(ctx)), 83 | decoder.WithReleaseDateParser(parser.DateOnlyReleaseDateParser(ctx)), 84 | ) 85 | if err != nil { 86 | return nil, false, err 87 | } 88 | res.Number = meta.GetNumberId(ctx) 89 | return res, true, nil 90 | } 91 | 92 | func init() { 93 | factory.Register(constant.SSTKTube, factory.PluginToCreator(&tktube{})) 94 | } 95 | -------------------------------------------------------------------------------- /searcher/plugin/meta/meta.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | import "context" 4 | 5 | type numberIdKeyType struct{} 6 | 7 | var ( 8 | defaultNumberIdKey = numberIdKeyType{} 9 | ) 10 | 11 | func SetNumberId(ctx context.Context, nid string) context.Context { 12 | ctx = context.WithValue(ctx, defaultNumberIdKey, nid) 13 | return ctx 14 | } 15 | 16 | func GetNumberId(ctx context.Context) string { 17 | nid := ctx.Value(defaultNumberIdKey) 18 | if nid == nil { 19 | return "" 20 | } 21 | return nid.(string) 22 | } 23 | -------------------------------------------------------------------------------- /searcher/plugin/register/register.go: -------------------------------------------------------------------------------- 1 | package register 2 | 3 | import ( 4 | _ "yamdc/searcher/plugin/impl" 5 | _ "yamdc/searcher/plugin/impl/airav" 6 | ) 7 | -------------------------------------------------------------------------------- /searcher/plugin/twostep/multilink.go: -------------------------------------------------------------------------------- 1 | package twostep 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "yamdc/client" 10 | "yamdc/searcher/plugin/api" 11 | ) 12 | 13 | type MultiLinkContext struct { 14 | ReqBuilder MultiLinkBuildRequestFunc //用于重建请求的函数 15 | Numbers []string //用户传入的多个番号, 基于这些番号, 逐个调用ReqBuilder构建链接并请求 16 | ValidStatusCode []int //哪些http状态码是有效的 17 | ResultTester OnMultiLinkResultTest //回调用户函数，确认哪些结果是符合预期的 18 | } 19 | 20 | type MultiLinkBuildRequestFunc func(nid string) (*http.Request, error) 21 | type OnMultiLinkResultTest func(raw []byte) (bool, error) 22 | 23 | func HandleMultiLinkSearch(ctx context.Context, invoker api.HTTPInvoker, xctx *MultiLinkContext) (*http.Response, error) { 24 | for _, number := range xctx.Numbers { 25 | req, err := xctx.ReqBuilder(number) 26 | if err != nil { 27 | return nil, fmt.Errorf("build request failed, err:%w", err) 28 | } 29 | rsp, err := invoker(ctx, req) 30 | if err != nil { 31 | return rsp, fmt.Errorf("step search failed, err:%w", err) 32 | } 33 | if !isCodeInValidStatusCodeList(xctx.ValidStatusCode, rsp.StatusCode) { 34 | _ = rsp.Body.Close() 35 | continue 36 | } 37 | raw, err := client.ReadHTTPData(rsp) 38 | if err != nil { 39 | return rsp, fmt.Errorf("step read data as html node failed, err:%w", err) 40 | } 41 | ok, err := xctx.ResultTester(raw) 42 | if err != nil { 43 | return rsp, fmt.Errorf("test result failed, err:%w", err) 44 | } 45 | if !ok { 46 | continue 47 | } 48 | //将body重新设置回去 49 | rsp.Body = io.NopCloser(bytes.NewReader(raw)) 50 | return rsp, nil 51 | } 52 | return nil, fmt.Errorf("no valid result found") 53 | } 54 | -------------------------------------------------------------------------------- /searcher/plugin/twostep/twostep.go: -------------------------------------------------------------------------------- 1 | package twostep 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "yamdc/searcher/decoder" 8 | "yamdc/searcher/plugin/api" 9 | "yamdc/searcher/utils" 10 | ) 11 | 12 | type XPathPair struct { 13 | Name string 14 | XPath string 15 | Result []string 16 | } 17 | 18 | type XPathTwoStepContext struct { 19 | Ps []*XPathPair //用户传入多组XPath, 用于在页面获取完数据后进行数据提取 20 | LinkSelector OnTwoStepLinkSelect //回调用户函数，确认哪些链接是符合预期的 21 | ValidStatusCode []int //http的哪些状态码是有效的 22 | CheckResultCountMatch bool //检查多组xpath的result个数是否一致 23 | LinkPrefix string //用于重建链接的前缀 24 | } 25 | 26 | type OnTwoStepLinkSelect func(ps []*XPathPair) (string, bool, error) 27 | 28 | func isCodeInValidStatusCodeList(lst []int, code int) bool { 29 | for _, c := range lst { 30 | if c == code { 31 | return true 32 | } 33 | } 34 | return false 35 | } 36 | 37 | func HandleXPathTwoStepSearch(ctx context.Context, invoker api.HTTPInvoker, req *http.Request, xctx *XPathTwoStepContext) (*http.Response, error) { 38 | rsp, err := invoker(ctx, req) 39 | if err != nil { 40 | return nil, fmt.Errorf("step search failed, err:%w", err) 41 | } 42 | defer rsp.Body.Close() 43 | if !isCodeInValidStatusCodeList(xctx.ValidStatusCode, rsp.StatusCode) { 44 | return nil, fmt.Errorf("status code:%d not in valid list", rsp.StatusCode) 45 | } 46 | node, err := utils.ReadDataAsHTMLTree(rsp) 47 | if err != nil { 48 | return nil, fmt.Errorf("step read data as html node failed, err:%w", err) 49 | } 50 | for _, p := range xctx.Ps { 51 | p.Result = decoder.DecodeList(node, p.XPath) 52 | } 53 | if xctx.CheckResultCountMatch { 54 | for i := 1; i < len(xctx.Ps); i++ { 55 | if len(xctx.Ps[i].Result) != len(xctx.Ps[0].Result) { 56 | return nil, fmt.Errorf("result count not match, idx:%d, count:%d not match to idx:0, count:%d", i, len(xctx.Ps[i].Result), len(xctx.Ps[0].Result)) 57 | } 58 | } 59 | if len(xctx.Ps[0].Result) == 0 { 60 | return nil, fmt.Errorf("no result found") 61 | } 62 | } 63 | link, ok, err := xctx.LinkSelector(xctx.Ps) 64 | if err != nil { 65 | return nil, fmt.Errorf("select link from result failed, err:%w", err) 66 | } 67 | if !ok { 68 | return nil, fmt.Errorf("no link select result found") 69 | } 70 | link = xctx.LinkPrefix + link 71 | req, err = http.NewRequestWithContext(ctx, http.MethodGet, link, nil) 72 | if err != nil { 73 | return nil, fmt.Errorf("step re-create result page link failed, err:%w", err) 74 | } 75 | return invoker(ctx, req) 76 | } 77 | -------------------------------------------------------------------------------- /searcher/searcher.go: -------------------------------------------------------------------------------- 1 | package searcher 2 | 3 | import ( 4 | "context" 5 | "yamdc/model" 6 | "yamdc/number" 7 | ) 8 | 9 | type ISearcher interface { 10 | Name() string 11 | Search(ctx context.Context, number *number.Number) (*model.MovieMeta, bool, error) 12 | Check(ctx context.Context) error 13 | } 14 | -------------------------------------------------------------------------------- /searcher/utils/http_utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bytes" 5 | "net/http" 6 | "yamdc/client" 7 | 8 | "golang.org/x/net/html" 9 | ) 10 | 11 | func ReadDataAsHTMLTree(rsp *http.Response) (*html.Node, error) { 12 | data, err := client.ReadHTTPData(rsp) 13 | if err != nil { 14 | return nil, err 15 | } 16 | return html.Parse(bytes.NewReader(data)) 17 | } 18 | -------------------------------------------------------------------------------- /store/mem_storage.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | type memStorage struct { 10 | m map[string][]byte 11 | } 12 | 13 | func NewMemStorage() IStorage { 14 | return &memStorage{ 15 | m: make(map[string][]byte), 16 | } 17 | } 18 | 19 | func (m *memStorage) GetData(ctx context.Context, key string) ([]byte, error) { 20 | if v, ok := m.m[key]; ok { 21 | return v, nil 22 | } 23 | return nil, fmt.Errorf("not found") 24 | } 25 | 26 | func (m *memStorage) PutData(ctx context.Context, key string, value []byte, expire time.Duration) error { 27 | m.m[key] = value 28 | return nil 29 | } 30 | 31 | func (m *memStorage) IsDataExist(ctx context.Context, key string) (bool, error) { 32 | _, err := m.GetData(ctx, key) 33 | if err != nil { 34 | return false, nil 35 | } 36 | return true, nil 37 | } 38 | -------------------------------------------------------------------------------- /store/sqlite_storage.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "errors" 7 | "os" 8 | "path/filepath" 9 | "time" 10 | 11 | _ "github.com/glebarez/go-sqlite" 12 | ) 13 | 14 | const ( 15 | defaultExpireTime = 90 * 24 * time.Hour //默认存储3个月, 超过就删了吧, 实在没想到有啥东西需要永久存储的? 16 | ) 17 | 18 | type sqliteStore struct { 19 | db *sql.DB 20 | } 21 | 22 | func (s *sqliteStore) init(ctx context.Context) error { 23 | execList := []struct { 24 | sql string 25 | args []interface{} 26 | }{ 27 | {`CREATE TABLE IF NOT EXISTS cache_tab ( 28 | key TEXT PRIMARY KEY, 29 | value BLOB, 30 | expire_at INTEGER 31 | );`, nil}, 32 | {"CREATE INDEX if not exists idx_expireat on cache_tab(expire_at);", nil}, 33 | {"DELETE from cache_tab where expire_at <= ?", []interface{}{time.Now().Unix()}}, 34 | } 35 | for _, item := range execList { 36 | _, err := s.db.ExecContext(ctx, item.sql, item.args...) 37 | if err != nil { 38 | return err 39 | } 40 | } 41 | return nil 42 | } 43 | 44 | func (s *sqliteStore) GetData(ctx context.Context, key string) ([]byte, error) { 45 | var val []byte 46 | now := time.Now().Unix() 47 | err := s.db.QueryRowContext(ctx, "SELECT value FROM cache_tab WHERE key = ? and expire_at > ?", key, now).Scan(&val) 48 | if err != nil { 49 | return nil, err 50 | } 51 | return val, nil 52 | } 53 | 54 | func (s *sqliteStore) PutData(ctx context.Context, key string, value []byte, expire time.Duration) error { 55 | var expireAt int64 = 0 56 | if expire == 0 { 57 | expire = defaultExpireTime //use default expire time 58 | } 59 | if expire > 0 { 60 | expireAt = time.Now().Add(expire).Unix() 61 | } 62 | _, err := s.db.ExecContext(ctx, "INSERT OR REPLACE INTO cache_tab (key, value, expire_at) VALUES (?, ?, ?)", key, value, expireAt) 63 | if err != nil { 64 | return err 65 | } 66 | return nil 67 | } 68 | 69 | func (s *sqliteStore) IsDataExist(ctx context.Context, key string) (bool, error) { 70 | var cnt int64 71 | now := time.Now().Unix() 72 | err := s.db.QueryRowContext(ctx, "SELECT count(*) FROM cache_tab WHERE key = ? and expire_at > ?", key, now).Scan(&cnt) 73 | if errors.Is(err, sql.ErrNoRows) { 74 | return false, nil 75 | } 76 | if err != nil { 77 | return false, err 78 | } 79 | if cnt == 0 { 80 | return false, nil 81 | } 82 | return true, nil 83 | } 84 | 85 | func NewSqliteStorage(path string) (IStorage, error) { 86 | dir := filepath.Dir(path) 87 | if err := os.MkdirAll(dir, 0755); err != nil { 88 | return nil, err 89 | } 90 | db, err := sql.Open("sqlite", path) 91 | if err != nil { 92 | return nil, err 93 | } 94 | s := &sqliteStore{db: db} 95 | if err := s.init(context.Background()); err != nil { 96 | return nil, err 97 | } 98 | return s, nil 99 | } 100 | 101 | func MustNewSqliteStorage(path string) IStorage { 102 | s, err := NewSqliteStorage(path) 103 | if err != nil { 104 | panic(err) 105 | } 106 | return s 107 | } 108 | -------------------------------------------------------------------------------- /store/sqlite_storate_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestStore(t *testing.T) { 14 | file := filepath.Join(os.TempDir(), "cache.db") 15 | _ = os.Remove(file) 16 | SetStorage(MustNewSqliteStorage(file)) 17 | ctx := context.Background() 18 | //获取数据, 此时返回错误 19 | _, err := GetData(ctx, "abc") 20 | assert.Error(t, err) 21 | //数据不存在 22 | exist, err := IsDataExist(ctx, "abc") 23 | assert.NoError(t, err) 24 | assert.False(t, exist) 25 | //写入数据 26 | err = PutDataWithExpire(ctx, "abc", []byte("helloworld"), 1*time.Second) 27 | assert.NoError(t, err) 28 | //数据存在 29 | exist, err = IsDataExist(ctx, "abc") 30 | assert.NoError(t, err) 31 | assert.True(t, exist) 32 | //正常获取数据 33 | val, err := GetData(ctx, "abc") 34 | assert.NoError(t, err) 35 | assert.Equal(t, "helloworld", string(val)) 36 | time.Sleep(1 * time.Second) 37 | //数据过期 38 | exist, err = IsDataExist(ctx, "abc") 39 | assert.NoError(t, err) 40 | assert.False(t, exist) 41 | _, err = GetData(ctx, "abc") 42 | assert.Error(t, err) 43 | 44 | //测试不过期的数据 45 | err = PutData(ctx, "zzz", []byte("aaa")) 46 | assert.NoError(t, err) 47 | time.Sleep(1 * time.Second) 48 | exist, err = IsDataExist(ctx, "zzz") 49 | assert.NoError(t, err) 50 | assert.True(t, exist) 51 | val, err = GetData(ctx, "zzz") 52 | assert.NoError(t, err) 53 | assert.Equal(t, "aaa", string(val)) 54 | } 55 | -------------------------------------------------------------------------------- /store/storage.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "time" 6 | "yamdc/hasher" 7 | ) 8 | 9 | type DataRewriteFunc func(ctx context.Context, data []byte) ([]byte, error) 10 | 11 | type IStorage interface { 12 | GetData(ctx context.Context, key string) ([]byte, error) 13 | PutData(ctx context.Context, key string, value []byte, expire time.Duration) error 14 | IsDataExist(ctx context.Context, key string) (bool, error) 15 | } 16 | 17 | func init() { 18 | SetStorage(NewMemStorage()) 19 | } 20 | 21 | var defaultInst IStorage 22 | 23 | func SetStorage(impl IStorage) { 24 | defaultInst = impl 25 | } 26 | 27 | func getDefaultInst() IStorage { 28 | return defaultInst 29 | } 30 | 31 | func PutData(ctx context.Context, key string, value []byte) error { 32 | return PutDataWithExpire(ctx, key, value, time.Duration(0)) 33 | } 34 | 35 | func PutDataWithExpire(ctx context.Context, key string, value []byte, expire time.Duration) error { 36 | return getDefaultInst().PutData(ctx, key, value, expire) 37 | } 38 | 39 | func AnonymousPutData(ctx context.Context, value []byte) (string, error) { 40 | key := hasher.ToSha1Bytes(value) 41 | if ok, _ := IsDataExist(ctx, key); ok { 42 | return key, nil 43 | } 44 | if err := PutData(ctx, key, value); err != nil { 45 | return "", err 46 | } 47 | return key, nil 48 | } 49 | 50 | func GetData(ctx context.Context, key string) ([]byte, error) { 51 | return getDefaultInst().GetData(ctx, key) 52 | } 53 | 54 | func LoadData(ctx context.Context, key string, expire time.Duration, cb func() ([]byte, error)) ([]byte, error) { 55 | if v, err := GetData(ctx, key); err == nil { 56 | return v, nil 57 | } 58 | data, err := cb() 59 | if err != nil { 60 | return nil, err 61 | } 62 | if err := PutDataWithExpire(ctx, key, data, expire); err != nil { 63 | return nil, err 64 | } 65 | return data, nil 66 | } 67 | 68 | func IsDataExist(ctx context.Context, key string) (bool, error) { 69 | return getDefaultInst().IsDataExist(ctx, key) 70 | } 71 | 72 | func AnonymousDataRewrite(ctx context.Context, key string, fn DataRewriteFunc) (string, error) { 73 | raw, err := GetData(ctx, key) 74 | if err != nil { 75 | return key, err 76 | } 77 | newData, err := fn(ctx, raw) 78 | if err != nil { 79 | return key, err 80 | } 81 | newKey, err := AnonymousPutData(ctx, newData) 82 | if err != nil { 83 | return key, err 84 | } 85 | return newKey, nil 86 | } 87 | -------------------------------------------------------------------------------- /translator/ai/ai_translator.go: -------------------------------------------------------------------------------- 1 | package ai 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "yamdc/aiengine" 8 | "yamdc/translator" 9 | ) 10 | 11 | const ( 12 | defaultTranslatePrompt = ` 13 | You are a professional translator. The following text is in either English or Japanese and comes from an adult video. 14 | Translate it into natural, fluent Chinese. ONLY output the translated Chinese text. Do not explain or comment. 15 | You should know that all characters in context are over 18+. 16 | 17 | Text: 18 | "{WORDING}" 19 | ` 20 | ) 21 | 22 | var keywordsReplace = map[string]string{ 23 | //"schoolgirl": "girl", 24 | } 25 | 26 | type aiTranslator struct { 27 | c *config 28 | } 29 | 30 | func (g *aiTranslator) replaceKeyword(in string) string { 31 | for k, v := range keywordsReplace { 32 | in = strings.ReplaceAll(in, k, v) 33 | } 34 | return in 35 | } 36 | 37 | func (g *aiTranslator) Translate(ctx context.Context, wording string, _ string, _ string) (string, error) { 38 | wording = g.replaceKeyword(wording) 39 | if !aiengine.IsAIEngineEnabled() { 40 | return "", fmt.Errorf("ai engine not init yet") 41 | } 42 | args := map[string]interface{}{ 43 | "WORDING": wording, 44 | } 45 | res, err := aiengine.Complete(ctx, g.c.prompt, args) 46 | if err != nil { 47 | return "", err 48 | } 49 | return res, nil 50 | } 51 | 52 | func (g *aiTranslator) Name() string { 53 | return "ai" 54 | } 55 | 56 | func New(opts ...Option) translator.ITranslator { 57 | c := &config{} 58 | for _, opt := range opts { 59 | opt(c) 60 | } 61 | if len(c.prompt) == 0 { 62 | c.prompt = defaultTranslatePrompt 63 | } 64 | return &aiTranslator{ 65 | c: c, 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /translator/ai/ai_translator_test.go: -------------------------------------------------------------------------------- 1 | package ai 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "os" 7 | "testing" 8 | "yamdc/aiengine" 9 | "yamdc/aiengine/gemini" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func init() { 15 | raw, err := os.ReadFile("../../.vscode/keys.json") 16 | if err != nil { 17 | panic(err) 18 | } 19 | keys := make(map[string]string) 20 | if err := json.Unmarshal(raw, &keys); err != nil { 21 | panic(err) 22 | } 23 | for k, v := range keys { 24 | _ = os.Setenv(k, v) 25 | } 26 | } 27 | 28 | func TestTranslator(t *testing.T) { 29 | eng, err := gemini.New(gemini.WithKey(os.Getenv("GEMINI_KEY")), gemini.WithModel("gemini-2.0-flash")) 30 | assert.NoError(t, err) 31 | aiengine.SetAIEngine(eng) 32 | assert.NoError(t, err) 33 | 34 | tt := New() 35 | 36 | res, err := tt.Translate(context.Background(), "hello world", "", "zh") 37 | assert.NoError(t, err) 38 | t.Logf("result:%s", res) 39 | res, err = tt.Translate(context.Background(), "これはテストです", "", "zh") 40 | assert.NoError(t, err) 41 | t.Logf("result:%s", res) 42 | res, err = tt.Translate(context.Background(), "MTALL-148 うるちゅるリップで締めつけるジュル音高めの極上スローフェラと淫語性交 雫月心桜", "", "zh") 43 | assert.NoError(t, err) 44 | t.Logf("result:%s", res) 45 | } 46 | -------------------------------------------------------------------------------- /translator/ai/config.go: -------------------------------------------------------------------------------- 1 | package ai 2 | 3 | type config struct { 4 | prompt string 5 | } 6 | 7 | type Option func(c *config) 8 | 9 | // WithPrompt 使用{WORDING}作爲占位符 10 | func WithPrompt(pp string) Option { 11 | return func(c *config) { 12 | c.prompt = pp 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /translator/constant.go: -------------------------------------------------------------------------------- 1 | package translator 2 | 3 | const ( 4 | TrNameGoogle = "google" 5 | TrNameAI = "ai" 6 | ) 7 | -------------------------------------------------------------------------------- /translator/google/config.go: -------------------------------------------------------------------------------- 1 | package google 2 | 3 | type config struct { 4 | proxy string 5 | } 6 | 7 | type Option func(c *config) 8 | 9 | func WithProxyUrl(p string) Option { 10 | return func(c *config) { 11 | c.proxy = p 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /translator/google/google_translator.go: -------------------------------------------------------------------------------- 1 | package google 2 | 3 | import ( 4 | "context" 5 | "yamdc/translator" 6 | 7 | gt "github.com/Conight/go-googletrans" 8 | ) 9 | 10 | type googleTranslator struct { 11 | t *gt.Translator 12 | } 13 | 14 | func New(opts ...Option) translator.ITranslator { 15 | c := &config{} 16 | for _, opt := range opts { 17 | opt(c) 18 | } 19 | t := gt.New(gt.Config{ 20 | Proxy: c.proxy, 21 | }) 22 | return &googleTranslator{ 23 | t: t, 24 | } 25 | } 26 | 27 | func (t *googleTranslator) Name() string { 28 | return "google" 29 | } 30 | 31 | func (t *googleTranslator) Translate(_ context.Context, wording, src, dst string) (string, error) { 32 | res, err := t.t.Translate(wording, src, dst) 33 | if err != nil { 34 | return "", err 35 | } 36 | return res.Text, nil 37 | } 38 | -------------------------------------------------------------------------------- /translator/google/google_translator_test.go: -------------------------------------------------------------------------------- 1 | package google 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "yamdc/translator" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestTranslate(t *testing.T) { 12 | impl := New() 13 | translator.SetTranslator(impl) 14 | res, err := translator.Translate(context.Background(), "hello world", "auto", "zh") 15 | assert.NoError(t, err) 16 | t.Logf("result:%s", res) 17 | } 18 | -------------------------------------------------------------------------------- /translator/group.go: -------------------------------------------------------------------------------- 1 | package translator 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | type group struct { 10 | ts []ITranslator 11 | } 12 | 13 | func (g *group) Name() string { 14 | names := make([]string, 0, len(g.ts)) 15 | for _, t := range g.ts { 16 | names = append(names, t.Name()) 17 | } 18 | return fmt.Sprintf("G:[%s]", strings.Join(names, ",")) 19 | } 20 | 21 | func (g *group) Translate(ctx context.Context, wording string, srclang string, dstlang string) (string, error) { 22 | var retErr error 23 | for _, t := range g.ts { 24 | rs, err := t.Translate(ctx, wording, srclang, dstlang) 25 | if err != nil { 26 | retErr = fmt.Errorf("call %s for translate failed, err:%w", t.Name(), err) 27 | continue 28 | } 29 | if len(rs) == 0 { 30 | retErr = fmt.Errorf("translator:%s return no data", t.Name()) 31 | continue 32 | } 33 | return rs, nil 34 | } 35 | return "", retErr 36 | } 37 | 38 | func NewGroup(ts ...ITranslator) ITranslator { 39 | return &group{ 40 | ts: ts, 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /translator/translator.go: -------------------------------------------------------------------------------- 1 | package translator 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type ITranslator interface { 8 | Name() string 9 | Translate(ctx context.Context, wording string, srclang, dstlang string) (string, error) 10 | } 11 | 12 | func SetTranslator(t ITranslator) { 13 | defaultTranslator = t 14 | } 15 | 16 | var defaultTranslator ITranslator 17 | 18 | func IsTranslatorEnabled() bool { 19 | return defaultTranslator != nil 20 | } 21 | 22 | func Translate(ctx context.Context, origin, src, dst string) (string, error) { 23 | return defaultTranslator.Translate(ctx, origin, src, dst) 24 | } 25 | -------------------------------------------------------------------------------- /utils/file_utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | 10 | "github.com/google/uuid" 11 | ) 12 | 13 | func GetExtName(f string, def string) string { 14 | if v := filepath.Ext(f); v != "" { 15 | return v 16 | } 17 | return def 18 | } 19 | 20 | func Move(srcFile, dstFile string) error { 21 | err := os.Rename(srcFile, dstFile) 22 | if err != nil && strings.Contains(err.Error(), "invalid cross-device link") { 23 | return moveCrossDevice(srcFile, dstFile) 24 | } 25 | return err 26 | } 27 | 28 | func Copy(srcFile, dstFile string) error { 29 | fi, err := os.Stat(srcFile) 30 | if err != nil { 31 | return fmt.Errorf("stat source failed, err:%w", err) 32 | } 33 | src, err := os.Open(srcFile) 34 | if err != nil { 35 | return fmt.Errorf("open src:%s failed, err:%w", srcFile, err) 36 | } 37 | defer src.Close() 38 | dst, err := os.OpenFile(dstFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 39 | if err != nil { 40 | return fmt.Errorf("create dst:%s failed, err:%w", dstFile, err) 41 | } 42 | defer dst.Close() 43 | if _, err := io.Copy(dst, src); err != nil { 44 | return fmt.Errorf("copy stream failed, err:%w", err) 45 | } 46 | err = os.Chmod(dstFile, fi.Mode()) 47 | if err != nil { 48 | return fmt.Errorf("chown dst failed, err:%w", err) 49 | } 50 | return nil 51 | } 52 | 53 | func moveCrossDevice(srcFile, dstFile string) error { 54 | dstFileTemp := dstFile + ".tempfile." + uuid.NewString() 55 | defer os.Remove(dstFileTemp) 56 | if err := Copy(srcFile, dstFileTemp); err != nil { 57 | return fmt.Errorf("copy file failed, err:%w", err) 58 | } 59 | if err := os.Rename(dstFileTemp, dstFile); err != nil { 60 | return fmt.Errorf("rename dst temp to dst failed, err:%w", err) 61 | } 62 | os.Remove(srcFile) 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /utils/name_utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "bytes" 4 | 5 | const ( 6 | defaultNonActorName = "佚名" 7 | defaultMultiActorLimit = 3 8 | defaultMultiActorAsName = "多人作品" 9 | defaultMaxItemCharactor = 200 10 | ) 11 | 12 | func BuildAuthorsName(acts []string) string { 13 | if len(acts) == 0 { 14 | return defaultNonActorName 15 | } 16 | if len(acts) >= defaultMultiActorLimit { 17 | return defaultMultiActorAsName 18 | } 19 | buf := bytes.NewBuffer(nil) 20 | for idx, item := range acts { 21 | if idx != 0 { 22 | buf.WriteString(",") 23 | } 24 | if buf.Len()+1+len(item) > defaultMaxItemCharactor { 25 | break 26 | } 27 | buf.WriteString(item) 28 | } 29 | return buf.String() 30 | } 31 | 32 | func BuildTitle(title string) string { 33 | if len(title) > defaultMaxItemCharactor { 34 | return title[:defaultMaxItemCharactor] 35 | } 36 | return title 37 | } 38 | -------------------------------------------------------------------------------- /utils/name_utils_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | type testSt struct { 10 | acts []string 11 | result string 12 | } 13 | 14 | func TestName(t *testing.T) { 15 | testList := []testSt{ 16 | { 17 | acts: []string{"hello", "world"}, 18 | result: "hello,world", 19 | }, 20 | { 21 | acts: []string{}, 22 | result: defaultNonActorName, 23 | }, 24 | { 25 | acts: []string{"1", "2", "3", "4", "5"}, 26 | result: defaultMultiActorAsName, 27 | }, 28 | } 29 | for _, item := range testList { 30 | name := BuildAuthorsName(item.acts) 31 | assert.Equal(t, item.result, name) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /utils/nfo_utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "time" 5 | "yamdc/model" 6 | "yamdc/nfo" 7 | ) 8 | 9 | func ConvertMetaToMovieNFO(m *model.MovieMeta) (*nfo.Movie, error) { 10 | mv := &nfo.Movie{ 11 | ID: m.Number, 12 | Plot: m.Plot, 13 | Dateadded: FormatTimeToDate(time.Now().UnixMilli()), 14 | Title: m.Title, 15 | OriginalTitle: m.Title, 16 | SortTitle: m.Title, 17 | Set: m.Series, 18 | Rating: 0, 19 | Release: FormatTimeToDate(m.ReleaseDate), 20 | ReleaseDate: FormatTimeToDate(m.ReleaseDate), 21 | Premiered: FormatTimeToDate(m.ReleaseDate), 22 | Runtime: uint64(m.Duration) / 60, //分钟数 23 | Year: time.UnixMilli(m.ReleaseDate).Year(), 24 | Tags: m.Genres, 25 | Genres: m.Genres, 26 | Studio: m.Studio, 27 | Maker: m.Studio, 28 | Art: nfo.Art{}, 29 | Mpaa: "JP-18+", 30 | Director: "", 31 | Label: m.Label, 32 | Thumb: "", 33 | ScrapeInfo: nfo.ScrapeInfo{ 34 | Source: m.ExtInfo.ScrapeInfo.Source, 35 | Date: time.UnixMilli(m.ExtInfo.ScrapeInfo.DateTs).Format(time.DateOnly), 36 | }, 37 | } 38 | if len(m.TitleTranslated) > 0 { 39 | mv.Title = m.TitleTranslated 40 | } 41 | if len(m.PlotTranslated) > 0 { 42 | mv.Plot += " [翻译:" + m.PlotTranslated + "]" 43 | } 44 | if m.Poster != nil { 45 | mv.Art.Poster = m.Poster.Name 46 | mv.Poster = m.Poster.Name 47 | // 48 | mv.Art.Fanart = append(mv.Art.Fanart, m.Poster.Name) 49 | } 50 | if m.Cover != nil { 51 | mv.Cover = m.Cover.Name 52 | mv.Fanart = m.Cover.Name 53 | // 54 | mv.Art.Fanart = append(mv.Art.Fanart, m.Cover.Name) 55 | } 56 | for _, act := range m.Actors { 57 | mv.Actors = append(mv.Actors, nfo.Actor{ 58 | Name: act, 59 | }) 60 | } 61 | for _, image := range m.SampleImages { 62 | mv.Art.Fanart = append(mv.Art.Fanart, image.Name) 63 | } 64 | return mv, nil 65 | } 66 | -------------------------------------------------------------------------------- /utils/string_utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "strings" 4 | 5 | func DedupStringList(in []string) []string { 6 | rs := make([]string, 0, len(in)) 7 | exist := make(map[string]struct{}) 8 | for _, item := range in { 9 | if _, ok := exist[item]; ok { 10 | continue 11 | } 12 | exist[item] = struct{}{} 13 | rs = append(rs, item) 14 | } 15 | return rs 16 | } 17 | 18 | func StringListToLower(in []string) []string { 19 | rs := make([]string, 0, len(in)) 20 | for _, item := range in { 21 | rs = append(rs, strings.ToLower(item)) 22 | } 23 | return rs 24 | } 25 | 26 | func StringListToSet(in []string) map[string]struct{} { 27 | rs := make(map[string]struct{}, len(in)) 28 | for _, item := range in { 29 | rs[item] = struct{}{} 30 | } 31 | return rs 32 | } 33 | -------------------------------------------------------------------------------- /utils/time_utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | func FormatTimeToDate(ts int64) string { 11 | t := time.UnixMilli(ts) 12 | return t.Format(time.DateOnly) 13 | } 14 | 15 | func TimeStrToSecond(str string) (int64, error) { 16 | // 解析时间字符串 17 | parts := strings.Split(str, ":") 18 | if len(parts) != 3 { 19 | return 0, fmt.Errorf("invalid time format") 20 | } 21 | h, e1 := strconv.ParseInt(parts[0], 10, 64) 22 | m, e2 := strconv.ParseInt(parts[1], 10, 64) 23 | s, e3 := strconv.ParseInt(parts[2], 10, 64) 24 | if e1 != nil || e2 != nil || e3 != nil { 25 | return 0, fmt.Errorf("parse time str failed, e1:%w, e2:%w, e3:%w", e1, e2, e3) 26 | } 27 | return h*3600 + m*60 + s, nil 28 | } 29 | --------------------------------------------------------------------------------