├── internal ├── pkg │ ├── ark │ │ ├── version.go │ │ └── server.go │ ├── arkscanner │ │ ├── scanner.go │ │ ├── scanner_picture.go │ │ └── scanner_character.go │ ├── util │ │ ├── cloze │ │ │ └── ignore.go │ │ ├── pathutil │ │ │ ├── pattern.go │ │ │ ├── ext.go │ │ │ └── parent.go │ │ └── fileutil │ │ │ └── files.go │ ├── arkprocessor │ │ ├── processor.go │ │ ├── processor_picture.go │ │ └── processor_character.go │ ├── cols │ │ ├── map.go │ │ └── filter.go │ ├── arkjson │ │ └── arkjson.go │ ├── arkdata │ │ ├── download.go │ │ ├── unzip.go │ │ ├── data_test.go │ │ └── data.go │ ├── arkparser │ │ ├── story_pictures.go │ │ ├── story_review_meta_table.go │ │ ├── directive.go │ │ ├── story_characters.go │ │ ├── story_review_table.go │ │ └── arkparser.go │ └── arkassets │ │ ├── assets_test.go │ │ ├── unzip.go │ │ ├── fetch.go │ │ ├── unpack.go │ │ ├── raw.go │ │ └── assets.go └── app │ ├── story │ ├── model_tree.go │ ├── story.go │ ├── model_art_picture.go │ ├── model_group.go │ ├── model_art_character.go │ ├── model_story.go │ ├── controller.go │ ├── repo.go │ └── service.go │ ├── artext │ ├── artext.go │ ├── service.go │ ├── controller.go │ ├── ext_character.go │ └── ext_story.go │ ├── app.go │ ├── gallery │ ├── gallery.go │ ├── service.go │ ├── model.go │ ├── controller.go │ └── repository.go │ ├── art │ ├── art.go │ ├── art_content.go │ ├── art_art.go │ ├── art_variant.go │ ├── service.go │ ├── controller.go │ └── repository.go │ ├── infra │ ├── fiber.go │ ├── config.go │ └── gorm.go │ └── updateloop │ ├── update_art_thumbnail.go │ ├── update_story.go │ ├── updateloop.go │ ├── repo_version.go │ ├── update_gallery.go │ └── update_art.go ├── deploy ├── reverse_proxy │ ├── Dockerfile │ └── Caddyfile └── docker-compose.yml ├── assets ├── arkwaifu_phantom.png ├── arkwaifu_phantom@0.25x.png └── README.md ├── tools └── extractor │ ├── requirements.txt │ ├── .gitignore │ └── main.py ├── cmd ├── service │ └── main.go └── updateloop │ └── main.go ├── .github └── workflows │ ├── automatic-release.yml │ ├── docker-image-service.yml │ └── docker-image-updateloop.yml ├── .gitignore ├── LICENSE ├── go.mod └── README.md /internal/pkg/ark/version.go: -------------------------------------------------------------------------------- 1 | package ark 2 | 3 | type Version = string 4 | -------------------------------------------------------------------------------- /internal/app/story/model_tree.go: -------------------------------------------------------------------------------- 1 | package story 2 | 3 | type Tree = []Group 4 | -------------------------------------------------------------------------------- /deploy/reverse_proxy/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM caddy:2.6 2 | 3 | COPY Caddyfile /etc/caddy/Caddyfile 4 | -------------------------------------------------------------------------------- /assets/arkwaifu_phantom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FlandiaYingman/arkwaifu/HEAD/assets/arkwaifu_phantom.png -------------------------------------------------------------------------------- /internal/pkg/arkscanner/scanner.go: -------------------------------------------------------------------------------- 1 | package arkscanner 2 | 3 | type Scanner struct { 4 | Root string 5 | } 6 | -------------------------------------------------------------------------------- /assets/arkwaifu_phantom@0.25x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FlandiaYingman/arkwaifu/HEAD/assets/arkwaifu_phantom@0.25x.png -------------------------------------------------------------------------------- /deploy/reverse_proxy/Caddyfile: -------------------------------------------------------------------------------- 1 | http:// 2 | 3 | reverse_proxy * {$FRONTEND_ADDRESS} 4 | reverse_proxy /api* {$BACKEND_ADDRESS} 5 | -------------------------------------------------------------------------------- /assets/README.md: -------------------------------------------------------------------------------- 1 | # Assets 2 | 3 | All resources in this directory are licensed under 4 | [Attribution-NonCommercial 4.0 International (CC BY-NC 4.0)](https://creativecommons.org/licenses/by-nc/4.0/). -------------------------------------------------------------------------------- /internal/pkg/util/cloze/ignore.go: -------------------------------------------------------------------------------- 1 | package cloze 2 | 3 | import "io" 4 | 5 | // IgnoreErr closes a closer but ignore its error explicitly. 6 | func IgnoreErr(closer io.Closer) { 7 | _ = closer.Close() 8 | } 9 | -------------------------------------------------------------------------------- /tools/extractor/requirements.txt: -------------------------------------------------------------------------------- 1 | archspec==0.2.5 2 | astc-encoder-py==0.1.11 3 | attrs==25.3.0 4 | Brotli==1.1.0 5 | etcpak==0.9.13 6 | fsspec==2025.3.2 7 | lz4==4.4.4 8 | pillow==11.1.0 9 | pyfmodex==0.7.2 10 | texture2ddecoder==1.0.5 11 | UnityPy==1.21.3 -------------------------------------------------------------------------------- /internal/pkg/arkprocessor/processor.go: -------------------------------------------------------------------------------- 1 | package arkprocessor 2 | 3 | // Processor provides functionalities for processing art. The term "process" 4 | // here refers to merging color channel and alpha channel of arts together, 5 | // merging faces variation and bodies of character arts together, etc. 6 | type Processor struct { 7 | Root string 8 | } 9 | -------------------------------------------------------------------------------- /internal/app/story/story.go: -------------------------------------------------------------------------------- 1 | package story 2 | 3 | import "go.uber.org/fx" 4 | 5 | func FxModule() fx.Option { 6 | return fx.Module("story", 7 | fx.Provide( 8 | newRepo, 9 | newService, 10 | newController, 11 | ), 12 | fx.Invoke( 13 | fx.Annotate( 14 | registerController, 15 | fx.ParamTags(``, `optional:"true"`), 16 | ), 17 | ), 18 | ) 19 | } 20 | -------------------------------------------------------------------------------- /internal/app/artext/artext.go: -------------------------------------------------------------------------------- 1 | package artext 2 | 3 | import ( 4 | "go.uber.org/fx" 5 | ) 6 | 7 | func FxModule() fx.Option { 8 | registerController := func(controller *Controller, param RegisterParams) { controller.register(param) } 9 | 10 | return fx.Module("art-extension", 11 | fx.Provide( 12 | newService, 13 | newController, 14 | ), 15 | fx.Invoke(registerController), 16 | ) 17 | } 18 | -------------------------------------------------------------------------------- /cmd/service/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/flandiayingman/arkwaifu/internal/app" 5 | "github.com/flandiayingman/arkwaifu/internal/app/infra" 6 | "go.uber.org/fx" 7 | ) 8 | 9 | func main() { 10 | fxApp := fx.New( 11 | fx.Provide( 12 | infra.ProvideConfig, 13 | infra.ProvideFiber, 14 | infra.ProvideGorm, 15 | ), 16 | app.FxModules(), 17 | ) 18 | fxApp.Run() 19 | } 20 | -------------------------------------------------------------------------------- /internal/app/artext/service.go: -------------------------------------------------------------------------------- 1 | package artext 2 | 3 | import ( 4 | "github.com/flandiayingman/arkwaifu/internal/app/art" 5 | "github.com/flandiayingman/arkwaifu/internal/app/story" 6 | ) 7 | 8 | type Service struct { 9 | art *art.Service 10 | story *story.Service 11 | } 12 | 13 | func newService(serviceArt *art.Service, serviceStory *story.Service) *Service { 14 | return &Service{ 15 | art: serviceArt, 16 | story: serviceStory, 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /cmd/updateloop/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/flandiayingman/arkwaifu/internal/app" 5 | "github.com/flandiayingman/arkwaifu/internal/app/infra" 6 | "github.com/flandiayingman/arkwaifu/internal/app/updateloop" 7 | "go.uber.org/fx" 8 | ) 9 | 10 | func main() { 11 | fxApp := fx.New( 12 | fx.Provide( 13 | infra.ProvideConfig, 14 | infra.ProvideGorm, 15 | ), 16 | app.FxModules(), 17 | updateloop.FxModule(), 18 | ) 19 | fxApp.Run() 20 | } 21 | -------------------------------------------------------------------------------- /internal/pkg/cols/map.go: -------------------------------------------------------------------------------- 1 | package cols 2 | 3 | func Map[In any, Out any](in []In, f func(i In) (o Out)) (out []Out) { 4 | for _, element := range in { 5 | out = append(out, f(element)) 6 | } 7 | return 8 | } 9 | 10 | func MapErr[In any, Out any](in []In, f func(i In) (o Out, err error)) (out []Out, err error) { 11 | for _, i := range in { 12 | var o Out 13 | o, err = f(i) 14 | if err != nil { 15 | return 16 | } 17 | out = append(out, o) 18 | } 19 | return 20 | } 21 | -------------------------------------------------------------------------------- /internal/app/app.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "github.com/flandiayingman/arkwaifu/internal/app/art" 5 | "github.com/flandiayingman/arkwaifu/internal/app/artext" 6 | "github.com/flandiayingman/arkwaifu/internal/app/gallery" 7 | "github.com/flandiayingman/arkwaifu/internal/app/story" 8 | "go.uber.org/fx" 9 | ) 10 | 11 | func FxModules() fx.Option { 12 | return fx.Options( 13 | artext.FxModule(), 14 | art.FxModule(), 15 | story.FxModule(), 16 | gallery.FxModule(), 17 | ) 18 | } 19 | -------------------------------------------------------------------------------- /internal/app/gallery/gallery.go: -------------------------------------------------------------------------------- 1 | package gallery 2 | 3 | import "go.uber.org/fx" 4 | 5 | // FxModule teaches a fx.App how to instantiate an art.Service, and also 6 | // optionally register its HTTP endpoints on api.V1 if provided. 7 | func FxModule() fx.Option { 8 | return fx.Module("gallery", 9 | fx.Provide( 10 | newRepository, 11 | newService, 12 | newController, 13 | ), 14 | fx.Invoke( 15 | fx.Annotate( 16 | registerController, 17 | fx.ParamTags(``, `optional:"true"`), 18 | ), 19 | ), 20 | ) 21 | } 22 | -------------------------------------------------------------------------------- /internal/pkg/cols/filter.go: -------------------------------------------------------------------------------- 1 | package cols 2 | 3 | func Filter[T any](in []T, predicate func(element T) bool) (out []T) { 4 | for _, element := range in { 5 | if predicate(element) { 6 | out = append(out, element) 7 | } 8 | } 9 | return 10 | } 11 | 12 | func FilterErr[T any](in []T, predicate func(element T) (bool, error)) (out []T, err error) { 13 | for _, element := range in { 14 | ok, err := predicate(element) 15 | if err != nil { 16 | return nil, err 17 | } 18 | if ok { 19 | out = append(out, element) 20 | } 21 | } 22 | return 23 | } 24 | -------------------------------------------------------------------------------- /internal/pkg/arkjson/arkjson.go: -------------------------------------------------------------------------------- 1 | package arkjson 2 | 3 | import ( 4 | "github.com/Jeffail/gabs/v2" 5 | "path/filepath" 6 | ) 7 | 8 | const ( 9 | StoryReviewMetaTablePath = "assets/torappu/dynamicassets/gamedata/excel/story_review_meta_table.json" 10 | ReplicateTable = "assets/torappu/dynamicassets/gamedata/excel/replicate_table.json" 11 | RetroTable = "assets/torappu/dynamicassets/gamedata/excel/retro_table.json" 12 | RoguelikeTopicTable = "assets/torappu/dynamicassets/gamedata/excel/roguelike_topic_table.json" 13 | ) 14 | 15 | func Get(root string, path string) (*gabs.Container, error) { 16 | json, err := gabs.ParseJSONFile(filepath.Join(root, path)) 17 | if err != nil { 18 | return nil, err 19 | } 20 | return json, nil 21 | } 22 | -------------------------------------------------------------------------------- /internal/app/artext/controller.go: -------------------------------------------------------------------------------- 1 | package artext 2 | 3 | import ( 4 | "github.com/gofiber/fiber/v2" 5 | "go.uber.org/fx" 6 | ) 7 | 8 | type Controller struct { 9 | service *Service 10 | } 11 | 12 | func newController(service *Service) *Controller { 13 | return &Controller{service: service} 14 | } 15 | 16 | type RegisterParams struct { 17 | fx.In 18 | Router fiber.Router `optional:"true"` 19 | } 20 | 21 | func (c *Controller) register(params RegisterParams) { 22 | router := params.Router 23 | if router == nil { 24 | return 25 | } 26 | 27 | // extension of story 28 | router.Get("arts", 29 | c.GetArtsOfStoryGroup, 30 | c.GetArtsOfStory, 31 | c.GetArtsExceptForStoryArts, 32 | ) 33 | 34 | // extension of character 35 | router.Get("arts/:id/siblings", c.GetSiblingsOfCharacterArt) 36 | } 37 | -------------------------------------------------------------------------------- /internal/app/story/model_art_picture.go: -------------------------------------------------------------------------------- 1 | package story 2 | 3 | import "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 4 | 5 | type PictureArt struct { 6 | Server ark.Server `json:"server" gorm:"primaryKey;type:game_server"` 7 | ID string `json:"id" gorm:"primaryKey;check:id=lower(id)"` 8 | StoryID string `json:"storyID" gorm:"primaryKey"` 9 | Category string `json:"category" gorm:""` 10 | 11 | Title string `json:"title" gorm:""` 12 | Subtitle string `json:"subtitle" gorm:""` 13 | 14 | SortID *uint64 `json:"-" gorm:"unique;autoIncrement"` 15 | } 16 | 17 | type AggregatedPictureArt struct { 18 | Server ark.Server `json:"server"` 19 | ID string `json:"id"` 20 | Category string `json:"category"` 21 | 22 | Title string `json:"title"` 23 | Subtitle string `json:"subtitle"` 24 | } 25 | -------------------------------------------------------------------------------- /internal/app/art/art.go: -------------------------------------------------------------------------------- 1 | // Package art provides functionalities related to Art and Variant of the game, 2 | // including serving them, manipulating them and keep them persisted. 3 | // 4 | // This package exposes its main interfaces by fx via FxModule function. However, 5 | // it is still easy to instantiate an art.Service by simply creating it. 6 | package art 7 | 8 | import ( 9 | "go.uber.org/fx" 10 | ) 11 | 12 | // FxModule teaches a fx.App how to instantiate an art.Service, and also 13 | // optionally register its HTTP endpoints on api.V1 if provided. 14 | func FxModule() fx.Option { 15 | return fx.Module("art", 16 | fx.Provide( 17 | newRepo, 18 | newService, 19 | newController, 20 | ), 21 | fx.Invoke( 22 | fx.Annotate( 23 | registerController, 24 | fx.ParamTags(``, `optional:"true"`), 25 | ), 26 | ), 27 | ) 28 | } 29 | -------------------------------------------------------------------------------- /internal/app/story/model_group.go: -------------------------------------------------------------------------------- 1 | package story 2 | 3 | import "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 4 | 5 | type Group struct { 6 | Server ark.Server `json:"server" gorm:"primaryKey;type:game_server"` 7 | ID string `json:"id" gorm:"primaryKey;check:id=lower(id)"` 8 | Name string `json:"name" gorm:""` 9 | Type GroupType `json:"type" gorm:"type:story_group_type"` 10 | Stories []Story `json:"stories" gorm:"foreignKey:Server,GroupID;reference:Server,ID;constraint:OnDelete:CASCADE"` 11 | 12 | SortID *uint64 `json:"-" gorm:"unique;autoIncrement"` 13 | } 14 | 15 | type GroupType = string 16 | 17 | const ( 18 | GroupTypeMainStory GroupType = "main-story" 19 | GroupTypeMajorEvent GroupType = "major-event" 20 | GroupTypeMinorEvent GroupType = "minor-event" 21 | GroupTypeOther GroupType = "other" 22 | ) 23 | -------------------------------------------------------------------------------- /.github/workflows/automatic-release.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: Automatic Release 4 | 5 | # Controls when the workflow will run 6 | on: 7 | # Triggers the workflow on push or pull request events but only for the master branch 8 | push: 9 | tags: [ "v**" ] 10 | 11 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 12 | jobs: 13 | # This workflow contains a single job called "build" 14 | build: 15 | # The type of runner that the job will run on 16 | runs-on: ubuntu-latest 17 | 18 | # Steps represent a sequence of tasks that will be executed as part of the job 19 | steps: 20 | - uses: "marvinpinto/action-automatic-releases@latest" 21 | with: 22 | repo_token: ${{ secrets.GITHUB_TOKEN }} 23 | prerelease: false 24 | -------------------------------------------------------------------------------- /internal/app/story/model_art_character.go: -------------------------------------------------------------------------------- 1 | package story 2 | 3 | import ( 4 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 5 | "github.com/lib/pq" 6 | ) 7 | 8 | type CharacterArt struct { 9 | Server ark.Server `json:"server" gorm:"primaryKey;type:game_server"` 10 | ID string `json:"id" gorm:"primaryKey;check:id=lower(id)"` 11 | StoryID string `json:"storyID" gorm:"primaryKey"` 12 | Category string `json:"category" gorm:""` 13 | 14 | Names pq.StringArray `json:"names" gorm:"type:text[];default:array[]::text[]"` 15 | 16 | SortID *uint64 `json:"-" gorm:"unique;autoIncrement"` 17 | } 18 | 19 | type AggregatedCharacterArt struct { 20 | Server ark.Server `json:"server"` 21 | ID string `json:"id"` 22 | Category string `json:"category"` 23 | 24 | Names pq.StringArray `json:"names" gorm:"type:text[];default:array[]::text[]"` 25 | } 26 | -------------------------------------------------------------------------------- /internal/app/gallery/service.go: -------------------------------------------------------------------------------- 1 | package gallery 2 | 3 | import "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 4 | 5 | type Service struct { 6 | r *repository 7 | } 8 | 9 | func newService(r *repository) *Service { 10 | return &Service{r: r} 11 | } 12 | 13 | func (s *Service) ListGalleries(server ark.Server) ([]Gallery, error) { 14 | return s.r.ListGalleries(server) 15 | } 16 | 17 | func (s *Service) GetGalleryByID(server ark.Server, id string) (*Gallery, error) { 18 | return s.r.GetGalleryByID(server, id) 19 | } 20 | 21 | func (s *Service) ListGalleryArts(server ark.Server) ([]Art, error) { 22 | return s.r.ListGalleryArts(server) 23 | } 24 | 25 | func (s *Service) GetGalleryArtByID(server ark.Server, id string) (*Art, error) { 26 | return s.r.GetGalleryArtByID(server, id) 27 | } 28 | 29 | func (s *Service) Put(g []Gallery) (err error) { 30 | return s.r.Put(g) 31 | } 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### Ark Waifu ### 2 | arkwaifu-root 3 | .env 4 | 5 | # Created by https://www.toptal.com/developers/gitignore/api/go 6 | # Edit at https://www.toptal.com/developers/gitignore?templates=go 7 | 8 | ### Go ### 9 | # If you prefer the allow list template instead of the deny list, see community template: 10 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 11 | # 12 | # Binaries for programs and plugins 13 | *.exe 14 | *.exe~ 15 | *.dll 16 | *.so 17 | *.dylib 18 | 19 | # Test binary, built with `go test -c` 20 | *.test 21 | 22 | # Output of the go coverage tool, specifically when used with LiteIDE 23 | *.out 24 | 25 | # Dependency directories (remove the comment below to include it) 26 | # vendor/ 27 | 28 | # Go workspace file 29 | go.work 30 | 31 | ### Go Patch ### 32 | /vendor/ 33 | /Godeps/ 34 | 35 | # End of https://www.toptal.com/developers/gitignore/api/go -------------------------------------------------------------------------------- /internal/app/infra/fiber.go: -------------------------------------------------------------------------------- 1 | package infra 2 | 3 | import ( 4 | "context" 5 | "go.uber.org/fx" 6 | "time" 7 | 8 | "github.com/gofiber/fiber/v2" 9 | "github.com/gofiber/fiber/v2/middleware/cors" 10 | "github.com/gofiber/fiber/v2/middleware/logger" 11 | ) 12 | 13 | func ProvideFiber(lc fx.Lifecycle, config *Config) (*fiber.App, fiber.Router) { 14 | app := fiber.New(fiber.Config{ 15 | AppName: "Arkwaifu", 16 | ReadTimeout: time.Second * 20, 17 | WriteTimeout: time.Second * 20, 18 | BodyLimit: 16 * 1024 * 1024, 19 | UnescapePath: true, 20 | }) 21 | app.Use(cors.New(cors.Config{ 22 | AllowOrigins: "*", 23 | })).Use(logger.New()) 24 | 25 | lc.Append(fx.Hook{ 26 | OnStart: func(ctx context.Context) error { 27 | go func() { 28 | err := app.Listen(config.Address) 29 | if err != nil { 30 | panic(err) 31 | } 32 | }() 33 | return nil 34 | }, 35 | OnStop: func(ctx context.Context) error { 36 | return app.ShutdownWithContext(ctx) 37 | }, 38 | }) 39 | 40 | return app, app.Group("api/v1") 41 | } 42 | -------------------------------------------------------------------------------- /internal/pkg/arkdata/download.go: -------------------------------------------------------------------------------- 1 | package arkdata 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/cavaliergopher/grab/v3" 7 | "github.com/pkg/errors" 8 | "os" 9 | ) 10 | 11 | func download(ctx context.Context, repoOwner, repoName, sha string) (string, string, error) { 12 | temp, err := os.MkdirTemp("", "arkdata_download-*") 13 | if err != nil { 14 | return "", "", errors.WithStack(err) 15 | } 16 | 17 | ctx, cancelCtx := context.WithCancel(ctx) 18 | defer cancelCtx() 19 | 20 | request, err := grab.NewRequest(temp, urlOfZipball(repoOwner, repoName, sha)) 21 | if err != nil { 22 | os.RemoveAll(temp) 23 | return "", "", errors.WithStack(err) 24 | } 25 | 26 | request = request.WithContext(ctx) 27 | client := grab.NewClient() 28 | client.UserAgent = "FlandiaYingman/arkwaifu" 29 | response := client.Do(request) 30 | return temp, response.Filename, response.Err() 31 | } 32 | 33 | func urlOfZipball(repoOwner, repoName, sha string) string { 34 | return fmt.Sprintf("https://api.github.com/repos/%s/%s/zipball/%s", repoOwner, repoName, sha) 35 | } 36 | -------------------------------------------------------------------------------- /internal/app/story/model_story.go: -------------------------------------------------------------------------------- 1 | package story 2 | 3 | import "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 4 | 5 | type Story struct { 6 | Server ark.Server `json:"server" gorm:"primaryKey;type:game_server"` 7 | ID string `json:"id" gorm:"primaryKey;check:id=lower(id)"` 8 | 9 | Tag Tag `json:"tag" gorm:"type:story_tag"` 10 | TagText string `json:"tagText" gorm:""` 11 | Code string `json:"code" gorm:""` 12 | Name string `json:"name" gorm:""` 13 | Info string `json:"info" gorm:""` 14 | 15 | GroupID string `json:"groupID" gorm:""` 16 | SortID *uint64 `json:"-" gorm:"unique;autoIncrement"` 17 | 18 | PictureArts []PictureArt `json:"pictureArts" gorm:"foreignKey:Server,StoryID;reference:(Server,ID);constraint:OnDelete:CASCADE"` 19 | CharacterArts []CharacterArt `json:"characterArts" gorm:"foreignKey:Server,StoryID;reference:(Server,ID);constraint:OnDelete:CASCADE"` 20 | } 21 | 22 | type Tag = string 23 | 24 | const ( 25 | TagBefore Tag = "before" 26 | TagAfter Tag = "after" 27 | TagInterlude Tag = "interlude" 28 | ) 29 | -------------------------------------------------------------------------------- /internal/app/art/art_content.go: -------------------------------------------------------------------------------- 1 | package art 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | _ "github.com/chai2010/webp" 7 | "image" 8 | _ "image/jpeg" 9 | _ "image/png" 10 | "path" 11 | ) 12 | 13 | // Content helps store and take the content of a variant to or from the 14 | // filesystem. 15 | type Content struct { 16 | ID string 17 | Category Category 18 | Variation Variation 19 | Content []byte 20 | } 21 | 22 | func (s *Content) String() string { 23 | if s.Variation == VariationOrigin { 24 | return fmt.Sprintf("%s", s.ID) 25 | } else { 26 | return fmt.Sprintf("%s/%s", s.Variation, s.ID) 27 | } 28 | } 29 | 30 | func (s *Content) PathRel() string { 31 | return path.Join(string(s.Category), s.String()+".webp") 32 | } 33 | 34 | func (s *Content) Check() (*image.Config, error) { 35 | config, format, err := image.DecodeConfig(bytes.NewReader(s.Content)) 36 | if err != nil { 37 | return nil, fmt.Errorf("cannot decode content file %v: %w", s, err) 38 | } 39 | if format != "webp" { 40 | return nil, fmt.Errorf("the format of content file %v is not webp", s) 41 | } 42 | return &config, nil 43 | } 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Flandia Yingman 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /internal/pkg/util/pathutil/pattern.go: -------------------------------------------------------------------------------- 1 | package pathutil 2 | 3 | import ( 4 | "github.com/gobwas/glob" 5 | "io/fs" 6 | "path/filepath" 7 | ) 8 | 9 | func MatchAny(patterns []string, name string) (bool, error) { 10 | name = filepath.ToSlash(name) 11 | for _, pattern := range patterns { 12 | match, err := Match(pattern, name) 13 | if err != nil { 14 | return false, err 15 | } 16 | if match { 17 | return true, nil 18 | } 19 | } 20 | return false, nil 21 | } 22 | 23 | func Match(pattern string, name string) (bool, error) { 24 | globber, err := glob.Compile(pattern, '/', '\\') 25 | if err != nil { 26 | return false, err 27 | } 28 | return globber.Match(name), nil 29 | } 30 | 31 | func Glob(pattern []string, root string) ([]string, error) { 32 | results := make([]string, 0) 33 | return results, filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { 34 | rel, err := filepath.Rel(root, p) 35 | if err != nil { 36 | return err 37 | } 38 | match, err := MatchAny(pattern, rel) 39 | if err != nil { 40 | return err 41 | } 42 | if match { 43 | results = append(results, p) 44 | } 45 | 46 | return nil 47 | }) 48 | } 49 | -------------------------------------------------------------------------------- /internal/app/gallery/model.go: -------------------------------------------------------------------------------- 1 | package gallery 2 | 3 | import "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 4 | 5 | type Gallery struct { 6 | Server ark.Server `json:"-" gorm:"primaryKey;type:game_server"` 7 | 8 | ID string `json:"id" gorm:"primaryKey;type:text COLLATE numeric;check:id=lower(id)"` 9 | Name string `json:"name" gorm:""` 10 | Description string `json:"description" gorm:""` 11 | Arts []Art `json:"arts" gorm:"foreignKey:Server,GalleryID;references:Server,ID;constraint:OnDelete:CASCADE"` 12 | } 13 | 14 | func (g Gallery) TableName() string { 15 | return "gallery_galleries" 16 | } 17 | 18 | type Art struct { 19 | Server ark.Server `json:"-" gorm:"primaryKey;type:game_server"` 20 | GalleryID string `json:"-" gorm:""` 21 | SortID int `json:"-" gorm:""` 22 | 23 | ID string `json:"id" gorm:"primaryKey;type:text COLLATE numeric;check:id=lower(id)"` 24 | Name string `json:"name" gorm:""` 25 | Description string `json:"description" gorm:""` 26 | ArtID string `json:"artID" gorm:"type:text COLLATE numeric;check:id=lower(id)"` 27 | } 28 | 29 | func (a Art) TableName() string { 30 | return "gallery_arts" 31 | } 32 | -------------------------------------------------------------------------------- /internal/app/infra/config.go: -------------------------------------------------------------------------------- 1 | package infra 2 | 3 | import ( 4 | "errors" 5 | "github.com/caarlos0/env/v6" 6 | "github.com/goccy/go-yaml" 7 | "github.com/google/uuid" 8 | "os" 9 | "path/filepath" 10 | ) 11 | 12 | type Config struct { 13 | Address string `env:"ADDRESS" envDefault:"0.0.0.0:7080"` 14 | Root string `env:"ROOT" envDefault:"./arkwaifu-root"` 15 | PostgresDSN string `env:"POSTGRES_DSN"` 16 | Users []User 17 | } 18 | 19 | type User struct { 20 | ID uuid.UUID 21 | Name string 22 | } 23 | 24 | func ProvideConfig() (*Config, error) { 25 | cfg := Config{} 26 | err := env.Parse(&cfg) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | cfg.Users, err = provideUsers(cfg.Root) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | return &cfg, nil 37 | } 38 | 39 | func provideUsers(root string) ([]User, error) { 40 | path := filepath.Join(root, "users.yaml") 41 | file, err := os.ReadFile(path) 42 | if err != nil && errors.Is(err, os.ErrNotExist) { 43 | return nil, nil 44 | } 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | users := make([]User, 0) 50 | err = yaml.Unmarshal(file, &users) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | return users, nil 56 | } 57 | -------------------------------------------------------------------------------- /internal/pkg/arkparser/story_pictures.go: -------------------------------------------------------------------------------- 1 | package arkparser 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | func (p *Parser) ParsePictures(directives []Directive) []*StoryPicture { 8 | pictureSlice := make([]string, 0) 9 | pictureMap := make(map[string]StoryPicture) 10 | 11 | addPicture := func(id string, category string) { 12 | picture := pictureMap[id] 13 | picture.ID = id 14 | picture.Category = category 15 | pictureMap[id] = picture 16 | pictureSlice = append(pictureSlice, id) 17 | } 18 | 19 | for _, directive := range directives { 20 | switch directive.Name { 21 | case "image": 22 | if id, ok := directive.Params["image"]; ok { 23 | addPicture(id, "image") 24 | } 25 | case "background": 26 | if id, ok := directive.Params["image"]; ok { 27 | addPicture(id, "background") 28 | } 29 | case "largebg", "gridbg": 30 | if ids, ok := directive.Params["imagegroup"]; ok { 31 | for _, id := range strings.Split(ids, "/") { 32 | addPicture(id, "background") 33 | } 34 | } 35 | case "showitem": 36 | if id, ok := directive.Params["image"]; ok { 37 | addPicture(id, "item") 38 | } 39 | } 40 | } 41 | 42 | result := make([]*StoryPicture, 0) 43 | for _, id := range pictureSlice { 44 | picture := pictureMap[id] 45 | result = append(result, &picture) 46 | } 47 | 48 | return result 49 | } 50 | -------------------------------------------------------------------------------- /internal/pkg/util/pathutil/ext.go: -------------------------------------------------------------------------------- 1 | package pathutil 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "strings" 7 | ) 8 | 9 | // Stem returns the name of the path without the extension (e.g. foo.bar.zip -> foo). 10 | func Stem(path string) string { 11 | return RemoveAllExt(filepath.Base(path)) 12 | } 13 | 14 | // RemoveExt returns the path without the extension. 15 | func RemoveExt(path string) string { 16 | return strings.TrimSuffix(path, filepath.Ext(path)) 17 | } 18 | 19 | // RemoveAllExt returns the path without all extensions. 20 | func RemoveAllExt(path string) string { 21 | return strings.TrimSuffix(path, AllExt(path)) 22 | } 23 | 24 | // AllExt returns all extensions (e.g., ".tar.gz" instead of ".gz") from the file name (including "."). 25 | func AllExt(path string) string { 26 | _, after, found := strings.Cut(filepath.Base(path), ".") 27 | if found { 28 | return "." + after 29 | } else { 30 | return "" 31 | } 32 | } 33 | 34 | // ReplaceExt replaces the extension of the given path with the given extension. 35 | func ReplaceExt(path string, ext string) string { 36 | withoutExt := RemoveExt(path) 37 | return fmt.Sprintf("%v%v", withoutExt, ext) 38 | } 39 | 40 | // ReplaceAllExt replaces all extensions of the given path with the given extension. 41 | func ReplaceAllExt(path string, ext string) string { 42 | withoutExt := RemoveAllExt(path) 43 | return fmt.Sprintf("%v%v", withoutExt, ext) 44 | } 45 | -------------------------------------------------------------------------------- /internal/pkg/arkprocessor/processor_picture.go: -------------------------------------------------------------------------------- 1 | package arkprocessor 2 | 3 | import ( 4 | _ "github.com/chai2010/webp" 5 | "github.com/flandiayingman/arkwaifu/internal/pkg/arkscanner" 6 | "github.com/pkg/errors" 7 | "image" 8 | _ "image/jpeg" 9 | _ "image/png" 10 | "os" 11 | "path/filepath" 12 | ) 13 | 14 | type PictureArt arkscanner.PictureArt 15 | 16 | type PictureArtImage struct { 17 | Image image.Image 18 | Art *PictureArt 19 | } 20 | 21 | // ProcessPictureArt process the picture art. 22 | // 23 | // Since picture arts are trivial and different from character arts, the only 24 | // thing this method does is to read the art image and return it. 25 | func (p *Processor) ProcessPictureArt(art *PictureArt) (*PictureArtImage, error) { 26 | img, err := art.decode(p.Root) 27 | if err != nil { 28 | return nil, errors.Wrapf(err, "process picture art %s", art.ID) 29 | } else { 30 | return &PictureArtImage{ 31 | Image: img, 32 | Art: art, 33 | }, nil 34 | } 35 | } 36 | 37 | func (a *PictureArt) decode(root string) (image.Image, error) { 38 | artPath := filepath.Join(root, (*arkscanner.PictureArt)(a).Path()) 39 | 40 | artFile, err := os.Open(artPath) 41 | if err != nil { 42 | return nil, errors.WithStack(err) 43 | } 44 | defer func() { _ = artFile.Close() }() 45 | 46 | img, _, err := image.Decode(artFile) 47 | if err != nil { 48 | return nil, errors.WithStack(err) 49 | } 50 | 51 | return img, nil 52 | } 53 | -------------------------------------------------------------------------------- /internal/app/art/art_art.go: -------------------------------------------------------------------------------- 1 | package art 2 | 3 | import "fmt" 4 | 5 | type Art struct { 6 | ID string `gorm:"primaryKey;type:text COLLATE numeric;check:id=lower(id)" json:"id"` 7 | Category Category `gorm:"type:art_category" json:"category"` 8 | 9 | Variants []Variant `gorm:"" json:"variants,omitempty" validate:""` 10 | } 11 | 12 | func NewArt(id string, category Category) *Art { 13 | return &Art{ 14 | ID: id, 15 | Category: category, 16 | Variants: nil, 17 | } 18 | } 19 | 20 | type Category string 21 | 22 | const ( 23 | CategoryImage Category = "image" 24 | CategoryBackground Category = "background" 25 | CategoryItem Category = "item" 26 | CategoryCharacter Category = "character" 27 | ) 28 | 29 | func ParseCategory(str string) (Category, error) { 30 | category := Category(str) 31 | switch category { 32 | case 33 | CategoryImage, 34 | CategoryBackground, 35 | CategoryItem, 36 | CategoryCharacter: 37 | return category, nil 38 | default: 39 | return "", fmt.Errorf("string %q is not a category", str) 40 | } 41 | } 42 | 43 | func MustParseCategory(str string) Category { 44 | category, err := ParseCategory(str) 45 | if err != nil { 46 | panic(err) 47 | } 48 | return category 49 | } 50 | 51 | func (c *Category) UnmarshalJSON(bytes []byte) error { 52 | category, err := ParseCategory(string(bytes)) 53 | if err != nil { 54 | return err 55 | } 56 | *c = category 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /internal/pkg/arkassets/assets_test.go: -------------------------------------------------------------------------------- 1 | package arkassets 2 | 3 | import ( 4 | "context" 5 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 6 | "github.com/flandiayingman/arkwaifu/internal/pkg/util/pathutil" 7 | "github.com/stretchr/testify/assert" 8 | "os" 9 | "path" 10 | "testing" 11 | ) 12 | 13 | func TestGetGameAssets(t *testing.T) { 14 | type args struct { 15 | ctx context.Context 16 | version ark.Version 17 | dst string 18 | patterns []string 19 | } 20 | tests := []struct { 21 | name string 22 | args args 23 | wantSuccess bool 24 | wantErr bool 25 | }{ 26 | { 27 | name: "avg/imgs/avg_img_0_0.ab", 28 | args: args{ 29 | ctx: context.Background(), 30 | version: "", 31 | dst: "./test_dst", 32 | patterns: []string{"avg/imgs/avg_img_0_0.ab"}, 33 | }, 34 | wantSuccess: true, 35 | wantErr: false, 36 | }, 37 | } 38 | for _, tt := range tests { 39 | t.Run(tt.name, func(t *testing.T) { 40 | defer os.RemoveAll(tt.args.dst) 41 | if err := GetGameAssets(tt.args.ctx, tt.args.version, tt.args.dst, tt.args.patterns); (err != nil) != tt.wantErr { 42 | t.Errorf("GetGameAssets() error = %v, wantErr %v", err, tt.wantErr) 43 | } 44 | files, err := pathutil.Glob([]string{"**"}, path.Join(tt.args.dst)) 45 | if err != nil { 46 | t.Error(err) 47 | } 48 | if tt.wantSuccess { 49 | assert.NotEmpty(t, files) 50 | } else { 51 | assert.Empty(t, files) 52 | } 53 | }) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /internal/pkg/arkparser/story_review_meta_table.go: -------------------------------------------------------------------------------- 1 | package arkparser 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/pkg/errors" 6 | "github.com/wk8/go-ordered-map/v2" 7 | "os" 8 | "path/filepath" 9 | ) 10 | 11 | type JsonResPic struct { 12 | Id string `json:"id"` 13 | Desc string `json:"desc"` 14 | AssetPath string `json:"assetPath"` 15 | PicDescription string `json:"picDescription"` 16 | } 17 | 18 | type JsonComponent struct { 19 | Pic struct { 20 | Pics orderedmap.OrderedMap[string, JsonPic] `json:"pics"` 21 | } `json:"pic"` 22 | } 23 | 24 | type JsonPic struct { 25 | PicId string `json:"picId"` 26 | PicSortId int `json:"picSortId"` 27 | } 28 | 29 | type JsonStoryReviewMetaTable struct { 30 | ActArchiveResData struct { 31 | Pics orderedmap.OrderedMap[string, JsonResPic] `json:"pics"` 32 | } `json:"actArchiveResData"` 33 | ActArchiveData struct { 34 | Components orderedmap.OrderedMap[string, JsonComponent] `json:"components"` 35 | } `json:"actArchiveData"` 36 | } 37 | 38 | func (p *Parser) ParseStoryReviewMetaTable() (*JsonStoryReviewMetaTable, error) { 39 | jsonPath := filepath.Join(p.Root, p.Prefix, "gamedata/excel/story_review_meta_table.json") 40 | jsonData, err := os.ReadFile(jsonPath) 41 | if err != nil { 42 | return nil, errors.WithStack(err) 43 | } 44 | 45 | var obj JsonStoryReviewMetaTable 46 | err = json.Unmarshal(jsonData, &obj) 47 | if err != nil { 48 | return nil, errors.WithStack(err) 49 | } 50 | 51 | return &obj, nil 52 | } 53 | -------------------------------------------------------------------------------- /internal/app/artext/ext_character.go: -------------------------------------------------------------------------------- 1 | package artext 2 | 3 | import ( 4 | "fmt" 5 | "github.com/flandiayingman/arkwaifu/internal/app/art" 6 | "github.com/gofiber/fiber/v2" 7 | "github.com/pkg/errors" 8 | "regexp" 9 | "strconv" 10 | ) 11 | 12 | func parseCharacterID(characterID string) (base string, body int, face int, err error) { 13 | // Regex Pattern like: {BASE}#{FACE}${BODY} 14 | pattern := regexp.MustCompile(`(\w+)#(\d+)\$(\d+)`) 15 | matches := pattern.FindStringSubmatch(characterID) 16 | if matches == nil { 17 | return "", 0, 0, errors.Errorf("invalid character ID %s", characterID) 18 | } 19 | 20 | base = matches[1] 21 | face, err = strconv.Atoi(matches[2]) 22 | body, err = strconv.Atoi(matches[3]) 23 | 24 | if err != nil { 25 | return "", 0, 0, errors.Errorf("invalid character ID %s: %w", characterID, err) 26 | } 27 | return base, body, face, nil 28 | } 29 | 30 | // GetSiblingsOfCharacterArt gets the sibling characters of the specified character. 31 | // 32 | // Sibling Characters: the characters that have the same base name as the specified character. 33 | func (s *Service) GetSiblingsOfCharacterArt(characterID string) (siblings []*art.Art, err error) { 34 | base, _, _, err := parseCharacterID(characterID) 35 | if err != nil { 36 | return nil, err 37 | } 38 | return s.art.SelectArtsByIDLike(fmt.Sprintf("%s%%", base)) 39 | } 40 | 41 | func (c *Controller) GetSiblingsOfCharacterArt(ctx *fiber.Ctx) error { 42 | characterID := ctx.Params("id") 43 | siblings, err := c.service.GetSiblingsOfCharacterArt(characterID) 44 | if err != nil { 45 | return err 46 | } 47 | return ctx.JSON(siblings) 48 | } 49 | -------------------------------------------------------------------------------- /.github/workflows/docker-image-service.yml: -------------------------------------------------------------------------------- 1 | name: "Docker Image CI: service" 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | tags: 7 | - "v**" 8 | pull_request: 9 | branches: [ master ] 10 | 11 | jobs: 12 | 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Set up Docker Buildx 19 | uses: docker/setup-buildx-action@v1 20 | 21 | - name: Docker Login 22 | uses: docker/login-action@v1.14.1 23 | with: 24 | # Server address of Docker registry. If not set then will default to Docker Hub 25 | registry: "ghcr.io" 26 | # Username used to log against the Docker registry 27 | username: ${{ github.actor }} 28 | # Password or personal access token used to log against the Docker registry 29 | password: ${{ secrets.GITHUB_TOKEN }} 30 | 31 | - name: Build and push Docker images 32 | uses: docker/build-push-action@v2.9.0 33 | with: 34 | # List of external cache sources for buildx (e.g., user/app:cache, type=local,src=path/to/dir) 35 | cache-from: type=gha 36 | # List of cache export destinations for buildx (e.g., user/app:cache, type=local,dest=path/to/dir) 37 | cache-to: type=gha,mode=max 38 | # Path to the Dockerfile 39 | file: "build/service.Dockerfile" 40 | # Push is a shorthand for --output=type=registry 41 | push: ${{ startsWith(github.ref, 'refs/tags/') }} 42 | # List of tags 43 | tags: | 44 | ghcr.io/flandiayingman/arkwaifu/service:latest 45 | ghcr.io/flandiayingman/arkwaifu/service:${{ github.ref_name }} 46 | -------------------------------------------------------------------------------- /.github/workflows/docker-image-updateloop.yml: -------------------------------------------------------------------------------- 1 | name: "Docker Image CI: updateloop" 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | tags: 7 | - "v**" 8 | pull_request: 9 | branches: [ master ] 10 | 11 | jobs: 12 | 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Set up Docker Buildx 19 | uses: docker/setup-buildx-action@v1 20 | 21 | - name: Docker Login 22 | uses: docker/login-action@v1.14.1 23 | with: 24 | # Server address of Docker registry. If not set then will default to Docker Hub 25 | registry: "ghcr.io" 26 | # Username used to log against the Docker registry 27 | username: ${{ github.actor }} 28 | # Password or personal access token used to log against the Docker registry 29 | password: ${{ secrets.GITHUB_TOKEN }} 30 | 31 | - name: Build and push Docker images 32 | uses: docker/build-push-action@v2.9.0 33 | with: 34 | # List of external cache sources for buildx (e.g., user/app:cache, type=local,src=path/to/dir) 35 | cache-from: type=gha 36 | # List of cache export destinations for buildx (e.g., user/app:cache, type=local,dest=path/to/dir) 37 | cache-to: type=gha,mode=max 38 | # Path to the Dockerfile 39 | file: "build/updateloop.Dockerfile" 40 | # Push is a shorthand for --output=type=registry 41 | push: ${{ startsWith(github.ref, 'refs/tags/') }} 42 | # List of tags 43 | tags: | 44 | ghcr.io/flandiayingman/arkwaifu/updateloop:latest 45 | ghcr.io/flandiayingman/arkwaifu/updateloop:${{ github.ref_name }} 46 | -------------------------------------------------------------------------------- /internal/app/art/art_variant.go: -------------------------------------------------------------------------------- 1 | package art 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type Variant struct { 8 | ArtID string `gorm:"primaryKey;type:text COLLATE numeric;check:art_id=lower(art_id)" json:"artID"` 9 | Variation Variation `gorm:"primaryKey;type:art_variation" json:"variation,omitempty"` 10 | 11 | ContentPresent bool `gorm:"" json:"contentPresent"` 12 | ContentWidth *int `gorm:"" json:"contentWidth,omitempty"` 13 | ContentHeight *int `gorm:"" json:"contentHeight,omitempty"` 14 | } 15 | 16 | func NewVariant(id string, variation Variation) *Variant { 17 | return &Variant{ 18 | ArtID: id, 19 | Variation: variation, 20 | ContentPresent: false, 21 | ContentWidth: nil, 22 | ContentHeight: nil, 23 | } 24 | } 25 | 26 | func (v *Variant) String() string { 27 | return fmt.Sprintf("%s/%s", v.Variation, v.ArtID) 28 | } 29 | 30 | type Variation string 31 | 32 | const ( 33 | VariationOrigin Variation = "origin" 34 | VariationRealEsrganX4Plus Variation = "real-esrgan(realesrgan-x4plus)" 35 | VariationRealEsrganX4PlusAnime Variation = "real-esrgan(realesrgan-x4plus-anime)" 36 | VariationThumbnail Variation = "thumbnail" 37 | ) 38 | 39 | func ParseVariation(str string) (Variation, error) { 40 | variation := Variation(str) 41 | switch variation { 42 | case 43 | VariationOrigin, 44 | VariationRealEsrganX4Plus, 45 | VariationRealEsrganX4PlusAnime, 46 | VariationThumbnail: 47 | return variation, nil 48 | default: 49 | return "", fmt.Errorf("string %q is not a variation", str) 50 | } 51 | } 52 | 53 | func (v *Variation) UnmarshalJSON(bytes []byte) error { 54 | variation, err := ParseVariation(string(bytes)) 55 | if err != nil { 56 | return err 57 | } 58 | *v = variation 59 | return nil 60 | } 61 | -------------------------------------------------------------------------------- /internal/pkg/ark/server.go: -------------------------------------------------------------------------------- 1 | package ark 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "strings" 6 | ) 7 | 8 | // Server represents different server of the game. E.g., CN and EN. 9 | type Server = string 10 | 11 | const ( 12 | // CnServer stands for China Server 13 | CnServer Server = "CN" 14 | 15 | // EnServer stands for English Server 16 | EnServer Server = "EN" 17 | 18 | // JpServer stands for Japan Server 19 | JpServer Server = "JP" 20 | 21 | // KrServer stands for Korea Server 22 | KrServer Server = "KR" 23 | 24 | // TwServer stands for Taiwan Server 25 | TwServer Server = "TW" 26 | ) 27 | 28 | var ( 29 | Servers = []Server{CnServer, EnServer, JpServer, KrServer, TwServer} 30 | ) 31 | 32 | func ParseServer(s string) (Server, error) { 33 | EqualFolds := func(s string, others ...string) bool { 34 | for _, other := range others { 35 | if strings.EqualFold(s, other) { 36 | return true 37 | } 38 | } 39 | return false 40 | } 41 | if EqualFolds(s, "CN", "zh-CN") { 42 | return CnServer, nil 43 | } 44 | if EqualFolds(s, "EN", "en-US", "en-GB") { 45 | return EnServer, nil 46 | } 47 | if EqualFolds(s, "JP", "ja-JP") { 48 | return JpServer, nil 49 | } 50 | if EqualFolds(s, "KR", "ko-KR") { 51 | return KrServer, nil 52 | } 53 | if EqualFolds(s, "TW", "zh-TW") { 54 | return TwServer, nil 55 | } 56 | return "", errors.Errorf("unknown server: %v", s) 57 | } 58 | 59 | func MustParseServer(s string) Server { 60 | server, err := ParseServer(s) 61 | if err != nil { 62 | panic(err) 63 | } 64 | return server 65 | } 66 | 67 | func LanguageCodeUnderscore(s Server) string { 68 | switch s { 69 | case CnServer: 70 | return "zh_CN" 71 | case EnServer: 72 | return "en_US" 73 | case JpServer: 74 | return "ja_JP" 75 | case KrServer: 76 | return "ko_KR" 77 | case TwServer: 78 | return "zh_TW" 79 | } 80 | return "" 81 | } 82 | -------------------------------------------------------------------------------- /internal/app/infra/gorm.go: -------------------------------------------------------------------------------- 1 | package infra 2 | 3 | import ( 4 | "fmt" 5 | "gorm.io/driver/postgres" 6 | "gorm.io/gorm" 7 | "gorm.io/gorm/logger" 8 | "log" 9 | "os" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | type Gorm struct { 15 | *gorm.DB 16 | } 17 | 18 | func ProvideGorm(config *Config) (*Gorm, error) { 19 | dsn := config.PostgresDSN 20 | 21 | l := logger.New(log.New(os.Stdout, "\r\n", log.LstdFlags), logger.Config{ 22 | SlowThreshold: 1 * time.Second, 23 | LogLevel: logger.Warn, 24 | IgnoreRecordNotFoundError: false, 25 | Colorful: true, 26 | }) 27 | 28 | db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{ 29 | Logger: l, 30 | }) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | return &Gorm{db}, err 36 | } 37 | 38 | func (db *Gorm) CreateCollateNumeric() error { 39 | // The collation is copied from Postgres Docs 24.2.2.3.2. ICU Collations 40 | // (https://www.postgresql.org/docs/current/collation.html) and 'IF NOT EXISTS' is inserted. 41 | return db.Exec(`CREATE COLLATION IF NOT EXISTS numeric (provider = icu, locale = 'en-u-kn-true');`).Error 42 | } 43 | 44 | func (db *Gorm) CreateEnum(name string, values ...string) error { 45 | // Surround enum values with double quotes, since it is an identifier. 46 | name = fmt.Sprintf("\"%s\"", name) 47 | for i, value := range values { 48 | // Surround enum values with single quotes, since they are strings 49 | values[i] = fmt.Sprintf("'%s'", value) 50 | } 51 | // Use fmt.Sprintf because a possible BUG occurs when I tried using parameterized query. 52 | // However, this function is called only by static arguments, so it is considered safe. 53 | return db.Exec(fmt.Sprintf( 54 | `DO $$ BEGIN CREATE TYPE %s AS ENUM (%s); EXCEPTION WHEN duplicate_object THEN null; END $$;`, 55 | name, 56 | strings.Join(values, ","), 57 | )).Error 58 | } 59 | -------------------------------------------------------------------------------- /internal/pkg/util/pathutil/parent.go: -------------------------------------------------------------------------------- 1 | package pathutil 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "strings" 7 | ) 8 | 9 | // ChangeParent changes the parent of srcPath from srcDir to dstDir, and returns the dstPath. 10 | // If srcPath is not a child of srcDir, it returns an error. 11 | // 12 | // For example 13 | // ChangeParent("/home/user/dir/file.txt", "/home", "/root") // -> "/root/user/dir/file.txt" 14 | // ChangeParent("/home/user/file.txt", "/home/user", "/home/user/dir") // -> "/home/user/dir/file.txt" 15 | func ChangeParent(srcPath string, srcDir string, dstDir string) (dstPath string, err error) { 16 | relativePath, err := filepath.Rel(srcDir, srcPath) 17 | if err != nil { 18 | err = fmt.Errorf("failed to make relative path: %w", err) 19 | return 20 | } 21 | dstPath = filepath.Join(dstDir, relativePath) 22 | return 23 | } 24 | 25 | // MustChangeParent calls ChangeParent and panics if an error occurs. 26 | // It is safe to use MustChangeParent in, for example, os.WalkDir. 27 | func MustChangeParent(srcPath string, srcDir string, dstDir string) string { 28 | dstPath, err := ChangeParent(srcPath, srcDir, dstDir) 29 | if err != nil { 30 | panic(err) 31 | } 32 | return dstPath 33 | } 34 | 35 | const slash = "/" 36 | 37 | func Splits(path string) (parts []string) { 38 | path = filepath.ToSlash(path) 39 | if filepath.IsAbs(path) && filepath.VolumeName(path) != "" { 40 | volume := filepath.VolumeName(path) + slash 41 | parts = append(parts, volume) 42 | path = strings.TrimPrefix(path, volume) 43 | } else if filepath.IsAbs(path) { 44 | root := slash 45 | parts = append(parts, root) 46 | path = strings.TrimPrefix(path, root) 47 | } else if filepath.VolumeName(path) != "" { 48 | volume := filepath.VolumeName(path) 49 | parts = append(parts, volume) 50 | path = strings.TrimPrefix(path, volume) 51 | } 52 | parts = append(parts, strings.Split(path, slash)...) 53 | return 54 | } 55 | -------------------------------------------------------------------------------- /internal/pkg/arkdata/unzip.go: -------------------------------------------------------------------------------- 1 | package arkdata 2 | 3 | import ( 4 | "context" 5 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 6 | "github.com/flandiayingman/arkwaifu/internal/pkg/cols" 7 | "github.com/flandiayingman/arkwaifu/internal/pkg/util/cloze" 8 | "github.com/flandiayingman/arkwaifu/internal/pkg/util/fileutil" 9 | "github.com/flandiayingman/arkwaifu/internal/pkg/util/pathutil" 10 | "github.com/mholt/archiver/v4" 11 | "github.com/pkg/errors" 12 | "os" 13 | "path" 14 | "path/filepath" 15 | ) 16 | 17 | func unzip(ctx context.Context, zipball string, patterns []string, server ark.Server) (string, error) { 18 | temp, err := os.MkdirTemp("", "arkdata_unzip") 19 | if err != nil { 20 | return "", errors.WithStack(err) 21 | } 22 | 23 | // Since the zipball contains a directory directly named by the stem of the zipball, 24 | // the prefix is gotten for generating the file path inside the zipball. 25 | patterns = cols.Map(patterns, func(pattern string) string { 26 | return path.Join(pathutil.Stem(zipball), ark.LanguageCodeUnderscore(server), pattern) 27 | }) 28 | 29 | reader, err := os.Open(zipball) 30 | if err != nil { 31 | return "", errors.WithStack(err) 32 | } 33 | defer cloze.IgnoreErr(reader) 34 | 35 | zip := archiver.Zip{} 36 | return temp, zip.Extract(ctx, reader, nil, func(ctx context.Context, file archiver.File) error { 37 | if file.IsDir() { 38 | return nil 39 | } 40 | 41 | match, err := pathutil.MatchAny(patterns, file.NameInArchive) 42 | if err != nil { 43 | return errors.WithStack(err) 44 | } 45 | if !match { 46 | return nil 47 | } 48 | 49 | fileName := path.Join(pathutil.Splits(file.NameInArchive)[2:]...) 50 | filePath := filepath.Join(temp, DefaultPrefix, fileName) 51 | 52 | reader, err := file.Open() 53 | if err != nil { 54 | return errors.WithStack(err) 55 | } 56 | 57 | err = fileutil.MkFileFromReader(filePath, reader) 58 | if err != nil { 59 | return errors.WithStack(err) 60 | } 61 | 62 | return nil 63 | }) 64 | } 65 | -------------------------------------------------------------------------------- /deploy/docker-compose.yml: -------------------------------------------------------------------------------- 1 | ### The Docker Compose file for deploying Arkwaifu on the official server. 2 | ### Others may modify the file to fit their condition. 3 | 4 | services: 5 | 6 | reverse_proxy: 7 | depends_on: 8 | - frontend 9 | - backend 10 | build: ./reverse_proxy 11 | ports: 12 | - 80:80 13 | environment: 14 | - FRONTEND_ADDRESS=frontend:80 15 | - BACKEND_ADDRESS=backend:7080 16 | 17 | frontend: 18 | image: ghcr.io/flandiayingman/arkwaifu-frontend:latest 19 | 20 | backend: 21 | depends_on: 22 | database: 23 | condition: service_healthy 24 | image: ghcr.io/flandiayingman/arkwaifu/service:latest 25 | volumes: 26 | - ./arkwaifu-root:/var/arkwaifu-root 27 | environment: 28 | - POSTGRES_DSN=postgres://postgres@database 29 | - ROOT=/var/arkwaifu-root 30 | restart: always 31 | 32 | updateloop: 33 | depends_on: 34 | database: 35 | condition: service_healthy 36 | image: ghcr.io/flandiayingman/arkwaifu/updateloop:latest 37 | volumes: 38 | - ./arkwaifu-root:/var/arkwaifu-root 39 | environment: 40 | - POSTGRES_DSN=postgres://postgres@database 41 | - ROOT=/var/arkwaifu-root 42 | restart: always 43 | 44 | database: 45 | image: postgres:15.3-alpine 46 | ports: 47 | # Only expose the port on localhost. That should prevent unauthorized accesses. 48 | # This is for administrators to manage the database, not for other containers to access. 49 | - 127.0.0.1:5432:5432 50 | environment: 51 | # Since the database is not exposed to public, it is considered safe to trust. 52 | - POSTGRES_HOST_AUTH_METHOD=trust 53 | # Persist the data. 54 | volumes: 55 | - ./arkwaifu-root/postgres-data:/var/lib/postgresql/data 56 | # We do not know when the Postgres is ready for listening connections, so we use pg_isready utility to check it. 57 | healthcheck: 58 | test: [ "CMD-SHELL", "pg_isready -U postgres" ] 59 | interval: 3s 60 | timeout: 3s 61 | retries: 10 -------------------------------------------------------------------------------- /internal/pkg/arkscanner/scanner_picture.go: -------------------------------------------------------------------------------- 1 | package arkscanner 2 | 3 | import ( 4 | "fmt" 5 | "github.com/flandiayingman/arkwaifu/internal/pkg/util/pathutil" 6 | "github.com/pkg/errors" 7 | "path" 8 | "path/filepath" 9 | ) 10 | 11 | type PictureArt struct { 12 | ID string 13 | Name string 14 | Kind string 15 | } 16 | 17 | func (a *PictureArt) Path() string { 18 | subdirectory := subdirectoryOfCategory(a.Kind) 19 | return path.Join("assets/torappu/dynamicassets", "avg", subdirectory, a.Name) 20 | } 21 | 22 | func subdirectoryOfCategory(category string) string { 23 | switch category { 24 | case "image": 25 | return "images" 26 | case "background": 27 | return "backgrounds" 28 | case "item": 29 | return "items" 30 | default: 31 | panic(fmt.Sprintf("unrecognized category %s", category)) 32 | } 33 | } 34 | 35 | const PictureArtPath = "assets/torappu/dynamicassets/avg" 36 | 37 | func (scanner *Scanner) ScanForPictureArts() ([]*PictureArt, error) { 38 | baseDir := filepath.Join(scanner.Root, PictureArtPath) 39 | imageArts, err := scanner.scanPictures(baseDir, "images", "image") 40 | if err != nil { 41 | return nil, errors.WithStack(err) 42 | } 43 | backgroundArts, err := scanner.scanPictures(baseDir, "backgrounds", "background") 44 | if err != nil { 45 | return nil, errors.WithStack(err) 46 | } 47 | itemArts, err := scanner.scanPictures(baseDir, "items", "item") 48 | if err != nil { 49 | return nil, errors.WithStack(err) 50 | } 51 | 52 | var arts []*PictureArt 53 | arts = append(arts, imageArts...) 54 | arts = append(arts, backgroundArts...) 55 | arts = append(arts, itemArts...) 56 | 57 | return arts, nil 58 | } 59 | 60 | func (scanner *Scanner) scanPictures(base string, sub string, kind string) ([]*PictureArt, error) { 61 | files, err := filepath.Glob(filepath.Join(base, sub, "*.png")) 62 | if err != nil { 63 | return nil, errors.WithStack(err) 64 | } 65 | 66 | var pictures []*PictureArt 67 | for _, file := range files { 68 | pictures = append(pictures, &PictureArt{ 69 | ID: pathutil.Stem(file), 70 | Name: filepath.Base(file), 71 | Kind: kind, 72 | }) 73 | } 74 | 75 | return pictures, nil 76 | } 77 | -------------------------------------------------------------------------------- /internal/app/gallery/controller.go: -------------------------------------------------------------------------------- 1 | package gallery 2 | 3 | import ( 4 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 5 | "github.com/gofiber/fiber/v2" 6 | ) 7 | 8 | type controller struct { 9 | service *Service 10 | } 11 | 12 | func newController(service *Service) *controller { 13 | return &controller{service} 14 | } 15 | func registerController(c *controller, router fiber.Router) { 16 | if router == nil { 17 | return 18 | } 19 | router.Get(":server/galleries", c.ListGalleries) 20 | router.Get(":server/galleries/:id", c.GetGalleryByID) 21 | router.Get(":server/gallery-arts", c.ListGalleryArt) 22 | router.Get(":server/gallery-arts/:id", c.GetGalleryArtByID) 23 | } 24 | 25 | func (c *controller) ListGalleries(ctx *fiber.Ctx) error { 26 | server, err := ark.ParseServer(ctx.Params("server")) 27 | if err != nil { 28 | return fiber.ErrBadRequest 29 | } 30 | 31 | galleries, err := c.service.ListGalleries(server) 32 | if err != nil { 33 | return err 34 | } 35 | 36 | return ctx.JSON(galleries) 37 | } 38 | func (c *controller) GetGalleryByID(ctx *fiber.Ctx) error { 39 | server, err := ark.ParseServer(ctx.Params("server")) 40 | if err != nil { 41 | return fiber.ErrBadRequest 42 | } 43 | id := ctx.Params("id") 44 | if id == "" { 45 | return fiber.ErrBadRequest 46 | } 47 | 48 | gallery, err := c.service.GetGalleryByID(server, id) 49 | 50 | return ctx.JSON(gallery) 51 | } 52 | 53 | func (c *controller) ListGalleryArt(ctx *fiber.Ctx) error { 54 | server, err := ark.ParseServer(ctx.Params("server")) 55 | if err != nil { 56 | return fiber.ErrBadRequest 57 | } 58 | 59 | galleryArts, err := c.service.ListGalleryArts(server) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | return ctx.JSON(galleryArts) 65 | } 66 | func (c *controller) GetGalleryArtByID(ctx *fiber.Ctx) error { 67 | server, err := ark.ParseServer(ctx.Params("server")) 68 | if err != nil { 69 | return fiber.ErrBadRequest 70 | } 71 | id := ctx.Params("id") 72 | if id == "" { 73 | return fiber.ErrBadRequest 74 | } 75 | 76 | galleryArt, err := c.service.GetGalleryArtByID(server, id) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | return ctx.JSON(galleryArt) 82 | } 83 | -------------------------------------------------------------------------------- /internal/pkg/arkassets/unzip.go: -------------------------------------------------------------------------------- 1 | package arkassets 2 | 3 | import ( 4 | "context" 5 | "github.com/mholt/archiver/v4" 6 | "github.com/pkg/errors" 7 | "github.com/rs/zerolog/log" 8 | 9 | "io" 10 | "io/fs" 11 | "os" 12 | "path" 13 | "path/filepath" 14 | ) 15 | 16 | func unzip(ctx context.Context, src string) (string, error) { 17 | tempDir, err := os.MkdirTemp("", "arkassets_unzip-*") 18 | if err != nil { 19 | return "", errors.WithStack(err) 20 | } 21 | 22 | return tempDir, filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { 23 | if err != nil { 24 | return errors.WithStack(err) 25 | } 26 | if d.IsDir() { 27 | return nil 28 | } 29 | if err := ctx.Err(); err != nil { 30 | return errors.WithStack(err) 31 | } 32 | 33 | err = unzipFile(ctx, path, tempDir) 34 | if err != nil { 35 | return errors.WithStack(err) 36 | } 37 | return nil 38 | }) 39 | } 40 | 41 | func unzipFile(ctx context.Context, src string, dst string) error { 42 | zip := archiver.Zip{} 43 | 44 | srcFile, err := os.Open(src) 45 | if err != nil { 46 | return errors.WithStack(err) 47 | } 48 | defer srcFile.Close() 49 | 50 | err = zip.Extract(ctx, srcFile, nil, func(ctx context.Context, f archiver.File) error { 51 | if err := ctx.Err(); err != nil { 52 | return errors.WithStack(err) 53 | } 54 | 55 | compFile, err := f.Open() 56 | if err != nil { 57 | return errors.WithStack(err) 58 | } 59 | defer compFile.Close() 60 | 61 | compFileName := f.NameInArchive 62 | 63 | src := filepath.ToSlash(filepath.Clean(src)) 64 | dst := filepath.ToSlash(filepath.Clean(path.Join(dst, compFileName))) 65 | err = os.MkdirAll(filepath.Dir(dst), os.ModePerm) 66 | if err != nil { 67 | return errors.WithStack(err) 68 | } 69 | 70 | dstFile, err := os.Create(dst) 71 | if err != nil { 72 | return errors.WithStack(err) 73 | } 74 | defer dstFile.Close() 75 | 76 | _, err = io.Copy(dstFile, compFile) 77 | if err != nil { 78 | return errors.WithStack(err) 79 | } 80 | 81 | log.Info(). 82 | Str("src", src). 83 | Str("dst", dst). 84 | Msg("Unzipped resource src to dst.") 85 | 86 | return nil 87 | }) 88 | return errors.WithStack(err) 89 | } 90 | -------------------------------------------------------------------------------- /internal/app/gallery/repository.go: -------------------------------------------------------------------------------- 1 | package gallery 2 | 3 | import ( 4 | "github.com/flandiayingman/arkwaifu/internal/app/infra" 5 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 6 | "gorm.io/gorm" 7 | "gorm.io/gorm/clause" 8 | ) 9 | 10 | type repository struct { 11 | db *infra.Gorm 12 | } 13 | 14 | func newRepository(db *infra.Gorm) (*repository, error) { 15 | r := repository{db: db} 16 | err := r.init() 17 | if err != nil { 18 | return nil, err 19 | } 20 | return &r, nil 21 | } 22 | 23 | func (r *repository) init() (err error) { 24 | err = r.db.CreateCollateNumeric() 25 | if err != nil { 26 | return err 27 | } 28 | err = r.db.CreateEnum("game_server", 29 | ark.CnServer, 30 | ark.EnServer, 31 | ark.JpServer, 32 | ark.KrServer, 33 | ark.TwServer, 34 | ) 35 | if err != nil { 36 | return err 37 | } 38 | err = r.db.AutoMigrate(&Gallery{}) 39 | if err != nil { 40 | return err 41 | } 42 | err = r.db.AutoMigrate(&Art{}) 43 | if err != nil { 44 | return err 45 | } 46 | return nil 47 | } 48 | 49 | func (r *repository) ListGalleries(server ark.Server) ([]Gallery, error) { 50 | var galleries []Gallery 51 | return galleries, r.db. 52 | Preload("Arts", func(db *gorm.DB) *gorm.DB { return db.Order("gallery_arts.sort_id") }). 53 | Order("id"). 54 | Find(&galleries, "server = ?", server). 55 | Error 56 | } 57 | 58 | func (r *repository) GetGalleryByID(server ark.Server, id string) (*Gallery, error) { 59 | var galleries Gallery 60 | return &galleries, r.db. 61 | Preload("Arts", func(db *gorm.DB) *gorm.DB { return db.Order("gallery_arts.sort_id") }). 62 | Take(&galleries, "(server, id) = (?, ?)", server, id). 63 | Error 64 | } 65 | 66 | func (r *repository) ListGalleryArts(server ark.Server) ([]Art, error) { 67 | var arts []Art 68 | return arts, r.db. 69 | Order("gallery_id, sort_id, id"). 70 | Find(&arts, "server = ?", server). 71 | Error 72 | } 73 | 74 | func (r *repository) GetGalleryArtByID(server ark.Server, id string) (*Art, error) { 75 | var art Art 76 | return &art, r.db. 77 | Take(&art, "(server, id) = (?, ?)", server, id). 78 | Error 79 | } 80 | 81 | func (r *repository) Put(g []Gallery) error { 82 | return r.db. 83 | Clauses(clause.OnConflict{UpdateAll: true}). 84 | Create(&g). 85 | Error 86 | } 87 | -------------------------------------------------------------------------------- /internal/app/updateloop/update_art_thumbnail.go: -------------------------------------------------------------------------------- 1 | package updateloop 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "github.com/chai2010/webp" 7 | "github.com/disintegration/imaging" 8 | "github.com/flandiayingman/arkwaifu/internal/app/art" 9 | "github.com/rs/zerolog/log" 10 | "golang.org/x/sync/errgroup" 11 | "image" 12 | "runtime" 13 | "time" 14 | ) 15 | 16 | func (s *Service) attemptUpdateArtThumbnails(ctx context.Context) { 17 | arts, err := s.artService.SelectArtsWhoseVariantAbsent("thumbnail") 18 | if err != nil { 19 | log.Error(). 20 | Err(err). 21 | Caller(). 22 | Msg("Failed to select arts.") 23 | } 24 | if len(arts) > 0 { 25 | err := s.updateArtsThumbnail(ctx, arts) 26 | if err != nil { 27 | log.Error(). 28 | Err(err). 29 | Caller(). 30 | Msg("Failed to update arts thumbnail.") 31 | } 32 | } 33 | } 34 | 35 | func (s *Service) updateArtsThumbnail(ctx context.Context, arts []*art.Art) error { 36 | log.Info().Msgf("Begin updating arts thumbnail, using %d goroutines", runtime.NumCPU()) 37 | begin := time.Now() 38 | 39 | ctx, cancel := context.WithCancel(ctx) 40 | defer cancel() 41 | 42 | eg, ctx := errgroup.WithContext(ctx) 43 | eg.SetLimit(runtime.NumCPU()) 44 | 45 | for _, art := range arts { 46 | art := art 47 | eg.Go(func() error { 48 | return s.updateArtThumbnail(ctx, art) 49 | }) 50 | } 51 | 52 | err := eg.Wait() 53 | if err != nil { 54 | return err 55 | } 56 | 57 | log.Info().Msgf("End updating arts thumbnail, elapsed %v", time.Since(begin)) 58 | return nil 59 | } 60 | 61 | func (s *Service) updateArtThumbnail(ctx context.Context, a *art.Art) error { 62 | content, err := s.artService.TakeContent(a.ID, art.VariationOrigin) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | img, _, err := image.Decode(bytes.NewReader(content)) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | img = imaging.Fit(img, 360, 360*4, imaging.Lanczos) 73 | 74 | buf := bytes.Buffer{} 75 | err = webp.Encode(&buf, img, &webp.Options{ 76 | Lossless: false, 77 | Quality: 75, 78 | Exact: false, 79 | }) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | err = s.artService.UpsertVariants(art.NewVariant(a.ID, art.VariationThumbnail)) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | err = s.artService.StoreContent(a.ID, art.VariationThumbnail, buf.Bytes()) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | return nil 95 | } 96 | -------------------------------------------------------------------------------- /internal/pkg/arkparser/directive.go: -------------------------------------------------------------------------------- 1 | package arkparser 2 | 3 | import ( 4 | "fmt" 5 | "github.com/pkg/errors" 6 | "os" 7 | "path" 8 | "regexp" 9 | "strings" 10 | ) 11 | 12 | type Directive struct { 13 | Name string 14 | Params map[string]string 15 | } 16 | 17 | var DirectiveRegex = regexp.MustCompile(`\[(\w*)(?:\((.*)\))?]|\[name="(.*)"]`) 18 | 19 | func ParseDirectives(raw string) []Directive { 20 | directives := make([]Directive, 0) 21 | matches := DirectiveRegex.FindAllStringSubmatch(raw, -1) 22 | for _, match := range matches { 23 | if match[3] == "" { 24 | name := strings.ToLower(match[1]) 25 | params := ParseDirectiveParams(match[2]) 26 | directives = append(directives, Directive{ 27 | Name: name, 28 | Params: params, 29 | }) 30 | } else { 31 | name := "" 32 | params := make(map[string]string) 33 | params["name"] = match[3] 34 | directives = append(directives, Directive{ 35 | Name: name, 36 | Params: params, 37 | }) 38 | } 39 | } 40 | return directives 41 | } 42 | 43 | func ParseDirectiveParams(rawParams string) map[string]string { 44 | params := make(map[string]string) 45 | for _, rawParam := range splitTokens(rawParams) { 46 | key, value, _ := strings.Cut(rawParam, "=") 47 | key = strings.ToLower(strings.TrimSpace(key)) 48 | value = strings.Trim(strings.TrimSpace(value), `"`) 49 | params[key] = value 50 | } 51 | return params 52 | } 53 | 54 | // splitTokens splits a string separated by comma which is not in double quotes. 55 | func splitTokens(s string) []string { 56 | if len(s) <= 0 { 57 | return nil 58 | } 59 | tokens := make([]string, 0) 60 | pos := 0 61 | inQuotes := false 62 | for i, char := range s { 63 | if char == '"' { 64 | inQuotes = !inQuotes 65 | } else if char == ',' && !inQuotes { 66 | tokens = append(tokens, s[pos:i]) 67 | pos = i + 1 68 | } 69 | } 70 | lastToken := s[pos:] 71 | if lastToken == "," { 72 | tokens = append(tokens, "") 73 | } else { 74 | tokens = append(tokens, lastToken) 75 | } 76 | return tokens 77 | } 78 | 79 | func (p *Parser) ParseStoryText(storyTextPath string) ([]Directive, error) { 80 | storyTextPath = fmt.Sprintf("%s.txt", storyTextPath) 81 | storyTextPath = path.Join(p.Root, p.Prefix, "gamedata/story", storyTextPath) 82 | storyTextData, err := os.ReadFile(storyTextPath) 83 | if err != nil { 84 | return nil, errors.WithStack(err) 85 | } 86 | 87 | return ParseDirectives(string(storyTextData)), nil 88 | } 89 | -------------------------------------------------------------------------------- /internal/pkg/arkassets/fetch.go: -------------------------------------------------------------------------------- 1 | package arkassets 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/flandiayingman/arkwaifu/internal/pkg/cols" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/cavaliergopher/grab/v3" 11 | "github.com/pkg/errors" 12 | "github.com/rs/zerolog/log" 13 | ) 14 | 15 | const fetchWorkers = 16 16 | const retryWorkers = 1 17 | 18 | func fetch(ctx context.Context, infoList []Info) (string, error) { 19 | tempDir, err := os.MkdirTemp("", "arkassets_fetch-*") 20 | if err != nil { 21 | return "", errors.WithStack(err) 22 | } 23 | 24 | ctx, cancelCtx := context.WithCancel(ctx) 25 | defer cancelCtx() 26 | 27 | requests, err := cols.MapErr(infoList, func(i Info) (*grab.Request, error) { 28 | request, err := grab.NewRequest(tempDir, i.Url()) 29 | if err != nil { 30 | return nil, errors.WithStack(err) 31 | } 32 | request = request.WithContext(ctx) 33 | return request, nil 34 | }) 35 | responses := grab.DefaultClient.DoBatch(fetchWorkers, requests...) 36 | 37 | num := 0 38 | total := len(requests) 39 | requests = nil 40 | for response := range responses { 41 | srcFile := response.Request.URL().String() 42 | dstFile := filepath.ToSlash(filepath.Clean(response.Filename)) 43 | 44 | log := log.With(). 45 | Str("src", srcFile). 46 | Str("dst", dstFile). 47 | Int("num", num). 48 | Int("total", total). 49 | Str("progress", fmt.Sprintf("%.3f", float64(num)/float64(total))). 50 | Logger() 51 | 52 | err := response.Err() 53 | if err != nil { 54 | req := response.Request 55 | req = req.WithContext(ctx) 56 | requests = append(requests, req) 57 | log.Info().Msg("Failed to fetch resource; appending to retry queue") 58 | } else { 59 | log.Info().Msg("Fetched resource.") 60 | } 61 | num++ 62 | } 63 | 64 | responses = grab.DefaultClient.DoBatch(retryWorkers, requests...) 65 | 66 | num = 0 67 | total = len(requests) 68 | for response := range responses { 69 | srcFile := response.Request.URL().String() 70 | dstFile := filepath.ToSlash(filepath.Clean(response.Filename)) 71 | 72 | err := response.Err() 73 | if err != nil { 74 | return "", errors.Wrapf(err, "srcFile：%v; dstFile: %v", srcFile, dstFile) 75 | } 76 | log.Info(). 77 | Str("src", srcFile). 78 | Str("dst", dstFile). 79 | Int("num", num). 80 | Int("total", total). 81 | Str("progress", fmt.Sprintf("%.3f", float64(num)/float64(total))). 82 | Msg("Fetched resource in retry queue.") 83 | num++ 84 | } 85 | return tempDir, nil 86 | } 87 | -------------------------------------------------------------------------------- /internal/app/art/service.go: -------------------------------------------------------------------------------- 1 | package art 2 | 3 | import ( 4 | "errors" 5 | "github.com/flandiayingman/arkwaifu/internal/app/infra" 6 | "github.com/google/uuid" 7 | "strings" 8 | ) 9 | 10 | type Service struct { 11 | repo *repository 12 | 13 | users []infra.User 14 | } 15 | 16 | func newService(config *infra.Config, repo *repository) *Service { 17 | return &Service{ 18 | repo: repo, 19 | users: config.Users, 20 | } 21 | } 22 | 23 | var ( 24 | ErrNotFound = errors.New("the resource with the identifier(s) is not found") 25 | ) 26 | 27 | func (s *Service) SelectArts() ([]*Art, error) { 28 | return s.repo.SelectArts() 29 | } 30 | func (s *Service) SelectArtsByCategory(category Category) ([]*Art, error) { 31 | return s.repo.SelectArtsByCategory(string(category)) 32 | } 33 | func (s *Service) SelectArtsByIDs(ids []string) ([]*Art, error) { 34 | for i, id := range ids { 35 | ids[i] = strings.ToLower(id) 36 | } 37 | return s.repo.SelectArtsByIDs(ids) 38 | } 39 | func (s *Service) SelectArtsByIDLike(like string) ([]*Art, error) { 40 | return s.repo.SelectArtsByIDLike(like) 41 | } 42 | func (s *Service) SelectArt(id string) (*Art, error) { 43 | return s.repo.SelectArt(strings.ToLower(id)) 44 | } 45 | func (s *Service) SelectVariants(id string) ([]*Variant, error) { 46 | return s.repo.SelectVariants(strings.ToLower(id)) 47 | } 48 | func (s *Service) SelectVariant(id string, variation Variation) (*Variant, error) { 49 | return s.repo.SelectVariant(strings.ToLower(id), string(variation)) 50 | } 51 | 52 | func (s *Service) UpsertArts(arts ...*Art) error { 53 | for _, art := range arts { 54 | art.ID = strings.ToLower(art.ID) 55 | } 56 | return s.repo.UpsertArts(arts...) 57 | } 58 | func (s *Service) UpsertVariants(variants ...*Variant) error { 59 | for _, variant := range variants { 60 | variant.ArtID = strings.ToLower(variant.ArtID) 61 | } 62 | return s.repo.UpsertVariants(variants...) 63 | } 64 | 65 | func (s *Service) StoreContent(id string, variation Variation, content []byte) (err error) { 66 | return s.repo.StoreContent(strings.ToLower(id), string(variation), content) 67 | } 68 | func (s *Service) TakeContent(id string, variation Variation) (content []byte, err error) { 69 | return s.repo.TakeContent(strings.ToLower(id), string(variation)) 70 | } 71 | 72 | func (s *Service) SelectArtsWhoseVariantAbsent(variation Variation) ([]*Art, error) { 73 | return s.repo.SelectArtsWhereVariantAbsent(string(variation)) 74 | } 75 | 76 | func (s *Service) Authenticate(uuid uuid.UUID) *infra.User { 77 | for _, user := range s.users { 78 | if user.ID == uuid { 79 | return &user 80 | } 81 | } 82 | return nil 83 | } 84 | -------------------------------------------------------------------------------- /internal/pkg/arkassets/unpack.go: -------------------------------------------------------------------------------- 1 | package arkassets 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "github.com/flandiayingman/arkwaifu/internal/pkg/util/fileutil" 7 | "github.com/pkg/errors" 8 | "github.com/rs/zerolog/log" 9 | "os" 10 | "os/exec" 11 | "path/filepath" 12 | "strings" 13 | ) 14 | 15 | const ( 16 | extractorLocation = "./tools/extractor" 17 | ) 18 | 19 | func unpack(ctx context.Context, src string) (string, error) { 20 | tempDir, err := os.MkdirTemp("", "arkassets_unpack-*") 21 | if err != nil { 22 | return "", errors.WithStack(err) 23 | } 24 | 25 | err = findExtractor() 26 | if err != nil { 27 | return "", errors.WithStack(err) 28 | } 29 | 30 | srcAbs, err := filepath.Abs(src) 31 | if err != nil { 32 | return "", errors.WithStack(err) 33 | } 34 | dstAbs, err := filepath.Abs(tempDir) 35 | if err != nil { 36 | return "", errors.WithStack(err) 37 | } 38 | 39 | args := []string{"-u", "main.py", srcAbs, dstAbs} 40 | cmd := exec.CommandContext(ctx, "python", args...) 41 | cmd.Dir = extractorLocation 42 | 43 | stdout, err := cmd.StdoutPipe() 44 | if err != nil { 45 | return "", errors.WithStack(err) 46 | } 47 | scanner := bufio.NewScanner(stdout) 48 | go func(scanner *bufio.Scanner) { 49 | for scanner.Scan() { 50 | t := scanner.Text() 51 | log.Debug(). 52 | Str("output", t). 53 | Msg("Output from stdout of the extractor... ") 54 | split := strings.SplitN(t, "=>", 2) 55 | if len(split) != 2 { 56 | continue 57 | } 58 | srcFile := filepath.ToSlash(filepath.Clean(split[0])) 59 | dstFile := filepath.ToSlash(filepath.Clean(split[1])) 60 | log.Info(). 61 | Str("src", srcFile). 62 | Str("dst", dstFile). 63 | Msg("Unpacked the resource src to dst. ") 64 | } 65 | }(scanner) 66 | stderr, err := cmd.StderrPipe() 67 | if err != nil { 68 | return "", errors.WithStack(err) 69 | } 70 | errScanner := bufio.NewScanner(stderr) 71 | go func(errScanner *bufio.Scanner) { 72 | for errScanner.Scan() { 73 | t := errScanner.Text() 74 | log.Warn(). 75 | Str("output", t). 76 | Msg("Output from stderr of the extractor... ") 77 | } 78 | }(errScanner) 79 | 80 | err = cmd.Start() 81 | if err != nil { 82 | return "", errors.WithStack(err) 83 | } 84 | 85 | err = cmd.Wait() 86 | if err != nil { 87 | return "", errors.WithStack(err) 88 | } 89 | 90 | return tempDir, nil 91 | } 92 | 93 | func findExtractor() error { 94 | exists, err := fileutil.Exists(extractorLocation) 95 | if err != nil { 96 | return errors.WithStack(err) 97 | } 98 | if !exists { 99 | return errors.New("cannot find extractor") 100 | } 101 | return nil 102 | } 103 | -------------------------------------------------------------------------------- /internal/pkg/arkassets/raw.go: -------------------------------------------------------------------------------- 1 | package arkassets 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/pkg/errors" 7 | "net/http" 8 | ) 9 | 10 | // Version represents a raw response of "https://ak-conf.hypergryph.com/config/prod/official/Android/version". 11 | type Version struct { 12 | ResVersion string `json:"resVersion"` 13 | ClientVersion string `json:"clientVersion"` 14 | } 15 | 16 | // HotUpdateList represents a raw response of "https://ak.hycdn.cn/assetbundle/official/Android/assets/{resVersion}/hot_update_list.json" 17 | type HotUpdateList struct { 18 | FullPack FullPack `json:"fullPack"` 19 | VersionID string `json:"versionId"` 20 | AbInfos []AbInfo `json:"abInfos"` 21 | CountOfTypedRes int `json:"countOfTypedRes"` 22 | PackInfos []PackInfo `json:"packInfos"` 23 | } 24 | 25 | type FullPack struct { 26 | TotalSize int `json:"totalSize"` 27 | AbSize int `json:"abSize"` 28 | Type string `json:"type"` 29 | Cid int `json:"cid"` 30 | } 31 | 32 | type AbInfo struct { 33 | Name string `json:"name"` 34 | Hash string `json:"hash"` 35 | MD5 string `json:"md5"` 36 | TotalSize int `json:"totalSize"` 37 | AbSize int `json:"abSize"` 38 | Cid int `json:"cid"` 39 | Pid string `json:"pid,omitempty"` 40 | Type string `json:"type,omitempty"` 41 | } 42 | 43 | type PackInfo struct { 44 | Name string `json:"name"` 45 | Hash string `json:"hash"` 46 | MD5 string `json:"md5"` 47 | TotalSize int `json:"totalSize"` 48 | AbSize int `json:"abSize"` 49 | Cid int `json:"cid"` 50 | } 51 | 52 | // GetRawVersion gets the raw response from Arknights version API. 53 | func GetRawVersion() (Version, error) { 54 | resp, err := http.Get("https://ak-conf.hypergryph.com/config/prod/official/Android/version") 55 | if err != nil { 56 | return Version{}, errors.WithStack(err) 57 | } 58 | defer resp.Body.Close() 59 | 60 | arkVersion := Version{} 61 | err = json.NewDecoder(resp.Body).Decode(&arkVersion) 62 | if err != nil { 63 | return Version{}, errors.WithStack(err) 64 | } 65 | return arkVersion, nil 66 | } 67 | 68 | // GetRawResources gets the raw response of Arknights resource API with specified resource version. 69 | func GetRawResources(resVersion string) (HotUpdateList, error) { 70 | urlResourceList := GetResURL(resVersion, "hot_update_list.json") 71 | 72 | resp, err := http.Get(urlResourceList) 73 | if err != nil { 74 | return HotUpdateList{}, errors.WithStack(err) 75 | } 76 | defer resp.Body.Close() 77 | 78 | resourcesList := HotUpdateList{} 79 | err = json.NewDecoder(resp.Body).Decode(&resourcesList) 80 | if err != nil { 81 | return HotUpdateList{}, errors.WithStack(err) 82 | } 83 | return resourcesList, nil 84 | } 85 | 86 | // GetResURL gets the URL to download the specified asset with specified resource version. 87 | func GetResURL(resVersion string, res string) string { 88 | return fmt.Sprintf("https://ak.hycdn.cn/assetbundle/official/Android/assets/%v/%v", resVersion, res) 89 | } 90 | -------------------------------------------------------------------------------- /internal/app/story/controller.go: -------------------------------------------------------------------------------- 1 | package story 2 | 3 | import ( 4 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 5 | "github.com/gofiber/fiber/v2" 6 | ) 7 | 8 | type controller struct { 9 | service *Service 10 | } 11 | 12 | func newController(service *Service) *controller { 13 | return &controller{service} 14 | } 15 | func registerController(c *controller, router fiber.Router) { 16 | if router == nil { 17 | return 18 | } 19 | router.Get(":server/story-groups", c.GetGroups) 20 | router.Get(":server/story-groups/:id", c.GetGroup) 21 | router.Get(":server/stories", c.GetStories) 22 | router.Get(":server/stories/:id", c.GetStory) 23 | 24 | router.Get(":server/aggregated-picture-arts/:id", c.GetAggregatedPictureArt) 25 | router.Get(":server/aggregated-character-arts/:id", c.GetAggregatedCharacterArt) 26 | } 27 | 28 | func (c *controller) GetGroups(ctx *fiber.Ctx) error { 29 | server, err := ark.ParseServer(ctx.Params("server")) 30 | if err != nil { 31 | return fiber.ErrBadRequest 32 | } 33 | 34 | queryType := ctx.Query("type") 35 | 36 | queryFilter := GroupFilter{ 37 | Type: queryType, 38 | } 39 | 40 | groups, err := c.service.GetStoryGroups(server, queryFilter) 41 | if err != nil { 42 | return err 43 | } 44 | return ctx.JSON(groups) 45 | } 46 | func (c *controller) GetGroup(ctx *fiber.Ctx) error { 47 | server, err := ark.ParseServer(ctx.Params("server")) 48 | if err != nil { 49 | return fiber.ErrBadRequest 50 | } 51 | id := ctx.Params("id") 52 | if id == "" { 53 | return fiber.ErrBadRequest 54 | } 55 | 56 | group, err := c.service.GetStoryGroup(server, id) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | return ctx.JSON(group) 62 | } 63 | 64 | func (c *controller) GetStories(ctx *fiber.Ctx) error { 65 | server, err := ark.ParseServer(ctx.Params("server")) 66 | if err != nil { 67 | return fiber.ErrBadRequest 68 | } 69 | 70 | stories, err := c.service.GetStories(server) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | return ctx.JSON(stories) 76 | } 77 | func (c *controller) GetStory(ctx *fiber.Ctx) error { 78 | server, err := ark.ParseServer(ctx.Params("server")) 79 | if err != nil { 80 | return fiber.ErrBadRequest 81 | } 82 | id := ctx.Params("id") 83 | if id == "" { 84 | return fiber.ErrBadRequest 85 | } 86 | 87 | story, err := c.service.GetStory(server, id) 88 | if err != nil { 89 | return err 90 | } 91 | 92 | return ctx.JSON(story) 93 | } 94 | 95 | func (c *controller) GetAggregatedPictureArt(ctx *fiber.Ctx) error { 96 | server, err := ark.ParseServer(ctx.Params("server")) 97 | if err != nil { 98 | return err 99 | } 100 | 101 | id := ctx.Params("id") 102 | 103 | art, err := c.service.GetAggregatedPictureArt(server, id) 104 | if err != nil { 105 | return err 106 | } 107 | 108 | return ctx.JSON(art) 109 | } 110 | func (c *controller) GetAggregatedCharacterArt(ctx *fiber.Ctx) error { 111 | server, err := ark.ParseServer(ctx.Params("server")) 112 | if err != nil { 113 | return err 114 | } 115 | 116 | id := ctx.Params("id") 117 | 118 | art, err := c.service.GetAggregatedCharacterArt(server, id) 119 | if err != nil { 120 | return err 121 | } 122 | 123 | return ctx.JSON(art) 124 | } 125 | -------------------------------------------------------------------------------- /internal/pkg/arkparser/story_characters.go: -------------------------------------------------------------------------------- 1 | package arkparser 2 | 3 | import ( 4 | "fmt" 5 | "github.com/wk8/go-ordered-map/v2" 6 | "regexp" 7 | ) 8 | 9 | func (p *Parser) ParseCharacters(directives []Directive) []*StoryCharacter { 10 | stage := characterStage{ 11 | spotlight: "", 12 | characters: make(map[string]string), 13 | history: make([]string, 0), 14 | names: make(map[string][]string), 15 | } 16 | for _, directive := range directives { 17 | switch directive.Name { 18 | case "": 19 | name := directive.Params["name"] 20 | stage.name(name) 21 | case "character": 22 | id1 := NormalizeCharacterID(directive.Params["name"]) 23 | id2 := NormalizeCharacterID(directive.Params["name2"]) 24 | stage.take("1", id1) 25 | stage.take("2", id2) 26 | stage.focus(directive.Params["focus"], "1") 27 | case "charslot": 28 | if id := NormalizeCharacterID(directive.Params["name"]); id != "" { 29 | stage.take(directive.Params["slot"], id) 30 | stage.focus(directive.Params["focus"], directive.Params["slot"]) 31 | } else { 32 | stage.exit() 33 | } 34 | case "dialog": 35 | stage.exit() 36 | } 37 | } 38 | 39 | result := make([]*StoryCharacter, 0) 40 | for _, id := range stage.history { 41 | result = append(result, &StoryCharacter{ 42 | ID: id, 43 | Names: unique(stage.names[id]), 44 | }) 45 | } 46 | 47 | return result 48 | } 49 | 50 | func NormalizeCharacterID(id string) string { 51 | if id == "" { 52 | return "" 53 | } 54 | 55 | regex := regexp.MustCompile(`^(.*?)(?:#(\d+))?(?:\$(\d+))?$`) 56 | matches := regex.FindStringSubmatch(id) 57 | if len(matches) == 0 { 58 | return "" 59 | } 60 | 61 | id = matches[1] 62 | faceNum, bodyNum := "1", "1" // default values are 1 63 | if matches[2] != "" { 64 | faceNum = matches[2] 65 | } 66 | if matches[3] != "" { 67 | bodyNum = matches[3] 68 | } 69 | 70 | return fmt.Sprintf("%s#%s$%s", id, faceNum, bodyNum) 71 | } 72 | 73 | type characterStage struct { 74 | spotlight string 75 | characters map[string]string 76 | 77 | history []string 78 | names map[string][]string 79 | } 80 | 81 | func (s *characterStage) protagonist() string { 82 | return s.characters[s.spotlight] 83 | } 84 | func (s *characterStage) focus(slot string, defaultSlot string) { 85 | if slot != "" { 86 | s.spotlight = slot 87 | } else if len(s.characters) == 1 { 88 | for slot, _ := range s.characters { 89 | s.spotlight = slot 90 | } 91 | } else { 92 | s.spotlight = "" 93 | } 94 | } 95 | func (s *characterStage) take(slot string, id string) { 96 | if id != "" { 97 | s.characters[slot] = id 98 | s.history = append(s.history, id) 99 | } else { 100 | delete(s.characters, slot) 101 | } 102 | } 103 | func (s *characterStage) name(name string) { 104 | s.names[s.protagonist()] = append(s.names[s.protagonist()], name) 105 | } 106 | func (s *characterStage) exit() { 107 | s.spotlight = "" 108 | s.characters = make(map[string]string) 109 | } 110 | 111 | func unique[T comparable](slice []T) []T { 112 | m := orderedmap.New[T, struct{}](len(slice)) 113 | for _, el := range slice { 114 | m.Set(el, struct{}{}) 115 | } 116 | slice = nil 117 | for pair := m.Oldest(); pair != nil; pair = pair.Next() { 118 | slice = append(slice, pair.Key) 119 | } 120 | return slice 121 | } 122 | -------------------------------------------------------------------------------- /internal/app/updateloop/update_story.go: -------------------------------------------------------------------------------- 1 | package updateloop 2 | 3 | import ( 4 | "context" 5 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 6 | "github.com/flandiayingman/arkwaifu/internal/pkg/arkdata" 7 | "github.com/flandiayingman/arkwaifu/internal/pkg/arkparser" 8 | "github.com/rs/zerolog/log" 9 | _ "image/jpeg" // register jpeg codec 10 | _ "image/png" // register png codec 11 | "os" 12 | "time" 13 | ) 14 | 15 | var ( 16 | storyPatterns = []string{ 17 | "gamedata/excel/**", 18 | "gamedata/story/**", 19 | } 20 | ) 21 | 22 | func (s *Service) getRemoteStoryVersion(ctx context.Context, server ark.Server) (ark.Version, error) { 23 | resourceVersion, err := arkdata.GetLatestDataVersion(ctx, server) 24 | if err != nil { 25 | return "", err 26 | } 27 | return resourceVersion.ResourceVersion, err 28 | } 29 | 30 | func (s *Service) getLocalStoryVersion(ctx context.Context, server ark.Server) (ark.Version, error) { 31 | return s.repo.selectStoryVersion(ctx, server) 32 | } 33 | 34 | func (s *Service) attemptUpdateStory(ctx context.Context, server ark.Server) { 35 | log := log.With(). 36 | Str("server", server). 37 | Logger() 38 | 39 | log.Info(). 40 | Msg("Attempting to update stories of the server...") 41 | 42 | localStoryVersion, err := s.getLocalStoryVersion(ctx, server) 43 | if err != nil { 44 | log.Err(err). 45 | Msg("Failed to get the local story version of the server. ") 46 | return 47 | } 48 | 49 | remoteStoryVersion, err := s.getRemoteStoryVersion(ctx, server) 50 | if err != nil { 51 | log.Err(err). 52 | Msg("Failed to get the remote story version of the server. ") 53 | return 54 | } 55 | 56 | log = log.With(). 57 | Str("localStoryVersion", localStoryVersion). 58 | Str("remoteStoryVersion", remoteStoryVersion). 59 | Logger() 60 | 61 | if localStoryVersion != remoteStoryVersion { 62 | log.Info(). 63 | Msg("Updating the stories of the server, since the story versions are not identical. ") 64 | begin := time.Now() 65 | err = s.updateStories(ctx, server, remoteStoryVersion) 66 | if err != nil { 67 | log.Err(err). 68 | Msg("Update loop failed to update the stories") 69 | } 70 | log.Info(). 71 | Dur("elapsed", time.Since(begin)). 72 | Msg("Updated the stories of the server successfully. ") 73 | } else { 74 | log.Info(). 75 | Msg("Skip updating the stories of the server, since the story versions are identical. ") 76 | } 77 | } 78 | 79 | func (s *Service) updateStories(ctx context.Context, server ark.Server, version ark.Version) error { 80 | root, err := os.MkdirTemp("", "arkwaifu-updateloop-story-*") 81 | if err != nil { 82 | return err 83 | } 84 | defer os.RemoveAll(root) 85 | 86 | err = arkdata.GetGameData(ctx, server, version, storyPatterns, root) 87 | if err != nil { 88 | return err 89 | } 90 | 91 | parser := arkparser.Parser{ 92 | Root: root, 93 | Prefix: "assets/torappu/dynamicassets", 94 | } 95 | 96 | storyTree, err := parser.Parse() 97 | if err != nil { 98 | return err 99 | } 100 | err = s.storyService.PopulateFrom(storyTree, server) 101 | if err != nil { 102 | return err 103 | } 104 | 105 | err = s.repo.upsertStoryVersion(ctx, &storyVersion{ 106 | Server: server, 107 | Version: version, 108 | }) 109 | if err != nil { 110 | return err 111 | } 112 | 113 | return nil 114 | } 115 | -------------------------------------------------------------------------------- /internal/pkg/arkdata/data_test.go: -------------------------------------------------------------------------------- 1 | package arkdata 2 | 3 | import ( 4 | "context" 5 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 6 | "github.com/flandiayingman/arkwaifu/internal/pkg/util/pathutil" 7 | "github.com/stretchr/testify/assert" 8 | "os" 9 | "path" 10 | "testing" 11 | ) 12 | 13 | func TestGetGameData(t *testing.T) { 14 | type args struct { 15 | ctx context.Context 16 | server ark.Server 17 | version ark.Version 18 | patterns []string 19 | dst string 20 | } 21 | tests := []struct { 22 | name string 23 | args args 24 | }{ 25 | { 26 | name: "specific version story_review_table", 27 | args: args{ 28 | ctx: context.Background(), 29 | server: ark.CnServer, 30 | version: "23-07-27-18-50-06-aeb568", 31 | patterns: []string{"**/story_review_table.json"}, 32 | dst: "./test_1", 33 | }, 34 | }, 35 | { 36 | name: "latest all .json", 37 | args: args{ 38 | ctx: context.Background(), 39 | server: ark.CnServer, 40 | version: "", 41 | patterns: []string{"**/*.json"}, 42 | dst: "./test_2", 43 | }, 44 | }, 45 | { 46 | name: "latest all stories", 47 | args: args{ 48 | ctx: context.Background(), 49 | server: ark.CnServer, 50 | version: "", 51 | patterns: []string{"gamedata/story/**/*.txt"}, 52 | dst: "./test_3", 53 | }, 54 | }, 55 | } 56 | for _, tt := range tests { 57 | t.Run(tt.name, func(t *testing.T) { 58 | defer os.RemoveAll(tt.args.dst) 59 | err := GetGameData(tt.args.ctx, tt.args.server, tt.args.version, tt.args.patterns, tt.args.dst) 60 | if err != nil { 61 | t.Error(err) 62 | } 63 | 64 | files, err := pathutil.Glob(tt.args.patterns, path.Join(tt.args.dst, DefaultPrefix)) 65 | if err != nil { 66 | t.Error(err) 67 | } 68 | 69 | assert.NotEmpty(t, files) 70 | }) 71 | } 72 | } 73 | 74 | func TestGetCompositeGameData(t *testing.T) { 75 | type args struct { 76 | ctx context.Context 77 | server ark.Server 78 | version ark.Version 79 | patterns []string 80 | dst string 81 | } 82 | tests := []struct { 83 | name string 84 | args args 85 | successful bool 86 | }{ 87 | { 88 | name: "specific version story_review_table", 89 | args: args{ 90 | ctx: context.Background(), 91 | server: ark.CnServer, 92 | version: "23-07-27-18-50-06-aeb568", 93 | patterns: []string{"**/story_review_table.json"}, 94 | dst: "./test_1", 95 | }, 96 | successful: true, 97 | }, 98 | { 99 | name: "latest all .json", 100 | args: args{ 101 | ctx: context.Background(), 102 | server: ark.CnServer, 103 | version: "", 104 | patterns: []string{"**/*.json"}, 105 | dst: "./test_2", 106 | }, 107 | successful: true, 108 | }, 109 | { 110 | name: "latest all stories", 111 | args: args{ 112 | ctx: context.Background(), 113 | server: ark.CnServer, 114 | version: "", 115 | patterns: []string{"gamedata/story/**/*.txt"}, 116 | dst: "./test_3", 117 | }, 118 | successful: false, 119 | }, 120 | } 121 | for _, tt := range tests { 122 | t.Run(tt.name, func(t *testing.T) { 123 | defer os.RemoveAll(tt.args.dst) 124 | err := GetCompositeGameData(tt.args.ctx, tt.args.server, tt.args.version, tt.args.patterns, tt.args.dst) 125 | 126 | files, err := pathutil.Glob(tt.args.patterns, path.Join(tt.args.dst, DefaultPrefix)) 127 | if err != nil { 128 | t.Error(err) 129 | } 130 | 131 | if tt.successful { 132 | assert.NotEmpty(t, files) 133 | } else { 134 | assert.Empty(t, files) 135 | } 136 | }) 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /tools/extractor/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.toptal.com/developers/gitignore/api/python 3 | # Edit at https://www.toptal.com/developers/gitignore?templates=python 4 | 5 | ### Python ### 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | share/python-wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | MANIFEST 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .nox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *.cover 54 | *.py,cover 55 | .hypothesis/ 56 | .pytest_cache/ 57 | cover/ 58 | 59 | # Translations 60 | *.mo 61 | *.pot 62 | 63 | # Django stuff: 64 | *.log 65 | local_settings.py 66 | db.sqlite3 67 | db.sqlite3-journal 68 | 69 | # Flask stuff: 70 | instance/ 71 | .webassets-cache 72 | 73 | # Scrapy stuff: 74 | .scrapy 75 | 76 | # Sphinx documentation 77 | docs/_build/ 78 | 79 | # PyBuilder 80 | .pybuilder/ 81 | target/ 82 | 83 | # Jupyter Notebook 84 | .ipynb_checkpoints 85 | 86 | # IPython 87 | profile_default/ 88 | ipython_config.py 89 | 90 | # pyenv 91 | # For a library or package, you might want to ignore these files since the code is 92 | # intended to run in multiple environments; otherwise, check them in: 93 | # .python-version 94 | 95 | # pipenv 96 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 97 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 98 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 99 | # install all needed dependencies. 100 | #Pipfile.lock 101 | 102 | # poetry 103 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 104 | # This is especially recommended for binary packages to ensure reproducibility, and is more 105 | # commonly ignored for libraries. 106 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 107 | #poetry.lock 108 | 109 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 110 | __pypackages__/ 111 | 112 | # Celery stuff 113 | celerybeat-schedule 114 | celerybeat.pid 115 | 116 | # SageMath parsed files 117 | *.sage.py 118 | 119 | # Environments 120 | .env 121 | .venv 122 | env/ 123 | venv/ 124 | ENV/ 125 | env.bak/ 126 | venv.bak/ 127 | 128 | # Spyder project settings 129 | .spyderproject 130 | .spyproject 131 | 132 | # Rope project settings 133 | .ropeproject 134 | 135 | # mkdocs documentation 136 | /site 137 | 138 | # mypy 139 | .mypy_cache/ 140 | .dmypy.json 141 | dmypy.json 142 | 143 | # Pyre type checker 144 | .pyre/ 145 | 146 | # pytype static type analyzer 147 | .pytype/ 148 | 149 | # Cython debug symbols 150 | cython_debug/ 151 | 152 | # PyCharm 153 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 154 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 155 | # and can be added to the global gitignore or merged into this file. For a more nuclear 156 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 157 | #.idea/ 158 | 159 | # End of https://www.toptal.com/developers/gitignore/api/python -------------------------------------------------------------------------------- /internal/app/artext/ext_story.go: -------------------------------------------------------------------------------- 1 | package artext 2 | 3 | import ( 4 | "github.com/flandiayingman/arkwaifu/internal/app/art" 5 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 6 | "github.com/gofiber/fiber/v2" 7 | ) 8 | 9 | func (c *Controller) GetArtsOfStoryGroup(ctx *fiber.Ctx) error { 10 | queryGroup := ctx.Query("group") 11 | if queryGroup == "" { 12 | return ctx.Next() 13 | } 14 | 15 | server, err := ark.ParseServer(ctx.Query("server")) 16 | if err != nil { 17 | return fiber.ErrBadRequest 18 | } 19 | 20 | arts, err := c.service.GetArtsOfStoryGroup(server, queryGroup) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | return ctx.JSON(arts) 26 | } 27 | 28 | func (c *Controller) GetArtsOfStory(ctx *fiber.Ctx) error { 29 | queryStory := ctx.Query("story") 30 | if queryStory == "" { 31 | return ctx.Next() 32 | } 33 | 34 | server, err := ark.ParseServer(ctx.Query("server")) 35 | if err != nil { 36 | return fiber.ErrBadRequest 37 | } 38 | 39 | arts, err := c.service.GetArtsOfStory(server, queryStory) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | return ctx.JSON(arts) 45 | } 46 | 47 | func (c *Controller) GetArtsExceptForStoryArts(ctx *fiber.Ctx) error { 48 | queryExceptForStoryArts := ctx.QueryBool("except-for-story-arts", false) 49 | if !queryExceptForStoryArts { 50 | return ctx.Next() 51 | } 52 | 53 | server, err := ark.ParseServer(ctx.Query("server")) 54 | if err != nil { 55 | return fiber.ErrBadRequest 56 | } 57 | 58 | arts, err := c.service.GetArtsExceptForStoryArts(server) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | return ctx.JSON(arts) 64 | } 65 | 66 | func (s *Service) GetArtsOfStoryGroup(server ark.Server, groupID string) ([]*art.Art, error) { 67 | group, err := s.story.GetStoryGroup(server, groupID) 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | ids := make([]string, 0) 73 | for _, story := range group.Stories { 74 | for _, art := range story.PictureArts { 75 | ids = append(ids, art.ID) 76 | } 77 | for _, art := range story.CharacterArts { 78 | ids = append(ids, art.ID) 79 | } 80 | } 81 | 82 | arts, err := s.art.SelectArtsByIDs(ids) 83 | if err != nil { 84 | return nil, err 85 | } 86 | 87 | return arts, err 88 | } 89 | 90 | func (s *Service) GetArtsOfStory(server ark.Server, storyID string) ([]*art.Art, error) { 91 | story, err := s.story.GetStory(server, storyID) 92 | if err != nil { 93 | return nil, err 94 | } 95 | 96 | ids := make([]string, 0) 97 | for _, art := range story.PictureArts { 98 | ids = append(ids, art.ID) 99 | } 100 | for _, art := range story.CharacterArts { 101 | ids = append(ids, art.ID) 102 | } 103 | 104 | arts, err := s.art.SelectArtsByIDs(ids) 105 | if err != nil { 106 | return nil, err 107 | } 108 | 109 | return arts, err 110 | } 111 | 112 | // GetArtsExceptForStoryArts gets all arts but except for the arts which are present in the story tree. 113 | func (s *Service) GetArtsExceptForStoryArts(server ark.Server) ([]*art.Art, error) { 114 | pictureArts, err := s.story.GetPictureArts(server) 115 | if err != nil { 116 | return nil, err 117 | } 118 | characterArts, err := s.story.GetCharacterArts(server) 119 | if err != nil { 120 | return nil, err 121 | } 122 | 123 | storyArtsIDSet := make(map[string]bool) 124 | for _, pictureArt := range pictureArts { 125 | storyArtsIDSet[pictureArt.ID] = true 126 | } 127 | for _, characterArt := range characterArts { 128 | storyArtsIDSet[characterArt.ID] = true 129 | } 130 | 131 | arts, err := s.art.SelectArts() 132 | if err != nil { 133 | return nil, err 134 | } 135 | 136 | result := make([]*art.Art, 0) 137 | for _, art := range arts { 138 | if !storyArtsIDSet[art.ID] { 139 | result = append(result, art) 140 | } 141 | } 142 | 143 | return result, nil 144 | } 145 | -------------------------------------------------------------------------------- /internal/pkg/arkparser/story_review_table.go: -------------------------------------------------------------------------------- 1 | package arkparser 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/pkg/errors" 7 | "github.com/wk8/go-ordered-map/v2" 8 | "os" 9 | "path" 10 | "path/filepath" 11 | ) 12 | 13 | type JsonStoryGroup struct { 14 | ID string `json:"id"` 15 | Name string `json:"name"` 16 | Type string `json:"actType"` 17 | Stories []*JsonStory `json:"infoUnlockDatas"` 18 | } 19 | type JsonStory struct { 20 | ID string `json:"storyId"` 21 | GroupID string `json:"storyGroup"` 22 | Code string `json:"storyCode"` 23 | Name string `json:"storyName"` 24 | Info string `json:"storyInfo"` 25 | Text string `json:"storyTxt"` 26 | Tag JsonStoryTag `json:"avgTag"` 27 | } 28 | type JsonStoryTag struct { 29 | Type string 30 | Text string 31 | } 32 | 33 | func (t *JsonStoryTag) UnmarshalJSON(data []byte) error { 34 | var s string 35 | err := json.Unmarshal(data, &s) 36 | if err != nil { 37 | return errors.WithStack(err) 38 | } 39 | switch s { 40 | case TagBeforeCN, TagBeforeEN, TagBeforeJP, TagBeforeKR: 41 | *t = JsonStoryTag{ 42 | Type: TagBefore, 43 | Text: s, 44 | } 45 | case TagAfterCN, TagAfterEN, TagAfterJP, TagAfterKR: 46 | *t = JsonStoryTag{ 47 | Type: TagAfter, 48 | Text: s, 49 | } 50 | case TagInterludeCN, TagInterludeEN, TagInterludeJP, TagInterludeKR: 51 | *t = JsonStoryTag{ 52 | Type: TagInterlude, 53 | Text: s, 54 | } 55 | default: 56 | return errors.Errorf("unknown story tag: %s", s) 57 | } 58 | return nil 59 | } 60 | 61 | const ( 62 | TypeMain string = "MAIN_STORY" 63 | TypeActivity string = "ACTIVITY_STORY" 64 | TypeMiniActivity string = "MINI_STORY" 65 | TypeNone string = "NONE" 66 | 67 | TagBefore string = "BEFORE" 68 | TagAfter string = "AFTER" 69 | TagInterlude string = "INTERLUDE" 70 | 71 | TagBeforeCN string = "行动前" 72 | TagAfterCN string = "行动后" 73 | TagInterludeCN string = "幕间" 74 | TagBeforeEN string = "Before Operation" 75 | TagAfterEN string = "After Operation" 76 | TagInterludeEN string = "Interlude" 77 | TagBeforeJP string = "戦闘前" 78 | TagAfterJP string = "戦闘後" 79 | TagInterludeJP string = "幕間" 80 | TagBeforeKR string = "작전 전" 81 | TagAfterKR string = "작전 후" 82 | TagInterludeKR string = "브릿지" 83 | ) 84 | 85 | func (p *Parser) ParseStoryReviewTable() ([]*JsonStoryGroup, error) { 86 | jsonPath := filepath.Join(p.Root, p.Prefix, "gamedata/excel/story_review_table.json") 87 | jsonData, err := os.ReadFile(jsonPath) 88 | if err != nil { 89 | return nil, errors.WithStack(err) 90 | } 91 | 92 | // key: story group id; value: story group 93 | dynamicJsonObject := orderedmap.New[string, JsonStoryGroup]() 94 | err = json.Unmarshal(jsonData, &dynamicJsonObject) 95 | if err != nil { 96 | return nil, errors.WithStack(err) 97 | } 98 | 99 | result := make([]*JsonStoryGroup, 0, dynamicJsonObject.Len()) 100 | for pair := dynamicJsonObject.Oldest(); pair != nil; pair = pair.Next() { 101 | result = append(result, &pair.Value) 102 | } 103 | 104 | return result, nil 105 | } 106 | 107 | func (p *Parser) GetInfo(infoPath string) (string, error) { 108 | infoPath = fmt.Sprintf("[uc]%s.txt", infoPath) 109 | infoPath = path.Join(p.Root, p.Prefix, "gamedata/story", infoPath) 110 | 111 | bytes, err := os.ReadFile(infoPath) 112 | if err != nil { 113 | return "", errors.WithStack(err) 114 | } 115 | 116 | return string(bytes), nil 117 | } 118 | func (p *Parser) GetText(textPath string) (string, error) { 119 | textPath = fmt.Sprintf("%s.txt", textPath) 120 | textPath = path.Join(p.Root, p.Prefix, "gamedata/story", textPath) 121 | 122 | bytes, err := os.ReadFile(textPath) 123 | if err != nil { 124 | return "", errors.WithStack(err) 125 | } 126 | 127 | return string(bytes), nil 128 | } 129 | -------------------------------------------------------------------------------- /internal/pkg/arkparser/arkparser.go: -------------------------------------------------------------------------------- 1 | package arkparser 2 | 3 | import "github.com/pkg/errors" 4 | 5 | type Parser struct { 6 | Root string 7 | Prefix string 8 | 9 | pictureMap map[string]*StoryPicture 10 | characterMap map[string]*StoryCharacter 11 | } 12 | 13 | type StoryTree struct { 14 | StoryGroups []*StoryGroup 15 | } 16 | 17 | type StoryGroup struct { 18 | ID string 19 | Name string 20 | Type string 21 | Stories []*Story 22 | } 23 | 24 | type Story struct { 25 | ID string 26 | GroupID string 27 | Code string 28 | Name string 29 | Info string 30 | TagType string 31 | TagText string 32 | 33 | Pictures []*StoryPicture 34 | Characters []*StoryCharacter 35 | } 36 | 37 | type StoryCharacter struct { 38 | ID string 39 | Names []string 40 | } 41 | 42 | type StoryPicture struct { 43 | ID string 44 | Category string 45 | Title string 46 | Subtitle string 47 | } 48 | 49 | func (p *Parser) getPicture(id string) *StoryPicture { 50 | picture, ok := p.pictureMap[id] 51 | if !ok { 52 | picture = &StoryPicture{} 53 | p.pictureMap[id] = picture 54 | } 55 | return picture 56 | } 57 | func (p *Parser) setPicture(id string, picture *StoryPicture) { 58 | p.pictureMap[id] = picture 59 | } 60 | func (p *Parser) getCharacter(id string) *StoryCharacter { 61 | character, ok := p.characterMap[id] 62 | if !ok { 63 | character = &StoryCharacter{} 64 | p.characterMap[id] = character 65 | } 66 | return character 67 | } 68 | func (p *Parser) setCharacter(id string, character *StoryCharacter) { 69 | p.characterMap[id] = character 70 | } 71 | 72 | func (p *Parser) Parse() (*StoryTree, error) { 73 | jsonStoryReviewTable, err := p.ParseStoryReviewTable() 74 | if err != nil { 75 | return nil, errors.WithStack(err) 76 | } 77 | 78 | tree := StoryTree{} 79 | picIndex := make(map[string][]*StoryPicture) 80 | charIndex := make(map[string][]*StoryCharacter) 81 | for _, jsonGroup := range jsonStoryReviewTable { 82 | group := StoryGroup{ 83 | ID: jsonGroup.ID, 84 | Name: jsonGroup.Name, 85 | Type: jsonGroup.Type, 86 | Stories: nil, 87 | } 88 | for _, jsonStory := range jsonGroup.Stories { 89 | info, _ := p.GetInfo(jsonStory.Info) 90 | // If info is not found, it is likely that there is no info for the story. 91 | // Example: main_14_level_main_14-20_beg 92 | //if err != nil { 93 | // return nil, errors.WithStack(err) 94 | //} 95 | directives, err := p.ParseStoryText(jsonStory.Text) 96 | if err != nil { 97 | return nil, errors.WithStack(err) 98 | } 99 | 100 | pictures := p.ParsePictures(directives) 101 | characters := p.ParseCharacters(directives) 102 | story := Story{ 103 | ID: jsonStory.ID, 104 | GroupID: jsonStory.ID, 105 | Code: jsonStory.Code, 106 | Name: jsonStory.Name, 107 | Info: info, 108 | TagType: jsonStory.Tag.Type, 109 | TagText: jsonStory.Tag.Text, 110 | Pictures: pictures, 111 | Characters: characters, 112 | } 113 | 114 | for _, picture := range pictures { 115 | picIndex[picture.ID] = append(picIndex[picture.ID], picture) 116 | } 117 | for _, character := range characters { 118 | charIndex[character.ID] = append(charIndex[character.ID], character) 119 | } 120 | 121 | group.Stories = append(group.Stories, &story) 122 | } 123 | tree.StoryGroups = append(tree.StoryGroups, &group) 124 | } 125 | 126 | storyReviewMetaTable, err := p.ParseStoryReviewMetaTable() 127 | if err != nil { 128 | return nil, errors.WithStack(err) 129 | } 130 | for pair := storyReviewMetaTable.ActArchiveResData.Pics.Oldest(); pair != nil; pair = pair.Next() { 131 | pic := pair.Value 132 | for _, picture := range picIndex[pic.AssetPath] { 133 | picture.Title = pic.Desc 134 | picture.Subtitle = pic.PicDescription 135 | } 136 | } 137 | 138 | return &tree, nil 139 | } 140 | -------------------------------------------------------------------------------- /internal/app/updateloop/updateloop.go: -------------------------------------------------------------------------------- 1 | // Package updateloop provide functionalities to keep the local assets 2 | // up-to-date. 3 | // 4 | // The operation "update" contains 3 sub-operations, and the subject of update 5 | // can be either PictureArt module or Story module. 6 | // 7 | // Pull. It pulls the assets from the remote API, either the HyperGryph API or 8 | // the ArknightsGameData GitHub repository API. It also unpacks or decompresses 9 | // the assets from the downloaded files, therefore, whether the assets are 10 | // pulled from either API, the result directory structure will turn out the 11 | // same. 12 | // 13 | // Preprocess. It converts the pulled assets to the form which the submitter of 14 | // the "submit" sub-operation can recognize. Moreover, it does some image 15 | // preprocessing such as merge the alpha channel and merge the different faces 16 | // onto the body for characters (合并差分). 17 | // 18 | // Submit. It parses the preprocessed assets and submits them to the art service 19 | // and the story service. 20 | // 21 | // After the above sub-operations, all intermediate files shall be deleted to 22 | // preserve the disk space. 23 | package updateloop 24 | 25 | import ( 26 | "context" 27 | "fmt" 28 | "github.com/flandiayingman/arkwaifu/internal/app/art" 29 | "github.com/flandiayingman/arkwaifu/internal/app/gallery" 30 | "github.com/flandiayingman/arkwaifu/internal/app/story" 31 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 32 | "github.com/rs/zerolog" 33 | "github.com/rs/zerolog/log" 34 | "go.uber.org/fx" 35 | "os" 36 | "time" 37 | ) 38 | 39 | func init() { 40 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 41 | } 42 | 43 | func FxModule() fx.Option { 44 | return fx.Module("updateloop", 45 | fx.Provide( 46 | newRepo, 47 | newService, 48 | ), 49 | fx.Invoke( 50 | registerService, 51 | ), 52 | ) 53 | } 54 | 55 | // Service provides all functionalities of update. 56 | type Service struct { 57 | repo *repo 58 | 59 | artService *art.Service 60 | storyService *story.Service 61 | galleryService *gallery.Service 62 | } 63 | 64 | func newService( 65 | artService *art.Service, 66 | storyService *story.Service, 67 | galleryService *gallery.Service, 68 | repo *repo, 69 | ) *Service { 70 | return &Service{ 71 | artService: artService, 72 | storyService: storyService, 73 | galleryService: galleryService, 74 | repo: repo, 75 | } 76 | } 77 | 78 | func registerService(service *Service, lc fx.Lifecycle) { 79 | loopCtx, cancelLoop := context.WithCancel(context.Background()) 80 | lc.Append(fx.Hook{ 81 | OnStart: func(ctx context.Context) error { 82 | go service.Loop(loopCtx) 83 | return nil 84 | }, 85 | OnStop: func(ctx context.Context) error { 86 | cancelLoop() 87 | return nil 88 | }, 89 | }) 90 | } 91 | 92 | func (s *Service) Loop(ctx context.Context) { 93 | for { 94 | select { 95 | case <-ctx.Done(): 96 | break 97 | default: 98 | s.AttemptUpdate(ctx) 99 | } 100 | time.Sleep(5 * time.Minute) 101 | } 102 | } 103 | 104 | // AttemptUpdate attempts to update the resources. 105 | func (s *Service) AttemptUpdate(ctx context.Context) { 106 | log := log.With(). 107 | Logger() 108 | log.Info().Msg("Update loop is attempting to update the assets... ") 109 | 110 | defer func() { 111 | v := recover() 112 | if v != nil { 113 | log.Error(). 114 | Str("err", fmt.Sprintf("%+v", v)). 115 | Msg("Update loop has recovered from panic.") 116 | } 117 | }() 118 | 119 | s.attemptUpdateArt(ctx) 120 | s.attemptUpdateArtThumbnails(ctx) 121 | 122 | for _, server := range ark.Servers { 123 | // Skip TW server, since we haven't implemented it yet. 124 | if server == ark.TwServer { 125 | continue 126 | } 127 | s.attemptUpdateStory(ctx, server) 128 | s.attemptUpdateGalleries(ctx, server) 129 | } 130 | 131 | log.Info().Msg("Update loop has completed this attempt to update the assets..") 132 | } 133 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/flandiayingman/arkwaifu 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/Jeffail/gabs/v2 v2.7.0 7 | github.com/ahmetb/go-linq/v3 v3.2.0 8 | github.com/alitto/pond v1.9.2 9 | github.com/caarlos0/env/v6 v6.10.1 10 | github.com/cavaliergopher/grab/v3 v3.0.1 11 | github.com/chai2010/webp v1.1.1 12 | github.com/disintegration/imaging v1.6.2 13 | github.com/gobwas/glob v0.2.3 14 | github.com/goccy/go-yaml v1.17.1 15 | github.com/gofiber/fiber/v2 v2.52.6 16 | github.com/google/go-github/v52 v52.0.0 17 | github.com/google/uuid v1.6.0 18 | github.com/lib/pq v1.10.9 19 | github.com/mholt/archiver/v4 v4.0.0-alpha.8 20 | github.com/pkg/errors v0.9.1 21 | github.com/rs/zerolog v1.34.0 22 | github.com/stretchr/testify v1.10.0 23 | github.com/wk8/go-ordered-map/v2 v2.1.8 24 | go.uber.org/fx v1.23.0 25 | golang.org/x/sync v0.12.0 26 | gorm.io/driver/postgres v1.5.11 27 | gorm.io/gorm v1.25.12 28 | ) 29 | 30 | require ( 31 | github.com/ProtonMail/go-crypto v1.1.6 // indirect 32 | github.com/STARRY-S/zip v0.2.2 // indirect 33 | github.com/andybalholm/brotli v1.1.1 // indirect 34 | github.com/bahlo/generic-list-go v0.2.0 // indirect 35 | github.com/bodgit/plumbing v1.3.0 // indirect 36 | github.com/bodgit/sevenzip v1.6.0 // indirect 37 | github.com/bodgit/windows v1.0.1 // indirect 38 | github.com/buger/jsonparser v1.1.1 // indirect 39 | github.com/cloudflare/circl v1.6.0 // indirect 40 | github.com/davecgh/go-spew v1.1.1 // indirect 41 | github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect 42 | github.com/fatih/color v1.18.0 // indirect 43 | github.com/golang/protobuf v1.5.4 // indirect 44 | github.com/golang/snappy v1.0.0 // indirect 45 | github.com/google/go-querystring v1.1.0 // indirect 46 | github.com/hashicorp/errwrap v1.1.0 // indirect 47 | github.com/hashicorp/go-multierror v1.1.1 // indirect 48 | github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect 49 | github.com/jackc/pgpassfile v1.0.0 // indirect 50 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect 51 | github.com/jackc/pgx/v5 v5.7.4 // indirect 52 | github.com/jackc/puddle/v2 v2.2.2 // indirect 53 | github.com/jinzhu/inflection v1.0.0 // indirect 54 | github.com/jinzhu/now v1.1.5 // indirect 55 | github.com/klauspost/compress v1.18.0 // indirect 56 | github.com/klauspost/pgzip v1.2.6 // indirect 57 | github.com/mailru/easyjson v0.9.0 // indirect 58 | github.com/mattn/go-colorable v0.1.14 // indirect 59 | github.com/mattn/go-isatty v0.0.20 // indirect 60 | github.com/mattn/go-runewidth v0.0.16 // indirect 61 | github.com/nwaples/rardecode/v2 v2.1.1 // indirect 62 | github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect 63 | github.com/pierrec/lz4/v4 v4.1.22 // indirect 64 | github.com/pmezard/go-difflib v1.0.0 // indirect 65 | github.com/rivo/uniseg v0.4.7 // indirect 66 | github.com/rogpeppe/go-internal v1.13.1 // indirect 67 | github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect 68 | github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect 69 | github.com/sorairolake/lzip-go v0.3.7 // indirect 70 | github.com/stretchr/objx v0.5.2 // indirect 71 | github.com/therootcompany/xz v1.0.1 // indirect 72 | github.com/tinylib/msgp v1.2.5 // indirect 73 | github.com/ulikunitz/xz v0.5.12 // indirect 74 | github.com/valyala/bytebufferpool v1.0.0 // indirect 75 | github.com/valyala/fasthttp v1.60.0 // indirect 76 | github.com/valyala/tcplisten v1.0.0 // indirect 77 | go.uber.org/atomic v1.11.0 // indirect 78 | go.uber.org/dig v1.18.1 // indirect 79 | go.uber.org/multierr v1.11.0 // indirect 80 | go.uber.org/zap v1.27.0 // indirect 81 | go4.org v0.0.0-20230225012048-214862532bf5 // indirect 82 | golang.org/x/crypto v0.36.0 // indirect 83 | golang.org/x/image v0.25.0 // indirect 84 | golang.org/x/net v0.38.0 // indirect 85 | golang.org/x/oauth2 v0.28.0 // indirect 86 | golang.org/x/sys v0.31.0 // indirect 87 | golang.org/x/text v0.23.0 // indirect 88 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect 89 | google.golang.org/appengine v1.6.8 // indirect 90 | google.golang.org/protobuf v1.36.6 // indirect 91 | gopkg.in/yaml.v3 v3.0.1 // indirect 92 | ) 93 | -------------------------------------------------------------------------------- /internal/pkg/arkassets/assets.go: -------------------------------------------------------------------------------- 1 | package arkassets 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/ahmetb/go-linq/v3" 7 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 8 | "github.com/flandiayingman/arkwaifu/internal/pkg/cols" 9 | "github.com/flandiayingman/arkwaifu/internal/pkg/util/fileutil" 10 | "github.com/flandiayingman/arkwaifu/internal/pkg/util/pathutil" 11 | "github.com/pkg/errors" 12 | "os" 13 | "regexp" 14 | "strings" 15 | ) 16 | 17 | const ( 18 | AssetBaseUrl = "https://ak.hycdn.cn/assetbundle/official/Android/assets" 19 | ) 20 | 21 | type Info struct { 22 | Name string 23 | MD5 string 24 | Version ark.Version 25 | } 26 | 27 | func (i Info) Url() string { 28 | i.Name = strings.ReplaceAll(i.Name, `/`, `_`) 29 | i.Name = strings.ReplaceAll(i.Name, `#`, `__`) 30 | i.Name = regexp.MustCompile(`\.(.*?)$`).ReplaceAllString(i.Name, ".dat") 31 | return fmt.Sprintf("%s/%s/%s", AssetBaseUrl, i.Version, i.Name) 32 | } 33 | 34 | func GetGameAssets(ctx context.Context, version ark.Version, dst string, patterns []string) error { 35 | var err error 36 | 37 | if version == "" { 38 | version, err = GetLatestVersion() 39 | if err != nil { 40 | return errors.WithStack(err) 41 | } 42 | } 43 | 44 | infoList, err := GetInfoList(version) 45 | if err != nil { 46 | return errors.WithStack(err) 47 | } 48 | 49 | infoList, err = filter(infoList, patterns) 50 | if err != nil { 51 | return errors.WithStack(err) 52 | } 53 | 54 | err = get(ctx, infoList, dst) 55 | if err != nil { 56 | return errors.WithStack(err) 57 | } 58 | 59 | return nil 60 | } 61 | func UpdateGameAssets(ctx context.Context, oldResVer string, newResVer string, dst string, patterns []string) error { 62 | var err error 63 | 64 | if oldResVer == "" { 65 | return GetGameAssets(ctx, newResVer, dst, patterns) 66 | } 67 | if newResVer == "" { 68 | newResVer, err = GetLatestVersion() 69 | if err != nil { 70 | return errors.WithStack(err) 71 | } 72 | } 73 | 74 | oldInfoList, err := GetInfoList(oldResVer) 75 | if err != nil { 76 | return errors.WithStack(err) 77 | } 78 | newInfoList, err := GetInfoList(newResVer) 79 | if err != nil { 80 | return errors.WithStack(err) 81 | } 82 | 83 | oldInfoList, err = filter(oldInfoList, patterns) 84 | if err != nil { 85 | return errors.WithStack(err) 86 | } 87 | newInfoList, err = filter(newInfoList, patterns) 88 | if err != nil { 89 | return errors.WithStack(err) 90 | } 91 | 92 | infoList := CalculateDifferences(newInfoList, oldInfoList) 93 | err = get(ctx, infoList, dst) 94 | if err != nil { 95 | return errors.WithStack(err) 96 | } 97 | 98 | return nil 99 | } 100 | 101 | func GetLatestVersion() (string, error) { 102 | version, err := GetRawVersion() 103 | if err != nil { 104 | return "", errors.WithStack(err) 105 | } 106 | 107 | return version.ResVersion, nil 108 | } 109 | 110 | func GetInfoList(version ark.Version) ([]Info, error) { 111 | resources, err := GetRawResources(version) 112 | if err != nil { 113 | return nil, errors.WithStack(err) 114 | } 115 | 116 | infos := make([]Info, len(resources.AbInfos)) 117 | for i, info := range resources.AbInfos { 118 | if err != nil { 119 | return nil, errors.WithStack(err) 120 | } 121 | infos[i] = Info{ 122 | Name: info.Name, 123 | MD5: info.MD5, 124 | Version: version, 125 | } 126 | } 127 | return infos, nil 128 | } 129 | 130 | func filter(infoList []Info, patterns []string) ([]Info, error) { 131 | return cols.FilterErr(infoList, func(i Info) (bool, error) { 132 | return pathutil.MatchAny(patterns, i.Name) 133 | }) 134 | } 135 | 136 | func get(ctx context.Context, infoList []Info, dst string) error { 137 | fetch, err := fetch(ctx, infoList) 138 | if err != nil { 139 | return errors.WithStack(err) 140 | } 141 | defer os.RemoveAll(fetch) 142 | unzip, err := unzip(ctx, fetch) 143 | if err != nil { 144 | return errors.WithStack(err) 145 | } 146 | defer os.RemoveAll(unzip) 147 | unpack, err := unpack(ctx, unzip) 148 | if err != nil { 149 | return errors.WithStack(err) 150 | } 151 | defer os.RemoveAll(unpack) 152 | 153 | err = fileutil.MoveAllContent(unpack, dst) 154 | if err != nil { 155 | return errors.WithStack(err) 156 | } 157 | return nil 158 | } 159 | 160 | func CalculateDifferences(new []Info, old []Info) []Info { 161 | convert := func(i interface{}) interface{} { 162 | info := i.(Info) 163 | return struct { 164 | name string 165 | md5 string 166 | }{ 167 | name: info.Name, 168 | md5: info.MD5, 169 | } 170 | } 171 | newQuery := linq.From(new) 172 | oldQuery := linq.From(old) 173 | var result []Info 174 | newQuery. 175 | ExceptBy(oldQuery, convert). 176 | ToSlice(&result) 177 | return result 178 | } 179 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | logo 3 | 4 | # Arkwaifu (arkwaifu) 5 | 6 | [![](https://pkg.go.dev/badge/github.com/flandiayingman/arkwaifu.svg)](https://pkg.go.dev/github.com/flandiayingman/arkwaifu) 7 | ![](https://img.shields.io/github/license/FlandiaYingman/arkwaifu?style=flat-square) 8 | ![](https://img.shields.io/github/v/release/FlandiaYingman/arkwaifu?style=flat-square) 9 | 10 | ![](https://img.shields.io/github/actions/workflow/status/FlandiaYingman/arkwaifu/docker-image-service.yml?style=flat-square&label=build%3A%20service) 11 | ![](https://img.shields.io/github/actions/workflow/status/FlandiaYingman/arkwaifu/docker-image-updateloop.yml?style=flat-square&label=build%3A%20updateloop) 12 | ![](https://img.shields.io/website?style=flat-square&url=https%3A%2F%2Farkwaifu.cc%2F) 13 | 14 | Arkwaifu is a website which, arranges and provides almost all artworks extracted from Arknights (the game). Arkwaifu 15 | also enlarges (4x) the artworks with super-resolution neural networks - Real-ESRGAN. 16 | 17 | 🎉 Arkwaifu v1 has released! Check it at [arkwaifu.cc](https://arkwaifu.cc/)! 18 | 19 | 🇨🇳 The China mirror has been suspended due to the v0 → v1 upgrade. We will make our best efforts to get it back soon. 20 | 21 | For more information of the frontend of Arkwaifu, please refer to 22 | the [frontend repo](https://github.com/FlandiaYingman/arkwaifu-frontend). 23 | 24 | ## Features 25 | 26 | - The arts are automatically updated as soon as the game updates. 27 | - Assets are enlarged with super-resolution neural networks (Real-ESRGAN). 28 | 29 | ## Available Arts 30 | 31 | Now, only arts that appear in the "in-game stories", are available, including **images**, **backgrounds** and * 32 | *characters**. 33 | 34 | ### Images 35 | 36 | Images are the exquisite artworks that appear when some special events in the stories happen. 37 | 38 | 39 | 40 | 41 | ### Backgrounds 42 | 43 | Backgrounds are the artworks that always appear during dialogue between characters. 44 | 45 | 46 | 47 | ### Characters 48 | 49 | Characters are the artworks that represents characters that appear during dialogue. 50 | 51 | 52 | 53 | ## V1 Roadmap 54 | 55 | - [x] Switchable language. 56 | - [ ] Support searching. 57 | 58 | ## Acknowledgements 59 | 60 | Thanks to my friend [Galvin Gao](https://github.com/GalvinGao)! 61 | He helped me a lot in the front-end development and choosing frameworks. I really appreciate the "getting hands dirty" 62 | methodology very much. 63 | 64 | Thanks to my friend [Martin Wang](https://github.com/martinwang2002)! 65 | He helped me in extracting the gamedata assets, and also in some details of automatically updating the assets from the 66 | game. 67 | 68 | Thanks to my friend Lily! She drew the fascinating [Phantom logo](assets/arkwaifu_phantom.png) of this project. 69 | 70 | Thanks to [Penguin Statistics](https://penguin-stats.io/)! 71 | The prototype of this project referenced and is inspired by Penguin 72 | Statistics' [backend v3](https://github.com/penguin-statistics/backend-next). 73 | 74 | Thanks to [xinntao](https://github.com/xinntao), [nihui](https://github.com/nihui), and the other contributors 75 | of [Real-ESRGAN](https://github.com/xinntao/Real-ESRGAN) 76 | and [Real-CUGAN](https://github.com/bilibili/ailab/tree/main/Real-CUGAN)! They created the neural networks this project 77 | utilizes for enlarging assets. 78 | 79 | ## License 80 | 81 | The source code of this project is licensed under the [MIT License](LICENSE). 82 | 83 | The assets of this project are licensed under 84 | [Attribution-NonCommercial 4.0 International (CC BY-NC 4.0)](https://creativecommons.org/licenses/by-nc/4.0/). 85 | 86 | This project utilizes resources and other works from the game Arknights. The copyright of such works belongs to the 87 | provider of the game, 上海鹰角网络科技有限公司 (Shanghai Hypergryph Network Technology Co., Ltd). 88 | 89 | This project utilizes [Real-ESRGAN](https://github.com/xinntao/Real-ESRGAN) 90 | and [Real-ESRGAN-ncnn-vulkan](https://github.com/xinntao/Real-ESRGAN-ncnn-vulkan), which are respectively licensed under 91 | the BSD-3-Clause license and the MIT License. 92 | 93 | This project utilizes [Real-CUGAN](https://github.com/bilibili/ailab/tree/main/Real-CUGAN) 94 | and [Real-CUGAN-ncnn-vulkan](https://github.com/nihui/realcugan-ncnn-vulkan), which are both licensed under the MIT 95 | License. 96 | 97 | Some initial template source code of this project is inspired by and copied from 98 | the [backend v3](https://github.com/penguin-statistics/backend-next) of [Penguin Statistics](https://penguin-stats.io/), 99 | which is licensed under the [MIT License](https://github.com/penguin-statistics/backend-next/blob/dev/LICENSE). 100 | -------------------------------------------------------------------------------- /internal/app/updateloop/repo_version.go: -------------------------------------------------------------------------------- 1 | package updateloop 2 | 3 | import ( 4 | "context" 5 | "github.com/flandiayingman/arkwaifu/internal/app/infra" 6 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 7 | "gorm.io/gorm" 8 | "gorm.io/gorm/clause" 9 | ) 10 | 11 | type artVersion struct { 12 | Lock *int `gorm:"primaryKey;check:lock=0"` 13 | Version ark.Version `gorm:""` 14 | } 15 | 16 | type storyVersion struct { 17 | Server ark.Server `gorm:"primaryKey;type:game_server"` 18 | Version ark.Version `gorm:""` 19 | } 20 | 21 | type galleryVersion struct { 22 | Server ark.Server `gorm:"primaryKey;type:game_server"` 23 | Version ark.Version `gorm:""` 24 | } 25 | 26 | type repo struct { 27 | *infra.Gorm 28 | } 29 | 30 | var zeroPtr = func() *int { 31 | zero := 0 32 | return &zero 33 | }() 34 | 35 | func newRepo(db *infra.Gorm) (*repo, error) { 36 | var err error 37 | repo := &repo{db} 38 | err = repo.initArtVersionTable() 39 | if err != nil { 40 | return nil, err 41 | } 42 | err = repo.initStoryVersionTable() 43 | if err != nil { 44 | return nil, err 45 | } 46 | err = repo.initGalleryVersionTable() 47 | if err != nil { 48 | return nil, err 49 | } 50 | return repo, nil 51 | } 52 | 53 | func (r *repo) initArtVersionTable() error { 54 | err := r.AutoMigrate(&artVersion{}) 55 | if err != nil { 56 | return err 57 | } 58 | var result *gorm.DB 59 | result = r. 60 | Clauses(clause.OnConflict{DoNothing: true}). 61 | Create(&artVersion{ 62 | Lock: zeroPtr, 63 | Version: "", 64 | }) 65 | if result.Error != nil { 66 | return result.Error 67 | } 68 | return nil 69 | } 70 | 71 | func (r *repo) initStoryVersionTable() error { 72 | err := r.AutoMigrate(&storyVersion{}) 73 | if err != nil { 74 | return err 75 | } 76 | var result *gorm.DB 77 | result = r. 78 | Clauses(clause.OnConflict{DoNothing: true}). 79 | Create(&storyVersion{Server: ark.CnServer}) 80 | if result.Error != nil { 81 | return result.Error 82 | } 83 | result = r. 84 | Clauses(clause.OnConflict{DoNothing: true}). 85 | Create(&storyVersion{Server: ark.EnServer}) 86 | if result.Error != nil { 87 | return result.Error 88 | } 89 | result = r. 90 | Clauses(clause.OnConflict{DoNothing: true}). 91 | Create(&storyVersion{Server: ark.JpServer}) 92 | if result.Error != nil { 93 | return result.Error 94 | } 95 | result = r. 96 | Clauses(clause.OnConflict{DoNothing: true}). 97 | Create(&storyVersion{Server: ark.KrServer}) 98 | if result.Error != nil { 99 | return result.Error 100 | } 101 | result = r. 102 | Clauses(clause.OnConflict{DoNothing: true}). 103 | Create(&storyVersion{Server: ark.TwServer}) 104 | if result.Error != nil { 105 | return result.Error 106 | } 107 | return nil 108 | } 109 | func (r *repo) initGalleryVersionTable() error { 110 | err := r.AutoMigrate(&galleryVersion{}) 111 | if err != nil { 112 | return err 113 | } 114 | var result *gorm.DB 115 | result = r. 116 | Clauses(clause.OnConflict{DoNothing: true}). 117 | Create(&galleryVersion{Server: ark.CnServer}) 118 | if result.Error != nil { 119 | return result.Error 120 | } 121 | result = r. 122 | Clauses(clause.OnConflict{DoNothing: true}). 123 | Create(&galleryVersion{Server: ark.EnServer}) 124 | if result.Error != nil { 125 | return result.Error 126 | } 127 | result = r. 128 | Clauses(clause.OnConflict{DoNothing: true}). 129 | Create(&galleryVersion{Server: ark.JpServer}) 130 | if result.Error != nil { 131 | return result.Error 132 | } 133 | result = r. 134 | Clauses(clause.OnConflict{DoNothing: true}). 135 | Create(&galleryVersion{Server: ark.KrServer}) 136 | if result.Error != nil { 137 | return result.Error 138 | } 139 | result = r. 140 | Clauses(clause.OnConflict{DoNothing: true}). 141 | Create(&galleryVersion{Server: ark.TwServer}) 142 | if result.Error != nil { 143 | return result.Error 144 | } 145 | return nil 146 | } 147 | 148 | func (r *repo) upsertArtVersion(ctx context.Context, v *artVersion) error { 149 | return r. 150 | WithContext(ctx). 151 | Clauses(clause.OnConflict{UpdateAll: true}). 152 | Create(&v). 153 | Error 154 | } 155 | func (r *repo) upsertStoryVersion(ctx context.Context, v *storyVersion) error { 156 | return r. 157 | WithContext(ctx). 158 | Clauses(clause.OnConflict{UpdateAll: true}). 159 | Create(&v). 160 | Error 161 | } 162 | func (r *repo) upsertGalleryVersion(ctx context.Context, v *galleryVersion) error { 163 | return r. 164 | WithContext(ctx). 165 | Clauses(clause.OnConflict{UpdateAll: true}). 166 | Create(&v). 167 | Error 168 | } 169 | func (r *repo) selectArtVersion(ctx context.Context) (ark.Version, error) { 170 | var v artVersion 171 | err := r. 172 | WithContext(ctx). 173 | Take(&v). 174 | Error 175 | return v.Version, err 176 | } 177 | func (r *repo) selectStoryVersion(ctx context.Context, server ark.Server) (ark.Version, error) { 178 | var v storyVersion 179 | err := r. 180 | WithContext(ctx). 181 | Where("server = ?", server). 182 | Take(&v). 183 | Error 184 | return v.Version, err 185 | } 186 | func (r *repo) selectGalleryVersion(ctx context.Context, server ark.Server) (ark.Version, error) { 187 | var v galleryVersion 188 | err := r. 189 | WithContext(ctx). 190 | Where("server = ?", server). 191 | Take(&v). 192 | Error 193 | return v.Version, err 194 | } 195 | -------------------------------------------------------------------------------- /internal/app/art/controller.go: -------------------------------------------------------------------------------- 1 | package art 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/gofiber/fiber/v2" 7 | "github.com/google/uuid" 8 | ) 9 | 10 | type controller struct { 11 | service *Service 12 | } 13 | 14 | func newController(s *Service) *controller { 15 | c := controller{s} 16 | return &c 17 | } 18 | func registerController(c *controller, router fiber.Router) { 19 | if router == nil { 20 | return 21 | } 22 | 23 | router.Use("arts", c.Authenticator) 24 | 25 | router.Get("arts", c.GetArts) 26 | router.Get("arts/:id", c.GetArt) 27 | router.Put("arts/:id", c.PutArt) 28 | 29 | router.Put("arts/:id/variants/:variation", c.PutVariant) 30 | 31 | router.Get("arts/:id/variants/:variation/content", c.GetContent) 32 | router.Put("arts/:id/variants/:variation/content", c.PutContent) 33 | } 34 | 35 | func (c *controller) GetArts(ctx *fiber.Ctx) error { 36 | categoryString := ctx.Query("category") 37 | variationString := ctx.Query("absent-variation") 38 | var arts []*Art 39 | var err error 40 | if variationString != "" { 41 | variation, err := ParseVariation(variationString) 42 | if err != nil { 43 | return errors.Join(fiber.ErrBadRequest, err) 44 | } 45 | arts, err = c.service.SelectArtsWhoseVariantAbsent(variation) 46 | if err != nil { 47 | return err 48 | } 49 | } else if categoryString != "" { 50 | category, err := ParseCategory(categoryString) 51 | if err != nil { 52 | return errors.Join(fiber.ErrBadRequest, err) 53 | } 54 | arts, err = c.service.SelectArtsByCategory(category) 55 | if err != nil { 56 | return err 57 | } 58 | } else { 59 | arts, err = c.service.SelectArts() 60 | if err != nil { 61 | return err 62 | } 63 | } 64 | return ctx.JSON(arts) 65 | } 66 | func (c *controller) GetArt(ctx *fiber.Ctx) error { 67 | id := ctx.Params("id") 68 | 69 | art, err := c.service.SelectArt(id) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | return ctx.JSON(art) 75 | } 76 | func (c *controller) GetVariants(ctx *fiber.Ctx) error { 77 | id := ctx.Params("id") 78 | 79 | variants, err := c.service.SelectVariants(id) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | return ctx.JSON(variants) 85 | } 86 | func (c *controller) GetVariant(ctx *fiber.Ctx) error { 87 | id := ctx.Params("id") 88 | variation, err := ParseVariation(ctx.Params("variation")) 89 | if err != nil { 90 | return errors.Join(fiber.ErrBadRequest, err) 91 | } 92 | 93 | variants, err := c.service.SelectVariant(id, variation) 94 | if err != nil { 95 | return err 96 | } 97 | 98 | return ctx.JSON(variants) 99 | } 100 | 101 | func (c *controller) PutArt(ctx *fiber.Ctx) error { 102 | id := ctx.Params("id") 103 | 104 | art := new(Art) 105 | err := ctx.BodyParser(&art) 106 | if err != nil { 107 | return errors.Join(fiber.ErrBadRequest, err) 108 | } 109 | art.ID = id 110 | 111 | err = c.service.UpsertArts(art) 112 | if err != nil { 113 | return err 114 | } 115 | 116 | return ctx.SendStatus(fiber.StatusOK) 117 | } 118 | func (c *controller) PutVariant(ctx *fiber.Ctx) error { 119 | id := ctx.Params("id") 120 | variation := ctx.Params("variation") 121 | 122 | variant := Variant{} 123 | err := ctx.BodyParser(&variant) 124 | if err != nil { 125 | return errors.Join(fiber.ErrBadRequest, err) 126 | } 127 | variant.ArtID = id 128 | variant.Variation, err = ParseVariation(variation) 129 | if err != nil { 130 | return errors.Join(fiber.ErrBadRequest, err) 131 | } 132 | 133 | err = c.service.UpsertVariants(&variant) 134 | if err != nil { 135 | return err 136 | } 137 | 138 | return ctx.SendStatus(fiber.StatusOK) 139 | } 140 | 141 | type ContentParams struct { 142 | ID string `param:"id"` 143 | Variation Variation `param:"variation"` 144 | } 145 | 146 | func (c *controller) GetContent(ctx *fiber.Ctx) error { 147 | params := ContentParams{} 148 | err := ctx.ParamsParser(¶ms) 149 | if err != nil { 150 | return err 151 | } 152 | 153 | content, err := c.service.TakeContent(params.ID, params.Variation) 154 | if err != nil { 155 | return err 156 | } 157 | 158 | ctx.Type("webp") 159 | return ctx.Send(content) 160 | } 161 | func (c *controller) PutContent(ctx *fiber.Ctx) error { 162 | params := ContentParams{} 163 | err := ctx.ParamsParser(¶ms) 164 | if err != nil { 165 | return err 166 | } 167 | 168 | err = c.service.StoreContent(params.ID, params.Variation, ctx.Body()) 169 | if err != nil { 170 | return err 171 | } 172 | 173 | return nil 174 | } 175 | 176 | type variantQueryFilter struct { 177 | } 178 | 179 | func (c *controller) Authenticator(ctx *fiber.Ctx) error { 180 | if c.SkipAuthentication(ctx) { 181 | return ctx.Next() 182 | } 183 | return c.Authenticate(ctx) 184 | } 185 | func (c *controller) SkipAuthentication(ctx *fiber.Ctx) bool { 186 | return ctx.Method() != fiber.MethodPut 187 | } 188 | 189 | func (c *controller) Authenticate(ctx *fiber.Ctx) error { 190 | idStr := ctx.Query("user", "") 191 | if idStr == "" { 192 | return ctx. 193 | Status(fiber.StatusUnauthorized). 194 | SendString("no user credential provided") 195 | } 196 | id, err := uuid.Parse(idStr) 197 | if err != nil { 198 | return ctx. 199 | Status(fiber.StatusBadRequest). 200 | SendString(fmt.Sprintf("cannot parse id of user: %s", idStr)) 201 | } 202 | user := c.service.Authenticate(id) 203 | if user == nil { 204 | return ctx. 205 | Status(fiber.StatusUnauthorized). 206 | SendString(fmt.Sprintf("cannot find user with id: %s", id)) 207 | } 208 | return ctx.Next() 209 | } 210 | -------------------------------------------------------------------------------- /internal/app/story/repo.go: -------------------------------------------------------------------------------- 1 | package story 2 | 3 | import ( 4 | "github.com/flandiayingman/arkwaifu/internal/app/infra" 5 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 6 | "gorm.io/gorm" 7 | ) 8 | 9 | type repo struct { 10 | db *infra.Gorm 11 | } 12 | 13 | func (r *repo) init() (err error) { 14 | err = r.db.CreateEnum("game_server", "CN", "EN", "JP", "KR", "TW") 15 | if err != nil { 16 | return err 17 | } 18 | err = r.db.CreateEnum("story_tag", "before", "after", "interlude") 19 | if err != nil { 20 | return err 21 | } 22 | err = r.db.CreateEnum("story_group_type", "main-story", "major-event", "minor-event", "other") 23 | if err != nil { 24 | return err 25 | } 26 | err = r.db.AutoMigrate(&Group{}) 27 | if err != nil { 28 | return err 29 | } 30 | err = r.db.AutoMigrate(&Story{}) 31 | if err != nil { 32 | return err 33 | } 34 | err = r.db.AutoMigrate(&PictureArt{}) 35 | if err != nil { 36 | return err 37 | } 38 | err = r.db.AutoMigrate(&CharacterArt{}) 39 | if err != nil { 40 | return err 41 | } 42 | return nil 43 | } 44 | 45 | func newRepo(db *infra.Gorm) (*repo, error) { 46 | r := repo{db} 47 | err := r.init() 48 | if err != nil { 49 | return nil, err 50 | } 51 | return &r, nil 52 | } 53 | 54 | func (r *repo) SelectStories(server ark.Server) ([]*Story, error) { 55 | stories := make([]*Story, 0) 56 | result := r.db. 57 | Preload("PictureArts", func(db *gorm.DB) *gorm.DB { return db.Order("picture_arts.sort_id") }). 58 | Preload("CharacterArts", func(db *gorm.DB) *gorm.DB { return db.Order("character_arts.sort_id") }). 59 | Order("sort_id"). 60 | Find(&stories, "server = ?", server) 61 | return stories, result.Error 62 | } 63 | func (r *repo) SelectStory(id string, server ark.Server) (*Story, error) { 64 | story := &Story{} 65 | result := r.db. 66 | Preload("PictureArts", func(db *gorm.DB) *gorm.DB { return db.Order("picture_arts.sort_id") }). 67 | Preload("CharacterArts", func(db *gorm.DB) *gorm.DB { return db.Order("character_arts.sort_id") }). 68 | Take(&story, "(id, server) = (?, ?)", id, server) 69 | return story, result.Error 70 | } 71 | func (r *repo) SelectStoryGroups(server ark.Server) ([]*Group, error) { 72 | groups := make([]*Group, 0) 73 | result := r.db. 74 | Preload("Stories", func(db *gorm.DB) *gorm.DB { return db.Order("stories.sort_id") }). 75 | Preload("Stories.PictureArts", func(db *gorm.DB) *gorm.DB { return db.Order("picture_arts.sort_id") }). 76 | Preload("Stories.CharacterArts", func(db *gorm.DB) *gorm.DB { return db.Order("character_arts.sort_id") }). 77 | Order("sort_id"). 78 | Find(&groups, "server = ?", server) 79 | return groups, result.Error 80 | } 81 | func (r *repo) SelectStoryGroupsByType(server ark.Server, groupType GroupType) ([]*Group, error) { 82 | groups := make([]*Group, 0) 83 | result := r.db. 84 | Preload("Stories", func(db *gorm.DB) *gorm.DB { return db.Order("stories.sort_id") }). 85 | Preload("Stories.PictureArts", func(db *gorm.DB) *gorm.DB { return db.Order("picture_arts.sort_id") }). 86 | Preload("Stories.CharacterArts", func(db *gorm.DB) *gorm.DB { return db.Order("character_arts.sort_id") }). 87 | Order("sort_id"). 88 | Find(&groups, "(server, type) = (?, ?)", server, groupType) 89 | return groups, result.Error 90 | } 91 | func (r *repo) SelectStoryGroup(id string, server ark.Server) (*Group, error) { 92 | group := &Group{} 93 | result := r.db. 94 | Preload("Stories", func(db *gorm.DB) *gorm.DB { return db.Order("stories.sort_id") }). 95 | Preload("Stories.PictureArts", func(db *gorm.DB) *gorm.DB { return db.Order("picture_arts.sort_id") }). 96 | Preload("Stories.CharacterArts", func(db *gorm.DB) *gorm.DB { return db.Order("character_arts.sort_id") }). 97 | Take(&group, "(id, server) = (?, ?)", id, server) 98 | return group, result.Error 99 | } 100 | 101 | func (r *repo) SelectPictureArts(server ark.Server) ([]*PictureArt, error) { 102 | arts := make([]*PictureArt, 0) 103 | return arts, r.db. 104 | Order("sort_id"). 105 | Find(&arts, "server = ?", server).Error 106 | } 107 | func (r *repo) SelectAggregatedPictureArtByID(server ark.Server, id string) (*AggregatedPictureArt, error) { 108 | art := &AggregatedPictureArt{} 109 | err := r.db. 110 | Model(&PictureArt{}). 111 | Take(&art, "(server, id) = (?, ?)", server, id). 112 | Error 113 | if err != nil { 114 | return nil, err 115 | } 116 | return art, nil 117 | } 118 | func (r *repo) SelectCharacterArts(server ark.Server) ([]*CharacterArt, error) { 119 | arts := make([]*CharacterArt, 0) 120 | return arts, r.db. 121 | Order("sort_id"). 122 | Find(&arts, "server = ?", server).Error 123 | } 124 | func (r *repo) SelectAggregatedCharacterArtByID(server ark.Server, id string) (*AggregatedCharacterArt, error) { 125 | var err error 126 | art := &AggregatedCharacterArt{} 127 | 128 | err = r.db. 129 | Model(&CharacterArt{}). 130 | Take(&art, "(server, id) = (?, ?)", server, id). 131 | Error 132 | if err != nil { 133 | return nil, err 134 | } 135 | 136 | err = r.db. 137 | Table("(?) as dt(aggregated_names)", 138 | r.db. 139 | Model(&CharacterArt{}). 140 | Select("unnest(names)"). 141 | Where("(server, id) = (?, ?)", server, id), 142 | ). 143 | Select("array_agg(DISTINCT aggregated_names) as names"). 144 | Scan(&art). 145 | Error 146 | if err != nil { 147 | return nil, err 148 | } 149 | 150 | return art, nil 151 | } 152 | 153 | func (r *repo) UpsertStoryGroups(groups []Group) error { 154 | // UpsertStoryGroups does not actually upsert values. 155 | // Because of the existence of SortID (we want to re-generate it), values are deleted and then inserted. 156 | return r.db.Transaction(func(tx *gorm.DB) error { 157 | var err error 158 | err = tx.Delete(&groups).Error 159 | if err != nil { 160 | return err 161 | } 162 | err = tx.Create(&groups).Error 163 | if err != nil { 164 | return err 165 | } 166 | return nil 167 | }) 168 | } 169 | -------------------------------------------------------------------------------- /internal/app/story/service.go: -------------------------------------------------------------------------------- 1 | package story 2 | 3 | import ( 4 | "fmt" 5 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 6 | arkparser2 "github.com/flandiayingman/arkwaifu/internal/pkg/arkparser" 7 | cols "github.com/flandiayingman/arkwaifu/internal/pkg/cols" 8 | "strings" 9 | ) 10 | 11 | type Service struct { 12 | r *repo 13 | } 14 | 15 | func newService(r *repo) *Service { 16 | return &Service{r: r} 17 | } 18 | 19 | type GroupFilter struct { 20 | Type string 21 | } 22 | 23 | func (s *Service) GetStories(server ark.Server) ([]*Story, error) { 24 | return s.r.SelectStories(server) 25 | } 26 | func (s *Service) GetStory(server ark.Server, id string) (*Story, error) { 27 | return s.r.SelectStory(id, server) 28 | } 29 | func (s *Service) GetStoryGroups(server ark.Server, filter GroupFilter) ([]*Group, error) { 30 | if filter.Type != "" { 31 | return s.r.SelectStoryGroupsByType(server, filter.Type) 32 | } 33 | return s.r.SelectStoryGroups(server) 34 | } 35 | func (s *Service) GetStoryGroup(server ark.Server, id string) (*Group, error) { 36 | return s.r.SelectStoryGroup(id, server) 37 | } 38 | 39 | func (s *Service) GetPictureArts(server ark.Server) ([]*PictureArt, error) { 40 | return s.r.SelectPictureArts(server) 41 | } 42 | func (s *Service) GetAggregatedPictureArt(server ark.Server, id string) (*AggregatedPictureArt, error) { 43 | return s.r.SelectAggregatedPictureArtByID(server, id) 44 | } 45 | 46 | func (s *Service) GetCharacterArts(server ark.Server) ([]*CharacterArt, error) { 47 | return s.r.SelectCharacterArts(server) 48 | } 49 | func (s *Service) GetAggregatedCharacterArt(server ark.Server, id string) (*AggregatedCharacterArt, error) { 50 | return s.r.SelectAggregatedCharacterArtByID(server, id) 51 | } 52 | 53 | func (s *Service) PopulateFrom(rawTree *arkparser2.StoryTree, server ark.Server) error { 54 | converter := objectConverter{server: server} 55 | 56 | tree, err := converter.convertStoryTree(rawTree) 57 | if err != nil { 58 | return fmt.Errorf("converter error when populating story service: %w", err) 59 | } 60 | 61 | for _, group := range tree { 62 | // Prevent error: extended protocol limited to 65535 parameters 63 | // TODO: Find a more efficient way to prevent error. 64 | err = s.r.UpsertStoryGroups([]Group{group}) 65 | if err != nil { 66 | return err 67 | } 68 | } 69 | 70 | return nil 71 | } 72 | 73 | type objectConverter struct { 74 | server ark.Server 75 | } 76 | 77 | func (c *objectConverter) convertStoryTree(tree *arkparser2.StoryTree) (Tree, error) { 78 | return cols.MapErr(tree.StoryGroups, c.convertStoryGroup) 79 | } 80 | func (c *objectConverter) convertStoryGroup(rawGroup *arkparser2.StoryGroup) (Group, error) { 81 | group := Group{ 82 | Server: c.server, 83 | ID: strings.ToLower(rawGroup.ID), 84 | Name: rawGroup.Name, 85 | Type: "", 86 | Stories: nil, 87 | SortID: nil, // SortID is auto-increment, no need to handle it. 88 | } 89 | 90 | var err error 91 | 92 | group.Type, err = c.convertGroupType(rawGroup.Type) 93 | if err != nil { 94 | return Group{}, err 95 | } 96 | 97 | group.Stories, err = cols.MapErr(rawGroup.Stories, c.convertStory) 98 | if err != nil { 99 | return Group{}, err 100 | } 101 | 102 | return group, nil 103 | } 104 | func (c *objectConverter) convertGroupType(rawType string) (GroupType, error) { 105 | switch rawType { 106 | case arkparser2.TypeMain: 107 | return GroupTypeMainStory, nil 108 | case arkparser2.TypeActivity: 109 | return GroupTypeMajorEvent, nil 110 | case arkparser2.TypeMiniActivity: 111 | return GroupTypeMinorEvent, nil 112 | case arkparser2.TypeNone: 113 | return GroupTypeOther, nil 114 | default: 115 | return "", fmt.Errorf("unknown story group type: %v", rawType) 116 | } 117 | } 118 | func (c *objectConverter) convertStory(rawStory *arkparser2.Story) (Story, error) { 119 | story := Story{ 120 | Server: c.server, 121 | ID: strings.ToLower(rawStory.ID), 122 | Tag: "", 123 | TagText: rawStory.TagText, 124 | Code: rawStory.Code, 125 | Name: rawStory.Name, 126 | Info: rawStory.Info, 127 | GroupID: rawStory.GroupID, 128 | SortID: nil, // SortID is auto-increment, no need to handle it. 129 | PictureArts: cols.Map(rawStory.Pictures, c.convertPictureArt), 130 | CharacterArts: cols.Map(rawStory.Characters, c.convertCharacterArt), 131 | } 132 | 133 | var err error 134 | 135 | story.Tag, err = c.convertStoryTagType(rawStory.TagType) 136 | if err != nil { 137 | return Story{}, err 138 | } 139 | 140 | return story, nil 141 | } 142 | func (c *objectConverter) convertStoryTagType(rawType string) (Tag, error) { 143 | switch rawType { 144 | case arkparser2.TagBefore: 145 | return TagBefore, nil 146 | case arkparser2.TagAfter: 147 | return TagAfter, nil 148 | case arkparser2.TagInterlude: 149 | return TagInterlude, nil 150 | default: 151 | return "", fmt.Errorf("unknown arkparser story group type: %v", rawType) 152 | } 153 | } 154 | func (c *objectConverter) convertPictureArt(rawPicture *arkparser2.StoryPicture) PictureArt { 155 | return PictureArt{ 156 | Server: c.server, 157 | ID: strings.ToLower(rawPicture.ID), 158 | StoryID: "", // This will be auto-generated by the ORM framework 159 | Category: rawPicture.Category, 160 | 161 | Title: rawPicture.Title, 162 | Subtitle: rawPicture.Subtitle, 163 | 164 | SortID: nil, // SortID is auto-increment, no need to handle it. 165 | } 166 | } 167 | func (c *objectConverter) convertCharacterArt(rawCharacter *arkparser2.StoryCharacter) CharacterArt { 168 | return CharacterArt{ 169 | Server: c.server, 170 | ID: strings.ToLower(rawCharacter.ID), 171 | StoryID: "", // This will be auto-generated by the ORM framework 172 | Category: "character", 173 | 174 | Names: rawCharacter.Names, 175 | 176 | SortID: nil, // SortID is auto-increment, no need to handle it. 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /internal/pkg/arkdata/data.go: -------------------------------------------------------------------------------- 1 | package arkdata 2 | 3 | import ( 4 | "context" 5 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 6 | "github.com/flandiayingman/arkwaifu/internal/pkg/util/fileutil" 7 | "github.com/google/go-github/v52/github" 8 | "github.com/pkg/errors" 9 | "os" 10 | "regexp" 11 | ) 12 | 13 | const ( 14 | RepoOwner = "Kengxxiao" 15 | RepoName = "ArknightsGameData" 16 | CompositeRepoOwner = "FlandiaYingman" 17 | CompositeRepoName = "ArknightsGameDataComposite" 18 | DefaultPrefix = "assets/torappu/dynamicassets" 19 | ) 20 | 21 | var ( 22 | githubClient = github.NewClient(nil) 23 | 24 | // RepoCommitMessageRegex is for matching that a commit is a resource-updating commit. 25 | // 26 | // Examples: 27 | // - [CN UPDATE] Client:2.0.01 Data:23-05-11-16-35-19-8a6fe7 [BOT TEST] - Match: CN, 2.0.01, 23-05-11-16-35-19-8a6fe7 28 | // - [EN UPDATE] Client:15.9.01 Data:23-04-25-10-10-55-972129 - Match: EN, 15.9.01, 23-04-25-10-10-55-972129 29 | RepoCommitMessageRegex = regexp.MustCompile("\\[(CN|EN|JP|KR) UPDATE] Client:([.\\d]*) Data:([-\\w]*)") 30 | ) 31 | 32 | type DataInfo struct { 33 | Name string 34 | } 35 | 36 | type DataVersion struct { 37 | GameServer ark.Server 38 | ClientVersion string 39 | ResourceVersion string 40 | CommitSHA string 41 | } 42 | 43 | func GetGameData(ctx context.Context, server ark.Server, version ark.Version, patterns []string, dst string) error { 44 | return getGameData(ctx, server, version, RepoOwner, RepoName, patterns, dst) 45 | } 46 | func GetLatestDataVersion(ctx context.Context, server ark.Server) (*DataVersion, error) { 47 | return getLatestDataVersion(ctx, server, RepoOwner, RepoName) 48 | } 49 | func GetDataVersion(ctx context.Context, server ark.Server, resVer string) (*DataVersion, error) { 50 | return getDataVersion(ctx, server, resVer, RepoOwner, RepoName) 51 | } 52 | 53 | func GetCompositeGameData(ctx context.Context, server ark.Server, version ark.Version, patterns []string, dst string) error { 54 | return getGameData(ctx, server, version, CompositeRepoOwner, CompositeRepoName, patterns, dst) 55 | } 56 | func GetLatestCompositeDataVersion(ctx context.Context, server ark.Server) (*DataVersion, error) { 57 | return getLatestDataVersion(ctx, server, CompositeRepoOwner, CompositeRepoName) 58 | } 59 | func GetCompositeDataVersion(ctx context.Context, server ark.Server, resVer string) (*DataVersion, error) { 60 | return getDataVersion(ctx, server, resVer, CompositeRepoOwner, CompositeRepoName) 61 | } 62 | 63 | func getGameData(ctx context.Context, server ark.Server, version ark.Version, repoOwner string, repoName string, patterns []string, dst string) error { 64 | var sha string 65 | if version != "" { 66 | dataVersion, err := getDataVersion(ctx, server, version, repoOwner, repoName) 67 | if err != nil { 68 | return errors.WithStack(err) 69 | } 70 | sha = dataVersion.CommitSHA 71 | } else { 72 | dataVersion, err := getLatestDataVersion(ctx, server, repoOwner, repoName) 73 | if err != nil { 74 | return errors.WithStack(err) 75 | } 76 | sha = dataVersion.CommitSHA 77 | } 78 | 79 | zipballDir, zipball, err := download(ctx, repoOwner, repoName, sha) 80 | if err != nil { 81 | return errors.WithStack(err) 82 | } 83 | defer os.RemoveAll(zipballDir) 84 | defer os.RemoveAll(zipball) 85 | 86 | data, err := unzip(ctx, zipball, patterns, server) 87 | if err != nil { 88 | return errors.WithStack(err) 89 | } 90 | defer os.RemoveAll(data) 91 | 92 | err = fileutil.MoveAllContent(data, dst) 93 | if err != nil { 94 | return errors.WithStack(err) 95 | } 96 | 97 | return nil 98 | } 99 | func getLatestDataVersion(ctx context.Context, server ark.Server, repoOwner string, repoName string) (*DataVersion, error) { 100 | commit, err := findCommit(ctx, repoOwner, repoName, func(commit *github.RepositoryCommit) bool { 101 | version := parseCommit(commit) 102 | return version != nil && version.GameServer == server 103 | }) 104 | if err != nil { 105 | return nil, errors.Wrapf(err, "cannot find latest DataVersion") 106 | } 107 | return parseCommit(commit), nil 108 | } 109 | func getDataVersion(ctx context.Context, server ark.Server, resVer string, repoOwner string, repoName string) (*DataVersion, error) { 110 | commit, err := findCommit(ctx, repoOwner, repoName, func(commit *github.RepositoryCommit) bool { 111 | version := parseCommit(commit) 112 | return version != nil && version.GameServer == server && version.ResourceVersion == resVer 113 | }) 114 | if err != nil { 115 | return nil, errors.Wrapf(err, "cannot find DataVersion by %s", resVer) 116 | } 117 | if commit == nil { 118 | return nil, errors.Errorf("no DataVersion can be found by %s", resVer) 119 | } 120 | return parseCommit(commit), nil 121 | } 122 | 123 | func parseCommit(commit *github.RepositoryCommit) *DataVersion { 124 | matches := RepoCommitMessageRegex.FindStringSubmatch(commit.GetCommit().GetMessage()) 125 | if matches == nil { 126 | return nil 127 | } else { 128 | return &DataVersion{ 129 | GameServer: ark.MustParseServer(matches[1]), 130 | ClientVersion: matches[2], 131 | ResourceVersion: matches[3], 132 | CommitSHA: commit.GetSHA(), 133 | } 134 | } 135 | } 136 | func findCommit(ctx context.Context, owner, repo string, predicate func(*github.RepositoryCommit) bool) (*github.RepositoryCommit, error) { 137 | const perPage = 100 138 | var page = 1 139 | var opts = &github.CommitsListOptions{} 140 | for { 141 | opts.PerPage = perPage 142 | opts.Page = page 143 | page++ 144 | 145 | commits, _, err := githubClient.Repositories.ListCommits(ctx, owner, repo, opts) 146 | if err != nil { 147 | return nil, errors.WithStack(err) 148 | } 149 | 150 | for _, commit := range commits { 151 | if predicate(commit) { 152 | return commit, nil 153 | } 154 | } 155 | 156 | if len(commits) < perPage { 157 | break 158 | } 159 | } 160 | return nil, nil 161 | } 162 | -------------------------------------------------------------------------------- /internal/app/updateloop/update_gallery.go: -------------------------------------------------------------------------------- 1 | package updateloop 2 | 3 | import ( 4 | "context" 5 | "github.com/flandiayingman/arkwaifu/internal/app/gallery" 6 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 7 | "github.com/flandiayingman/arkwaifu/internal/pkg/arkdata" 8 | "github.com/flandiayingman/arkwaifu/internal/pkg/arkjson" 9 | "github.com/pkg/errors" 10 | "github.com/rs/zerolog/log" 11 | "os" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | var ( 17 | galleryPatterns = []string{ 18 | "gamedata/excel/story_review_meta_table.json", 19 | "gamedata/excel/replicate_table.json", 20 | "gamedata/excel/retro_table.json", 21 | "gamedata/excel/roguelike_topic_table.json", 22 | } 23 | ) 24 | 25 | func (s *Service) getRemoteGalleryVersion(ctx context.Context, server ark.Server) (ark.Version, error) { 26 | v, err := arkdata.GetLatestDataVersion(ctx, server) 27 | return v.ResourceVersion, err 28 | } 29 | 30 | func (s *Service) getLocalGalleryVersion(ctx context.Context, server ark.Server) (ark.Version, error) { 31 | return s.repo.selectGalleryVersion(ctx, server) 32 | } 33 | 34 | func (s *Service) attemptUpdateGalleries(ctx context.Context, server ark.Server) { 35 | log := log.With(). 36 | Str("server", server). 37 | Logger() 38 | log.Info().Msg("Attempting to update galleries of the server. ") 39 | localVersion, err := s.getLocalGalleryVersion(ctx, server) 40 | if err != nil { 41 | log.Err(err). 42 | Msg("Failed to get the local gallery version of the server. ") 43 | return 44 | } 45 | remoteVersion, err := s.getRemoteGalleryVersion(ctx, server) 46 | if err != nil { 47 | log.Err(err). 48 | Msg("Failed to get the remote gallery version of the server. ") 49 | return 50 | } 51 | log = log.With(). 52 | Str("localVersion", localVersion). 53 | Str("remoteVersion", remoteVersion). 54 | Logger() 55 | if localVersion != remoteVersion { 56 | log.Info().Msg("Updating the stories of the server, since the gallery versions are not identical. ") 57 | begin := time.Now() 58 | err = s.updateGalleries(ctx, server, remoteVersion) 59 | if err != nil { 60 | log.Err(err).Msg("Update loop failed to update the stories") 61 | } 62 | log.Info(). 63 | Str("elapsed", time.Since(begin).String()). 64 | Msg("Updated the stories of the server successfully. ") 65 | } else { 66 | log.Info().Msg("Skip updating the stories of the server, since the gallery versions are identical. ") 67 | } 68 | } 69 | 70 | func (s *Service) updateGalleries(ctx context.Context, server ark.Server, version ark.Version) error { 71 | root, err := os.MkdirTemp("", "arkwaifu-updateloop-gallery-*") 72 | if err != nil { 73 | return err 74 | } 75 | defer os.RemoveAll(root) 76 | 77 | err = arkdata.GetGameData(ctx, server, version, galleryPatterns, root) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | galleries, err := ParseToGalleries(server, root) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | err = s.galleryService.Put(galleries) 88 | if err != nil { 89 | return errors.WithStack(err) 90 | } 91 | 92 | err = s.repo.upsertGalleryVersion(ctx, &galleryVersion{ 93 | Server: server, 94 | Version: version, 95 | }) 96 | if err != nil { 97 | return err 98 | } 99 | 100 | return nil 101 | } 102 | 103 | func ParseToGalleries(server ark.Server, root string) ([]gallery.Gallery, error) { 104 | jsonStoryReviewMetaTable, err := arkjson.Get(root, arkjson.StoryReviewMetaTablePath) 105 | if err != nil { 106 | return nil, err 107 | } 108 | jsonRetroTable, err := arkjson.Get(root, arkjson.RetroTable) 109 | if err != nil { 110 | return nil, err 111 | } 112 | jsonReplicateTable, err := arkjson.Get(root, arkjson.ReplicateTable) 113 | if err != nil { 114 | return nil, err 115 | } 116 | jsonRoguelikeTopicTable, err := arkjson.Get(root, arkjson.RoguelikeTopicTable) 117 | if err != nil { 118 | return nil, err 119 | } 120 | 121 | artMap := make(map[string]gallery.Art) 122 | for _, c := range jsonStoryReviewMetaTable.S("actArchiveResData", "pics").Children() { 123 | artMap[strings.ToLower(c.S("id").Data().(string))] = gallery.Art{ 124 | Server: server, 125 | GalleryID: "", // Auto Generated 126 | SortID: 0, 127 | ID: strings.ToLower(c.S("id").Data().(string)), 128 | Name: c.S("desc").Data().(string), 129 | Description: c.S("picDescription").Data().(string), 130 | ArtID: strings.ToLower(c.S("assetPath").Data().(string)), 131 | } 132 | } 133 | 134 | galleryMap := make(map[string]gallery.Gallery) 135 | for _, c := range jsonRetroTable.S("retroActList").Children() { 136 | for _, actID := range c.S("linkedActId").Children() { 137 | galleryMap[strings.ToLower(actID.Data().(string))] = gallery.Gallery{ 138 | Server: server, 139 | ID: strings.ToLower(actID.Data().(string)), 140 | Name: c.S("name").Data().(string), 141 | Description: c.S("detail").Data().(string), 142 | Arts: nil, 143 | } 144 | } 145 | } 146 | for _, c := range jsonRoguelikeTopicTable.S("topics").Children() { 147 | galleryMap[strings.ToLower(c.S("id").Data().(string))] = gallery.Gallery{ 148 | Server: server, 149 | ID: strings.ToLower(c.S("id").Data().(string)), 150 | Name: c.S("name").Data().(string), 151 | Description: c.S("lineText").Data().(string), 152 | Arts: nil, 153 | } 154 | } 155 | 156 | galleries := make([]gallery.Gallery, 0) 157 | for id, c := range jsonStoryReviewMetaTable.Search("actArchiveData", "components").ChildrenMap() { 158 | if jsonReplicateTable.Exists(id) { 159 | continue 160 | } 161 | if gallery, ok := galleryMap[strings.ToLower(id)]; ok { 162 | for _, pic := range c.S("pic", "pics").Children() { 163 | art := artMap[strings.ToLower(pic.S("picId").Data().(string))] 164 | art.SortID = int(pic.S("picSortId").Data().(float64)) 165 | gallery.Arts = append(gallery.Arts, art) 166 | } 167 | galleries = append(galleries, gallery) 168 | } 169 | } 170 | 171 | return galleries, nil 172 | } 173 | -------------------------------------------------------------------------------- /internal/pkg/arkprocessor/processor_character.go: -------------------------------------------------------------------------------- 1 | package arkprocessor 2 | 3 | import ( 4 | "fmt" 5 | _ "github.com/chai2010/webp" 6 | "github.com/disintegration/imaging" 7 | "github.com/flandiayingman/arkwaifu/internal/pkg/arkscanner" 8 | "github.com/pkg/errors" 9 | "image" 10 | "image/color" 11 | "image/draw" 12 | _ "image/jpeg" 13 | _ "image/png" 14 | "math" 15 | "os" 16 | "path/filepath" 17 | ) 18 | 19 | type CharacterArt arkscanner.CharacterArt 20 | 21 | type CharacterArtImage struct { 22 | Image image.Image 23 | Art *CharacterArt 24 | BodyNum int 25 | FaceNum int 26 | } 27 | 28 | func (i *CharacterArtImage) ID() string { 29 | return fmt.Sprintf("%s#%d$%d", i.Art.ID, i.FaceNum, i.BodyNum) 30 | } 31 | 32 | // ProcessCharacterArt process a character art. 33 | // 34 | // Since picture arts are complicated, the process operation consists of: 35 | // - Decode all relevant images, including color channel and alpha channel of 36 | // faces and bodies image. 37 | // - For each face or body, merge the alpha channels onto the color channels. 38 | // - For each face, merge it onto its corresponding body. 39 | func (p *Processor) ProcessCharacterArt(art *CharacterArt) ([]CharacterArtImage, error) { 40 | var result []CharacterArtImage 41 | for i, body := range art.BodyVariations { 42 | bodyImage, err := art.decode(p.Root, i+1, 0) 43 | if err != nil { 44 | return nil, errors.WithStack(err) 45 | } 46 | 47 | // Sometimes, there are some character arts that have no actual images given. We'll just skip them. 48 | // Hope HyperGryph will remember what they've removed and remember to remove them from the stories. 49 | // Thanks to HyperGryph ^^ 50 | if bodyImage == nil { 51 | continue 52 | } 53 | 54 | if body.PixelToUnits != 1.0 { 55 | bodyImage = imaging.Resize( 56 | bodyImage, 57 | int(math.Round(float64(bodyImage.Bounds().Dx())/body.PixelToUnits)), 58 | int(math.Round(float64(bodyImage.Bounds().Dy())/body.PixelToUnits)), 59 | imaging.Lanczos, 60 | ) 61 | } 62 | 63 | for j, face := range body.FaceVariations { 64 | faceImage, err := art.decode(p.Root, i+1, j+1) 65 | if err != nil { 66 | return nil, errors.WithStack(err) 67 | } 68 | 69 | // Same reason above. 70 | // Thanks to HyperGryph ^^ 71 | if faceImage == nil { 72 | continue 73 | } 74 | 75 | if face.PixelToUnits != 1.0 { 76 | faceImage = imaging.Resize( 77 | faceImage, 78 | int(math.Round(float64(faceImage.Bounds().Dx())/face.PixelToUnits)), 79 | int(math.Round(float64(faceImage.Bounds().Dy())/face.PixelToUnits)), 80 | imaging.Lanczos, 81 | ) 82 | } 83 | 84 | if !face.WholeBody { 85 | faceImage, err = mergeCharacterFace(bodyImage, faceImage, body.FaceRectangle) 86 | if err != nil { 87 | return nil, errors.WithStack(err) 88 | } 89 | } 90 | result = append(result, CharacterArtImage{ 91 | Image: faceImage, 92 | Art: art, 93 | BodyNum: i + 1, 94 | FaceNum: j + 1, 95 | }) 96 | } 97 | } 98 | return result, nil 99 | } 100 | 101 | func (a *CharacterArt) decode(root string, bodyNum, faceNum int) (image.Image, error) { 102 | var filePath, filePathAlpha string 103 | if faceNum > 0 { 104 | filePath = filepath.Join(root, (*arkscanner.CharacterArt)(a).FacePath(bodyNum, faceNum)) 105 | filePathAlpha = filepath.Join(root, (*arkscanner.CharacterArt)(a).FacePathAlpha(bodyNum, faceNum)) 106 | } else { 107 | filePath = filepath.Join(root, (*arkscanner.CharacterArt)(a).BodyPath(bodyNum)) 108 | filePathAlpha = filepath.Join(root, (*arkscanner.CharacterArt)(a).BodyPathAlpha(bodyNum)) 109 | } 110 | if filePath == root { 111 | return nil, nil 112 | } 113 | 114 | var img, imgAlpha image.Image 115 | var err error 116 | img, err = decode(filePath) 117 | if err != nil { 118 | return nil, errors.WithStack(err) 119 | } 120 | if filePathAlpha != root { 121 | imgAlpha, err = decode(filePathAlpha) 122 | if err != nil { 123 | return nil, errors.WithStack(err) 124 | } 125 | } 126 | 127 | if imgAlpha != nil { 128 | img, err = mergeAlphaChannel(img, imgAlpha) 129 | if err != nil { 130 | return nil, errors.WithStack(err) 131 | } 132 | } 133 | 134 | return img, nil 135 | } 136 | 137 | func decode(filePath string) (image.Image, error) { 138 | file, err := os.Open(filePath) 139 | if err != nil { 140 | return nil, errors.WithStack(err) 141 | } 142 | defer func() { _ = file.Close() }() 143 | 144 | img, _, err := image.Decode(file) 145 | if err != nil { 146 | return nil, errors.WithStack(err) 147 | } 148 | return img, nil 149 | } 150 | func mergeAlphaChannel(base image.Image, alpha image.Image) (*image.NRGBA, error) { 151 | // TODO: DrawMask instead 152 | if base.Bounds() != alpha.Bounds() { 153 | alpha = imaging.Resize(alpha, base.Bounds().Dx(), base.Bounds().Dy(), imaging.Lanczos) 154 | } 155 | canvas := image.NewNRGBA(base.Bounds()) 156 | for i := canvas.Bounds().Min.X; i < canvas.Bounds().Max.X; i++ { 157 | for j := canvas.Bounds().Min.Y; j < canvas.Bounds().Max.Y; j++ { 158 | baseColor := color.NRGBAModel.Convert(base.At(i, j)).(color.NRGBA) 159 | alphaColor := color.GrayModel.Convert(alpha.At(i, j)).(color.Gray) 160 | baseColor.A = alphaColor.Y 161 | canvas.SetNRGBA(i, j, baseColor) 162 | } 163 | } 164 | return canvas, nil 165 | } 166 | 167 | func mergeCharacterFace(body image.Image, face image.Image, faceRect image.Rectangle) (*image.NRGBA, error) { 168 | if body == nil || face == nil { 169 | return nil, errors.Errorf("the images are nil! ") 170 | } 171 | 172 | if !faceRect.In(body.Bounds()) { 173 | return nil, errors.Errorf("merge character face: face rectangle %v is not in the body's bounds %v", faceRect, body.Bounds()) 174 | } 175 | face = imaging.Resize(face, faceRect.Dx(), faceRect.Dy(), imaging.Lanczos) 176 | 177 | canvas := image.NewNRGBA(body.Bounds()) 178 | draw.Draw(canvas, body.Bounds(), body, image.Point{}, draw.Over) 179 | draw.Draw(canvas, faceRect, face, image.Point{}, draw.Over) 180 | return canvas, nil 181 | } 182 | -------------------------------------------------------------------------------- /tools/extractor/main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import functools 3 | import json 4 | import os 5 | import sys 6 | from concurrent.futures import ProcessPoolExecutor 7 | from pathlib import Path 8 | from typing import Dict, Tuple 9 | 10 | import PIL.Image 11 | import UnityPy 12 | from UnityPy.classes import Object, Texture2D, Sprite, TextAsset, MonoBehaviour, GameObject 13 | 14 | # flush every line to prevent blocking outputs 15 | # noinspection PyShadowingBuiltins 16 | print = functools.partial(print, flush=True) 17 | 18 | # initialize PIL to preload supported formats 19 | PIL.Image.preinit() 20 | PIL.Image.init() 21 | 22 | 23 | def unpack_assets(src: Path, dst: Path, workers=None): 24 | try: 25 | if src.is_dir(): 26 | print(f"searching files in {src}...") 27 | with ProcessPoolExecutor(max_workers=workers) as executor: 28 | for it in src.glob('**/*'): 29 | if it.is_file(): 30 | print(f"found {it} in {src}...") 31 | executor.submit(unpack_assets, it, dst, None) 32 | 33 | elif src.is_file(): 34 | env = UnityPy.load(str(src)) 35 | for container, obj_reader in env.container.items(): 36 | container = Path(container) 37 | obj = obj_reader.read() 38 | 39 | dst_subdir = container / '..' / obj.m_Name 40 | path_id_path = (dst / container / '..' / f"{obj.m_Name}.json").resolve() 41 | tt_path_id_path = (dst / container / '..' / f"{obj.m_Name}.typetree.json").resolve() 42 | path_id_dict, tt_path_id_dict = export(obj, dst, dst_subdir) 43 | if len(path_id_dict) > 0: 44 | with open(path_id_path, "w", encoding="utf8") as file: 45 | json.dump(path_id_dict, file, ensure_ascii=False, indent=4) 46 | with open(tt_path_id_path, "w", encoding="utf8") as file: 47 | json.dump(tt_path_id_dict, file, ensure_ascii=False, indent=4) 48 | 49 | else: 50 | print(f"WARN: {src} is not dir neither file; skipping", file=sys.stderr) 51 | except Exception as e: 52 | print(f"ERROR: {src} failed to unpack: {e}", file=sys.stderr) 53 | 54 | 55 | def export( 56 | obj: Object, dst: Path, dst_subdir: Path, 57 | path_id_dict: Dict[int, str] = None, 58 | tt_path_id_dict: Dict[int, str] = None 59 | ) -> Tuple[Dict[int, str], Dict[int, str]]: 60 | path_id_dict = {} if path_id_dict is None else path_id_dict 61 | tt_path_id_dict = {} if tt_path_id_dict is None else tt_path_id_dict 62 | 63 | container = Path(obj.object_reader.container) if obj.object_reader.container else None 64 | 65 | obj_name = getattr(obj, 'm_Name', '') 66 | obj_type = obj.object_reader.type.name 67 | 68 | path_id = obj.object_reader.path_id 69 | 70 | match obj: 71 | case Texture2D() | Sprite(): 72 | obj: Texture2D | Sprite 73 | 74 | obj_path = Path(os.path.normpath(container or dst_subdir / f"{obj_name}.png")) 75 | 76 | dest = (dst / (container or obj_path)).resolve() 77 | json_dest = (dst / (container or obj_path).with_suffix(f'.{obj_type}.json')).resolve() 78 | dest.parent.mkdir(parents=True, exist_ok=True) 79 | json_dest.parent.mkdir(parents=True, exist_ok=True) 80 | path_id_dict[path_id] = str(dest.name) 81 | tt_path_id_dict[path_id] = str(json_dest.name) 82 | 83 | if dest.exists() and obj_type in ["Sprite"]: 84 | # Sometimes there are some Sprite and Texture2D with the same name 85 | # Texture2D is preferred over Sprite, so we skip the Sprite 86 | # This is because Texture2D is usually the original image 87 | print(f"skipping {obj_path}({obj_type}), file already exists", file=sys.stderr) 88 | else: 89 | if dest.suffix in PIL.Image.EXTENSION and PIL.Image.EXTENSION[dest.suffix] in PIL.Image.SAVE: 90 | obj.image.save(dest) 91 | print(f"{obj_path}({obj_type})=>{dest}") 92 | else: 93 | print(f"cannot export {obj_path}({obj_type}), format is not supported", file=sys.stderr) 94 | 95 | with open(json_dest, "w", encoding="utf8") as file: 96 | json.dump(obj.object_reader.read_typetree(), file, 97 | ensure_ascii=False, 98 | indent=4, 99 | default=lambda o: '') 100 | print(f"{obj_path}({obj_type})=>{json_dest}") 101 | 102 | case MonoBehaviour(): 103 | obj: MonoBehaviour 104 | 105 | script = obj.m_Script.read() 106 | obj_name = script.m_Name 107 | 108 | obj_path = Path(os.path.normpath(container or dst_subdir / f"{obj_name}.json")) 109 | 110 | dest = (dst / (container or obj_path)).resolve() 111 | dest.parent.mkdir(parents=True, exist_ok=True) 112 | path_id_dict[path_id] = str(dest.name) 113 | 114 | with open(dest, "w", encoding="utf8") as file: 115 | json.dump(obj.object_reader.read_typetree(), file, 116 | ensure_ascii=False, 117 | indent=4, 118 | default=lambda o: '') 119 | print(f"{obj_path}({obj_type})=>{dest}") 120 | 121 | case GameObject(): 122 | obj: GameObject 123 | 124 | obj_readers = obj.assets_file.objects.values() 125 | dst_subdir = dst_subdir / '..' / obj_name 126 | for obj_reader in obj_readers: 127 | if obj_reader is obj.object_reader: 128 | continue 129 | export(obj_reader.read(), dst, dst_subdir, path_id_dict, tt_path_id_dict) 130 | 131 | case _: 132 | print(f"skipping {obj_name}({obj_type}), type {obj_type} not supported", file=sys.stderr) 133 | 134 | return path_id_dict, tt_path_id_dict 135 | 136 | 137 | def main(): 138 | parser = argparse.ArgumentParser() 139 | parser.add_argument( 140 | "src", nargs="+", 141 | help="Path to source file or directory." 142 | ) 143 | parser.add_argument( 144 | "dst", 145 | help="Path to destination directory." 146 | ) 147 | parser.add_argument( 148 | "-w", "--workers", nargs="?", default=None, 149 | help="Specify the concurrency workers count." 150 | ) 151 | args = parser.parse_args() 152 | 153 | for src in args.src: 154 | unpack_assets( 155 | Path(src), 156 | Path(args.dst), 157 | workers=int(args.workers) if args.workers else None 158 | ) 159 | 160 | 161 | if __name__ == '__main__': 162 | main() 163 | -------------------------------------------------------------------------------- /internal/app/updateloop/update_art.go: -------------------------------------------------------------------------------- 1 | package updateloop 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "github.com/alitto/pond" 8 | "github.com/chai2010/webp" 9 | "github.com/flandiayingman/arkwaifu/internal/app/art" 10 | "github.com/flandiayingman/arkwaifu/internal/pkg/ark" 11 | "github.com/flandiayingman/arkwaifu/internal/pkg/arkassets" 12 | "github.com/flandiayingman/arkwaifu/internal/pkg/arkprocessor" 13 | "github.com/flandiayingman/arkwaifu/internal/pkg/arkscanner" 14 | "github.com/flandiayingman/arkwaifu/internal/pkg/cols" 15 | "github.com/rs/zerolog/log" 16 | "os" 17 | "runtime" 18 | "time" 19 | ) 20 | 21 | var ( 22 | artPatterns = []string{ 23 | "avg/imgs/**", 24 | "avg/bg/**", 25 | "avg/items/**", 26 | "avg/characters/**", 27 | } 28 | ) 29 | 30 | type task func() error 31 | 32 | func (s *Service) getRemoteArtVersion(ctx context.Context) (ark.Version, error) { 33 | return arkassets.GetLatestVersion() 34 | } 35 | func (s *Service) getLocalArtVersion(ctx context.Context) (ark.Version, error) { 36 | return s.repo.selectArtVersion(ctx) 37 | } 38 | 39 | func (s *Service) attemptUpdateArt(ctx context.Context) { 40 | localArtVersion, err := s.getLocalArtVersion(ctx) 41 | if err != nil { 42 | log.Error(). 43 | Err(err). 44 | Caller(). 45 | Msg("Failed to get the local art version.") 46 | return 47 | } 48 | remoteArtVersion, err := s.getRemoteArtVersion(ctx) 49 | if err != nil { 50 | log.Error(). 51 | Err(err). 52 | Caller(). 53 | Msg("Failed to get the remote art version.") 54 | return 55 | } 56 | if localArtVersion != remoteArtVersion { 57 | err := s.updateArts(ctx, localArtVersion, remoteArtVersion) 58 | if err != nil { 59 | log.Error(). 60 | Err(err). 61 | Str("localArtVersion", localArtVersion). 62 | Str("remoteArtVersion", remoteArtVersion). 63 | Msg("Failed to update arts") 64 | } 65 | } 66 | } 67 | 68 | func (s *Service) updateArts(ctx context.Context, oldVersion, newVersion string) error { 69 | root, err := os.MkdirTemp("", "arkwaifu-updateloop-art-*") 70 | if err != nil { 71 | return err 72 | } 73 | defer func() { _ = os.RemoveAll(root) }() 74 | 75 | err = arkassets.UpdateGameAssets(ctx, oldVersion, newVersion, root, artPatterns) 76 | if err != nil { 77 | return err 78 | } 79 | 80 | pictureArtTasks, err := s.createPictureArtSubmitTasks(root) 81 | if err != nil { 82 | return err 83 | } 84 | characterArtTasks, err := s.createCharacterArtSubmitTasks(root) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | workerNum := runtime.NumCPU() 90 | taskNum := len(pictureArtTasks) + len(characterArtTasks) 91 | log.Info().Msgf("Begin submitting arts, using %v workers to run %d tasks", workerNum, taskNum) 92 | 93 | begin := time.Now() 94 | pool := pond.New(workerNum, taskNum) 95 | defer pool.Stop() 96 | group, _ := pool.GroupContext(ctx) 97 | for _, artTask := range pictureArtTasks { 98 | group.Submit(artTask) 99 | } 100 | for _, artTask := range characterArtTasks { 101 | group.Submit(artTask) 102 | } 103 | err = group.Wait() 104 | if err != nil { 105 | return err 106 | } 107 | 108 | err = s.repo.upsertArtVersion(ctx, &artVersion{ 109 | Lock: zeroPtr, 110 | Version: newVersion, 111 | }) 112 | if err != nil { 113 | return err 114 | } 115 | 116 | log.Info().Msgf("End submitting arts, elapsed %v", time.Since(begin)) 117 | return nil 118 | } 119 | 120 | func (s *Service) createPictureArtSubmitTasks(root string) (tasks []task, err error) { 121 | scanner := arkscanner.Scanner{Root: root} 122 | 123 | pictureArts, err := scanner.ScanForPictureArts() 124 | if err != nil { 125 | return nil, err 126 | } 127 | 128 | for _, art := range pictureArts { 129 | tasks = append(tasks, s.createPictureArtSubmitTask(root, art)) 130 | } 131 | 132 | return 133 | } 134 | func (s *Service) createPictureArtSubmitTask(root string, art *arkscanner.PictureArt) task { 135 | return func() error { 136 | log.Info().Msgf("Submitting the picture art %v...", *art) 137 | err := s.submitPictureArt(root, art) 138 | if err != nil { 139 | return fmt.Errorf("failed to submit picture art %v: %w", art, err) 140 | } 141 | return nil 142 | } 143 | } 144 | func (s *Service) submitPictureArt(root string, pic *arkscanner.PictureArt) error { 145 | processor := arkprocessor.Processor{Root: root} 146 | img, err := processor.ProcessPictureArt((*arkprocessor.PictureArt)(pic)) 147 | if err != nil { 148 | return err 149 | } 150 | 151 | err = s.artService.UpsertArts(art.NewArt(img.Art.ID, art.MustParseCategory(img.Art.Kind))) 152 | if err != nil { 153 | return err 154 | } 155 | 156 | err = s.artService.UpsertVariants(art.NewVariant(img.Art.ID, art.VariationOrigin)) 157 | if err != nil { 158 | return err 159 | } 160 | 161 | var buf bytes.Buffer 162 | err = webp.Encode(&buf, img.Image, &webp.Options{Lossless: true}) 163 | if err != nil { 164 | return err 165 | } 166 | err = s.artService.StoreContent(img.Art.ID, art.VariationOrigin, buf.Bytes()) 167 | if err != nil { 168 | return err 169 | } 170 | 171 | return nil 172 | } 173 | 174 | func (s *Service) createCharacterArtSubmitTasks(root string) (tasks []task, err error) { 175 | scanner := arkscanner.Scanner{Root: root} 176 | 177 | characterArts, err := scanner.ScanForCharacterArts() 178 | if err != nil { 179 | return nil, err 180 | } 181 | 182 | for _, art := range characterArts { 183 | tasks = append(tasks, s.createCharacterArtSubmitTask(root, art)) 184 | } 185 | 186 | return 187 | } 188 | func (s *Service) createCharacterArtSubmitTask(root string, art *arkscanner.CharacterArt) task { 189 | return func() error { 190 | log.Info().Msgf("Submitting the character art %v...", *art) 191 | err := s.submitCharacterArt(root, art) 192 | if err != nil { 193 | return fmt.Errorf("failed to submit picture art %v: %w", art, err) 194 | } 195 | return nil 196 | } 197 | } 198 | func (s *Service) submitCharacterArt(root string, char *arkscanner.CharacterArt) error { 199 | processor := arkprocessor.Processor{Root: root} 200 | imgs, err := processor.ProcessCharacterArt((*arkprocessor.CharacterArt)(char)) 201 | if err != nil { 202 | return err 203 | } 204 | if len(imgs) == 0 { 205 | return nil 206 | } 207 | 208 | err = s.artService.UpsertArts(cols.Map(imgs, func(img arkprocessor.CharacterArtImage) *art.Art { 209 | return art.NewArt(img.ID(), art.MustParseCategory(img.Art.Kind)) 210 | })...) 211 | if err != nil { 212 | return err 213 | } 214 | 215 | err = s.artService.UpsertVariants(cols.Map(imgs, func(img arkprocessor.CharacterArtImage) *art.Variant { 216 | return art.NewVariant(img.ID(), art.VariationOrigin) 217 | })...) 218 | if err != nil { 219 | return err 220 | } 221 | 222 | for _, img := range imgs { 223 | var buf bytes.Buffer 224 | err := webp.Encode(&buf, img.Image, &webp.Options{Lossless: true}) 225 | if err != nil { 226 | return err 227 | } 228 | err = s.artService.StoreContent(img.ID(), art.VariationOrigin, buf.Bytes()) 229 | if err != nil { 230 | return err 231 | } 232 | } 233 | return nil 234 | } 235 | -------------------------------------------------------------------------------- /internal/app/art/repository.go: -------------------------------------------------------------------------------- 1 | package art 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/flandiayingman/arkwaifu/internal/app/infra" 7 | "github.com/flandiayingman/arkwaifu/internal/pkg/util/fileutil" 8 | "gorm.io/gorm" 9 | "gorm.io/gorm/clause" 10 | "os" 11 | "path/filepath" 12 | ) 13 | 14 | type repository struct { 15 | db *infra.Gorm 16 | ContentDir string 17 | } 18 | 19 | func newRepo(conf *infra.Config, db *infra.Gorm) (*repository, error) { 20 | r := repository{ 21 | db: db, 22 | ContentDir: filepath.Join(conf.Root, "arts-content"), 23 | } 24 | err := r.init() 25 | if err != nil { 26 | return nil, err 27 | } 28 | return &r, nil 29 | } 30 | 31 | func (r *repository) init() (err error) { 32 | err = r.db.CreateCollateNumeric() 33 | if err != nil { 34 | return err 35 | } 36 | err = r.db.CreateEnum("art_category", 37 | string(CategoryImage), 38 | string(CategoryBackground), 39 | string(CategoryItem), 40 | string(CategoryCharacter), 41 | ) 42 | if err != nil { 43 | return err 44 | } 45 | err = r.db.CreateEnum("art_variation", 46 | string(VariationOrigin), 47 | string(VariationRealEsrganX4Plus), 48 | string(VariationRealEsrganX4PlusAnime), 49 | string(VariationThumbnail), 50 | ) 51 | if err != nil { 52 | return err 53 | } 54 | err = r.db.AutoMigrate(&Art{}) 55 | if err != nil { 56 | return err 57 | } 58 | err = r.db.AutoMigrate(&Variant{}) 59 | if err != nil { 60 | return err 61 | } 62 | return nil 63 | } 64 | 65 | func (r *repository) SelectArts() ([]*Art, error) { 66 | arts := make([]*Art, 0) 67 | result := r.db. 68 | Preload("Variants", func(db *gorm.DB) *gorm.DB { return db.Order("variants.variation") }). 69 | Order("category, id"). 70 | Find(&arts) 71 | if result.Error != nil { 72 | return nil, result.Error 73 | } 74 | return arts, nil 75 | } 76 | func (r *repository) SelectArtsByIDs(ids []string) ([]*Art, error) { 77 | arts := make([]*Art, 0) 78 | result := r.db. 79 | Preload("Variants", func(db *gorm.DB) *gorm.DB { return db.Order("variants.variation") }). 80 | Joins("JOIN UNNEST(?) WITH ORDINALITY t(id, ord) USING (id)", clause.Expr{SQL: "ARRAY[?]", Vars: []any{ids}, WithoutParentheses: true}). 81 | Order("t.ord"). 82 | Find(&arts, ids) 83 | if result.Error != nil { 84 | return nil, result.Error 85 | } 86 | return arts, nil 87 | } 88 | func (r *repository) SelectArtsByCategory(category string) ([]*Art, error) { 89 | arts := make([]*Art, 0) 90 | result := r.db. 91 | Preload("Variants", func(db *gorm.DB) *gorm.DB { return db.Order("variants.variation") }). 92 | Where("category = ?", category). 93 | Order("category, id"). 94 | Find(&arts) 95 | if result.Error != nil { 96 | return nil, result.Error 97 | } 98 | return arts, nil 99 | } 100 | func (r *repository) SelectArtsByIDLike(like string) ([]*Art, error) { 101 | arts := make([]*Art, 0) 102 | result := r.db. 103 | Preload("Variants", func(db *gorm.DB) *gorm.DB { return db.Order("variants.variation") }). 104 | Where("id LIKE ?", like). 105 | Order("category, id"). 106 | Find(&arts) 107 | if result.Error != nil { 108 | return nil, result.Error 109 | } 110 | return arts, nil 111 | } 112 | func (r *repository) SelectArt(id string) (*Art, error) { 113 | art := new(Art) 114 | result := r.db. 115 | Preload("Variants", func(db *gorm.DB) *gorm.DB { return db.Order("variants.variation") }). 116 | Take(&art, "id = ?", id) 117 | if errors.Is(result.Error, gorm.ErrRecordNotFound) { 118 | return nil, errors.Join(ErrNotFound, result.Error) 119 | } 120 | if result.Error != nil { 121 | return nil, result.Error 122 | } 123 | return art, nil 124 | } 125 | 126 | func (r *repository) SelectVariants(id string) ([]*Variant, error) { 127 | variants := make([]*Variant, 0) 128 | result := r.db. 129 | Where("art_id = ?", id). 130 | Order("variation"). 131 | Find(&variants) 132 | if result.Error != nil { 133 | return nil, result.Error 134 | } 135 | return variants, nil 136 | } 137 | func (r *repository) SelectVariant(artID string, typ string) (*Variant, error) { 138 | variant := new(Variant) 139 | result := r.db.Take(&variant, "art_id = ?, type = ?", artID, typ) 140 | if errors.Is(result.Error, gorm.ErrRecordNotFound) { 141 | return nil, errors.Join(ErrNotFound, result.Error) 142 | } 143 | if result.Error != nil { 144 | return nil, result.Error 145 | } 146 | return variant, nil 147 | } 148 | 149 | func (r *repository) UpsertArts(arts ...*Art) error { 150 | return r.db. 151 | Clauses(clause.OnConflict{UpdateAll: true}). 152 | Create(&arts).Error 153 | } 154 | func (r *repository) UpsertVariants(variants ...*Variant) error { 155 | err := r.db. 156 | Clauses(clause.OnConflict{UpdateAll: true}). 157 | Create(&variants).Error 158 | if err != nil { 159 | return err 160 | } 161 | return nil 162 | } 163 | 164 | func (r *repository) StoreContent(id string, variation string, content []byte) (err error) { 165 | err = r.db.Transaction(func(tx *gorm.DB) (err error) { 166 | // If the corresponding art exists, select it. 167 | art := Art{} 168 | err = r.db. 169 | Where("id = ?", id). 170 | Take(&art). 171 | Error 172 | if err != nil { 173 | return err 174 | } 175 | 176 | // If the corresponding variant exists, select it. 177 | variant := Variant{} 178 | err = r.db. 179 | Clauses(clause.Locking{Strength: "UPDATE"}). 180 | Where("(art_id, variation) = (?, ?)", id, variation). 181 | Take(&variant). 182 | Error 183 | if err != nil { 184 | return err 185 | } 186 | 187 | contentObj := Content{ 188 | ID: art.ID, 189 | Category: art.Category, 190 | Variation: variant.Variation, 191 | Content: content, 192 | } 193 | 194 | // Check whether the content object is valid. 195 | config, err := contentObj.Check() 196 | if err != nil { 197 | return err 198 | } 199 | 200 | // Update the corresponding variant. 201 | variant.ContentPresent = true 202 | variant.ContentWidth = &config.Width 203 | variant.ContentHeight = &config.Height 204 | 205 | // Write content's change on variant to database 206 | err = r.UpsertVariants(&variant) 207 | if err != nil { 208 | return err 209 | } 210 | 211 | // Write content to filesystem. 212 | path := filepath.Join(r.ContentDir, contentObj.PathRel()) 213 | err = fileutil.MkFileFromBytes(path, contentObj.Content) 214 | if err != nil { 215 | return err 216 | } 217 | 218 | return nil 219 | }) 220 | return 221 | } 222 | func (r *repository) TakeContent(id string, variation string) (content []byte, err error) { 223 | // If the corresponding art exists, select it. 224 | art := Art{} 225 | err = r.db. 226 | Where("id = ?", id). 227 | Take(&art). 228 | Error 229 | if err != nil { 230 | return nil, err 231 | } 232 | 233 | // If the corresponding variant exists, select it. 234 | variant := Variant{} 235 | err = r.db. 236 | Where("(art_id, variation) = (?, ?)", id, variation). 237 | Take(&variant). 238 | Error 239 | if err != nil { 240 | return nil, err 241 | } 242 | 243 | if variant.ContentPresent { 244 | contentObj := Content{ 245 | ID: art.ID, 246 | Category: art.Category, 247 | Variation: variant.Variation, 248 | Content: nil, 249 | } 250 | // Read content from the filesystem. 251 | path := filepath.Join(r.ContentDir, contentObj.PathRel()) 252 | return os.ReadFile(path) 253 | } else { 254 | // Error because content is not present. 255 | return nil, fmt.Errorf("content id=%s variation=%s is not present", id, variation) 256 | } 257 | } 258 | 259 | func (r *repository) SelectArtsWhereVariantAbsent(variation string) ([]*Art, error) { 260 | arts := make([]*Art, 0) 261 | err := r.db. 262 | Preload("Variants", func(db *gorm.DB) *gorm.DB { return db.Order("variants.variation") }). 263 | Order("category, id"). 264 | Not("EXISTS (SELECT 1 FROM variants WHERE (arts.id, ?) = (variants.art_id, variants.variation))", variation). 265 | Find(&arts). 266 | Error 267 | return arts, err 268 | } 269 | -------------------------------------------------------------------------------- /internal/pkg/util/fileutil/files.go: -------------------------------------------------------------------------------- 1 | package fileutil 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "golang.org/x/sync/errgroup" 8 | "io" 9 | "io/fs" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | 14 | "github.com/flandiayingman/arkwaifu/internal/pkg/util/pathutil" 15 | ) 16 | 17 | // MkParents creates all parents of a file with mode 0755 (before unmask). 18 | // If the parents of the file already exist, MkParents does nothing. 19 | func MkParents(filePath string) error { 20 | err := os.MkdirAll(filepath.Dir(filePath), 0755) 21 | if err != nil { 22 | return fmt.Errorf("failed to create parent directories of %s: %v", filePath, err) 23 | } 24 | return err 25 | } 26 | 27 | // MkFile creates a file and its parents with respectively mode 0666 and 0755 (both before unmask). 28 | // If the parents of the file already exist, MkFile does nothing. 29 | // If the file already exist, MkFile truncates it. 30 | func MkFile(filePath string) (*os.File, error) { 31 | err := MkParents(filePath) 32 | if err != nil { 33 | return nil, err 34 | } 35 | file, err := os.Create(filePath) 36 | if err != nil { 37 | return nil, fmt.Errorf("failed to create file %s: %v", filePath, err) 38 | } 39 | return file, nil 40 | } 41 | 42 | // CopyAllContent copies all files from srcDir to dstDir recursively. 43 | // Only files' name and content are guaranteed to be the same. 44 | // Only files are copied, (empty) directories are ignored. 45 | // Any existing files will be overridden. 46 | func CopyAllContent(srcDir string, dstDir string) error { 47 | return filepath.WalkDir(srcDir, func(srcPath string, entry fs.DirEntry, err error) error { 48 | if err != nil { 49 | return err 50 | } 51 | if entry.IsDir() { 52 | return nil 53 | } 54 | 55 | dstPath := pathutil.MustChangeParent(srcPath, srcDir, dstDir) 56 | err = CopyContent(srcPath, dstPath) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | return nil 62 | }) 63 | } 64 | 65 | // MoveAllContent moves all files from srcDir to dstDir recursively. 66 | // Only files' name and content are guaranteed to be the same. 67 | // Only files are moved, (empty) directories are ignored. 68 | // Any existing files will be overridden. 69 | // 70 | // Use MoveAllContent to prevent os.Rename to panic "invalid cross-device link". 71 | func MoveAllContent(srcDir string, dstDir string) error { 72 | return filepath.WalkDir(srcDir, func(srcPath string, entry fs.DirEntry, err error) error { 73 | if err != nil { 74 | return err 75 | } 76 | if entry.IsDir() { 77 | return nil 78 | } 79 | 80 | dstPath := pathutil.MustChangeParent(srcPath, srcDir, dstDir) 81 | err = MoveContent(srcPath, dstPath) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | return nil 87 | }) 88 | } 89 | 90 | // CopyContent copies the file src to dst. 91 | // Only the file's name and content are guaranteed to be the same. 92 | // The parents will be created if they don't exist. 93 | // The existing file will be overridden. 94 | func CopyContent(src string, dst string) error { 95 | srcFile, err := os.Open(src) 96 | if err != nil { 97 | return fmt.Errorf("failed to open src %s: %w", src, err) 98 | } 99 | defer srcFile.Close() 100 | 101 | dstFile, err := MkFile(dst) 102 | if err != nil { 103 | return fmt.Errorf("failed to make dst %s: %w", dst, err) 104 | } 105 | defer dstFile.Close() 106 | 107 | _, err = io.Copy(dstFile, srcFile) 108 | if err != nil { 109 | return fmt.Errorf("failed to copy to dst %s from src %s: %w", dst, src, err) 110 | } 111 | 112 | return nil 113 | } 114 | 115 | // MoveContent moves the file src to dst. 116 | // Only the file's name and content are guaranteed to be the same. 117 | // The parents will be created if they don't exist. 118 | // The existing file will be overridden. 119 | func MoveContent(src string, dst string) error { 120 | err := os.Rename(src, dst) 121 | if err != nil { 122 | err = CopyContent(src, dst) 123 | if err != nil { 124 | return err 125 | } 126 | 127 | err = os.Remove(src) 128 | if err != nil { 129 | return fmt.Errorf("failed to remove src %s: %w", src, err) 130 | } 131 | } 132 | 133 | return nil 134 | } 135 | 136 | // Exists checks if the file or directory exists. 137 | // If Exists cannot determinate whether the file or directory exists (e.g., permission error), it returns an error. 138 | func Exists(path string) (bool, error) { 139 | _, err := os.Stat(path) 140 | if errors.Is(err, os.ErrNotExist) { 141 | return false, nil 142 | } 143 | if err != nil { 144 | return false, err 145 | } 146 | return true, nil 147 | } 148 | 149 | // LowercaseAll renames the directory to lowercase recursively. 150 | // If a file or directory is already lower-cased, LowercaseAll does nothing to it. 151 | func LowercaseAll(dirPath string) error { 152 | return filepath.WalkDir(dirPath, func(path string, entry fs.DirEntry, err error) error { 153 | if err != nil { 154 | return err 155 | } 156 | return Lowercase(path) 157 | }) 158 | } 159 | 160 | // Lowercase renames the file or directory to lowercase. 161 | // If the file or directory is already lower-cased, it does nothing. 162 | func Lowercase(path string) error { 163 | lowerPath := filepath.Join(filepath.Dir(path), strings.ToLower(filepath.Base(path))) 164 | if path != lowerPath { 165 | return os.Rename(path, lowerPath) 166 | } else { 167 | return nil 168 | } 169 | } 170 | 171 | // List lists all files and directories under the directory dirPath (no recursive). 172 | func List(dirPath string) ([]string, error) { 173 | dirEntries, err := os.ReadDir(dirPath) 174 | if err != nil { 175 | return nil, err 176 | } 177 | var all []string 178 | for _, dirEntry := range dirEntries { 179 | all = append(all, filepath.Join(dirPath, dirEntry.Name())) 180 | } 181 | return all, err 182 | } 183 | 184 | // ListFiles lists all files under the directory dirPath (no recursive). 185 | func ListFiles(dirPath string) ([]string, error) { 186 | dirEntries, err := os.ReadDir(dirPath) 187 | if err != nil { 188 | return nil, err 189 | } 190 | var all []string 191 | for _, dirEntry := range dirEntries { 192 | if dirEntry.IsDir() { 193 | continue 194 | } 195 | all = append(all, filepath.Join(dirPath, dirEntry.Name())) 196 | } 197 | return all, err 198 | } 199 | 200 | // ListDirs lists all directories under the directory dirPath (no recursive). 201 | func ListDirs(dirPath string) ([]string, error) { 202 | dirEntries, err := os.ReadDir(dirPath) 203 | if err != nil { 204 | return nil, err 205 | } 206 | var all []string 207 | for _, dirEntry := range dirEntries { 208 | if dirEntry.IsDir() { 209 | all = append(all, filepath.Join(dirPath, dirEntry.Name())) 210 | } 211 | } 212 | return all, err 213 | } 214 | 215 | // ListAll lists all files and directories under the directory dirPath (recursive). 216 | func ListAll(dirPath string) ([]string, error) { 217 | var all []string 218 | err := filepath.WalkDir(dirPath, func(path string, entry fs.DirEntry, err error) error { 219 | if err != nil { 220 | return err 221 | } 222 | all = append(all, path) 223 | return nil 224 | }) 225 | return all, err 226 | } 227 | 228 | // ListAllFiles lists all files under the directory dirPath (recursive). 229 | func ListAllFiles(dirPath string) ([]string, error) { 230 | var allFiles []string 231 | err := filepath.WalkDir(dirPath, func(path string, entry fs.DirEntry, err error) error { 232 | if err != nil { 233 | return err 234 | } 235 | if entry.IsDir() { 236 | return nil 237 | } 238 | allFiles = append(allFiles, path) 239 | return nil 240 | }) 241 | return allFiles, err 242 | } 243 | 244 | // ListALlDirs lists all directories under the directory dirPath (recursive). 245 | func ListALlDirs(dirPath string) ([]string, error) { 246 | var allFiles []string 247 | err := filepath.WalkDir(dirPath, func(path string, entry fs.DirEntry, err error) error { 248 | if err != nil { 249 | return err 250 | } 251 | if entry.IsDir() { 252 | allFiles = append(allFiles, path) 253 | } 254 | return nil 255 | }) 256 | return allFiles, err 257 | } 258 | 259 | // MkFileFromReader calls MkFile and then writes the content of the reader to the file. 260 | // 261 | // If the reader implements io.Closer, it would be closed when the method returns. 262 | func MkFileFromReader(filePath string, r io.Reader) error { 263 | defer func() { _ = CloseIfIsCloser(r) }() 264 | 265 | f, err := MkFile(filePath) 266 | if err != nil { 267 | return err 268 | } 269 | defer func() { _ = f.Close() }() 270 | 271 | _, err = io.Copy(f, r) 272 | return nil 273 | } 274 | 275 | // MkFileFromBytes calls MkFile and then writes the content of the bytes to the file. 276 | func MkFileFromBytes(filePath string, b []byte) error { 277 | return MkFileFromReader(filePath, bytes.NewReader(b)) 278 | } 279 | 280 | type PathWithContent struct { 281 | FilePath string 282 | Content []byte 283 | } 284 | 285 | func MkFilesFromBytes(goroutines int, s ...PathWithContent) error { 286 | eg := errgroup.Group{} 287 | eg.SetLimit(goroutines) 288 | for i := range s { 289 | pwc := s[i] 290 | work := func() error { return MkFileFromBytes(pwc.FilePath, pwc.Content) } 291 | eg.Go(work) 292 | } 293 | return eg.Wait() 294 | } 295 | 296 | func CloseIfIsCloser(r any) error { 297 | if closer, ok := r.(io.Closer); ok { 298 | return closer.Close() 299 | } 300 | return nil 301 | } 302 | -------------------------------------------------------------------------------- /internal/pkg/arkscanner/scanner_character.go: -------------------------------------------------------------------------------- 1 | package arkscanner 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/flandiayingman/arkwaifu/internal/pkg/cols" 7 | "github.com/flandiayingman/arkwaifu/internal/pkg/util/pathutil" 8 | "github.com/pkg/errors" 9 | "image" 10 | "math" 11 | "os" 12 | "path" 13 | "path/filepath" 14 | ) 15 | 16 | type ( 17 | CharacterArt struct { 18 | ID string 19 | Kind string 20 | BodyVariations []CharacterArtBodyVariation 21 | } 22 | CharacterArtBodyVariation struct { 23 | BodySprite string 24 | BodySpriteAlpha string 25 | FaceRectangle image.Rectangle 26 | FaceVariations []CharacterArtFaceVariation 27 | 28 | // PixelToUnits is the ratio of pixel to unit. 29 | // In other words, the resulting image is scaled by *dividing* this value. 30 | // This is because the original image is in units, and we convert it to pixels. 31 | PixelToUnits float64 32 | } 33 | CharacterArtFaceVariation struct { 34 | FaceSprite string 35 | FaceSpriteAlpha string 36 | WholeBody bool 37 | 38 | // PixelToUnits is the ratio of pixel to unit. 39 | // In other words, the resulting image is scaled by *dividing* this value. 40 | // This is because the original image is in units, and we convert it to pixels. 41 | PixelToUnits float64 42 | } 43 | ) 44 | 45 | func (a *CharacterArt) FacePath(bodyNum int, faceNum int) string { 46 | // note: "*num" starts with 1, not 0 47 | sprite := a.BodyVariations[bodyNum-1].FaceVariations[faceNum-1].FaceSprite 48 | if sprite == "" { 49 | return "" 50 | } 51 | return path.Join(characterPrefix, a.ID, sprite) 52 | } 53 | func (a *CharacterArt) FacePathAlpha(bodyNum int, faceNum int) string { 54 | // note: "*num" starts with 1, not 0 55 | sprite := a.BodyVariations[bodyNum-1].FaceVariations[faceNum-1].FaceSpriteAlpha 56 | if sprite == "" { 57 | return "" 58 | } 59 | return path.Join(characterPrefix, a.ID, sprite) 60 | } 61 | func (a *CharacterArt) BodyPath(bodyNum int) string { 62 | // note: "*num" starts with 1, not 0 63 | sprite := a.BodyVariations[bodyNum-1].BodySprite 64 | if sprite == "" { 65 | return "" 66 | } 67 | return path.Join(characterPrefix, a.ID, sprite) 68 | } 69 | func (a *CharacterArt) BodyPathAlpha(bodyNum int) string { 70 | // note: "*num" starts with 1, not 0 71 | sprite := a.BodyVariations[bodyNum-1].BodySpriteAlpha 72 | if sprite == "" { 73 | return "" 74 | } 75 | return path.Join(characterPrefix, a.ID, sprite) 76 | } 77 | 78 | var ( 79 | characterPrefix = "assets/torappu/dynamicassets/avg/characters" 80 | ) 81 | 82 | func (scanner *Scanner) ScanForCharacterArts() ([]*CharacterArt, error) { 83 | rootCharacterArts := filepath.Join(scanner.Root, characterPrefix) 84 | characterEntries, err := os.ReadDir(rootCharacterArts) 85 | if errors.Is(err, os.ErrNotExist) { 86 | return nil, nil 87 | } 88 | if err != nil { 89 | return nil, errors.WithStack(err) 90 | } 91 | 92 | characterEntries = cols.Filter(characterEntries, func(element os.DirEntry) bool { return element.IsDir() }) 93 | characterIDs := cols.Map(characterEntries, func(i os.DirEntry) (o string) { return pathutil.Stem(i.Name()) }) 94 | characterArts, err := cols.MapErr(characterIDs, scanner.ScanCharacter) 95 | if err != nil { 96 | return nil, errors.WithStack(err) 97 | } 98 | 99 | return characterArts, nil 100 | } 101 | func (scanner *Scanner) ScanCharacter(id string) (*CharacterArt, error) { 102 | hubGroupArt, err := scanner.scanHubGroupOfCharacter(id) 103 | if err != nil { 104 | return nil, errors.WithStack(err) 105 | } 106 | hubArt, err := scanner.scanHubOfCharacter(id) 107 | if err != nil { 108 | return nil, errors.WithStack(err) 109 | } 110 | 111 | if hubGroupArt == nil && hubArt == nil { 112 | return nil, errors.Errorf("scan character %s: neither the hub group nor the hub exist", id) 113 | } 114 | if hubGroupArt != nil && hubArt != nil { 115 | return nil, errors.Errorf("scan character %s: both the hub group and the hub exist", id) 116 | } 117 | 118 | if hubGroupArt != nil { 119 | return hubGroupArt, nil 120 | } else { 121 | return hubArt, nil 122 | } 123 | } 124 | 125 | func (scanner *Scanner) scanHubGroupOfCharacter(id string) (*CharacterArt, error) { 126 | hubPath := filepath.Join(scanner.Root, characterPrefix, id, "AVGCharacterSpriteHubGroup.json") 127 | hubJson, err := os.ReadFile(hubPath) 128 | if errors.Is(err, os.ErrNotExist) { 129 | return nil, nil 130 | } 131 | if err != nil { 132 | return nil, errors.WithStack(err) 133 | } 134 | 135 | var hubGroup CharacterSpriteHubGroup 136 | err = json.Unmarshal(hubJson, &hubGroup) 137 | if err != nil { 138 | return nil, errors.WithStack(err) 139 | } 140 | 141 | pathIDMap, err := scanner.scanPathIDMapOfCharacter(id) 142 | if err != nil { 143 | return nil, errors.WithStack(err) 144 | } 145 | 146 | ttMap, err := scanner.scanTypeTreeMapOfCharacter(id) 147 | if err != nil { 148 | return nil, errors.WithStack(err) 149 | } 150 | 151 | art := hubGroup.toArt(id, pathIDMap, ttMap) 152 | return &art, nil 153 | } 154 | func (scanner *Scanner) scanHubOfCharacter(id string) (*CharacterArt, error) { 155 | hubPath := filepath.Join(scanner.Root, characterPrefix, id, "AVGCharacterSpriteHub.json") 156 | hubJson, err := os.ReadFile(hubPath) 157 | if errors.Is(err, os.ErrNotExist) { 158 | return nil, nil 159 | } 160 | if err != nil { 161 | return nil, errors.WithStack(err) 162 | } 163 | 164 | var hub CharacterSpriteHub 165 | err = json.Unmarshal(hubJson, &hub) 166 | if err != nil { 167 | return nil, errors.WithStack(err) 168 | } 169 | 170 | pathIDMap, err := scanner.scanPathIDMapOfCharacter(id) 171 | if err != nil { 172 | return nil, errors.WithStack(err) 173 | } 174 | 175 | ttMap, err := scanner.scanTypeTreeMapOfCharacter(id) 176 | if err != nil { 177 | return nil, errors.WithStack(err) 178 | } 179 | 180 | hubGroup := CharacterSpriteHubGroup{SpriteGroups: []CharacterSpriteHub{hub}} 181 | art := hubGroup.toArt(id, pathIDMap, ttMap) 182 | return &art, nil 183 | } 184 | func (scanner *Scanner) scanPathIDMapOfCharacter(id string) (map[int64]string, error) { 185 | mapPath := filepath.Join(scanner.Root, characterPrefix, fmt.Sprintf("%s.json", id)) 186 | mapJson, err := os.ReadFile(mapPath) 187 | if err != nil { 188 | return nil, errors.WithStack(err) 189 | } 190 | 191 | pathIDMap := make(map[int64]string) 192 | err = json.Unmarshal(mapJson, &pathIDMap) 193 | if err != nil { 194 | return nil, errors.WithStack(err) 195 | } 196 | 197 | return pathIDMap, nil 198 | } 199 | func (scanner *Scanner) scanTypeTreeMapOfCharacter(id string) (map[int64]SpriteTypeTree, error) { 200 | mapPath := filepath.Join(scanner.Root, characterPrefix, fmt.Sprintf("%s.typetree.json", id)) 201 | mapJson, err := os.ReadFile(mapPath) 202 | if err != nil { 203 | return nil, errors.WithStack(err) 204 | } 205 | 206 | ttPathIDMap := make(map[int64]string) 207 | err = json.Unmarshal(mapJson, &ttPathIDMap) 208 | if err != nil { 209 | return nil, errors.WithStack(err) 210 | } 211 | 212 | ttMap := make(map[int64]SpriteTypeTree) 213 | for k, v := range ttPathIDMap { 214 | subMapPath := filepath.Join(scanner.Root, characterPrefix, id, v) 215 | subMapJson, err := os.ReadFile(subMapPath) 216 | if err != nil { 217 | return nil, errors.WithStack(err) 218 | } 219 | 220 | var tt SpriteTypeTree 221 | err = json.Unmarshal(subMapJson, &tt) 222 | if err != nil { 223 | return nil, errors.WithStack(err) 224 | } 225 | ttMap[k] = tt 226 | } 227 | 228 | return ttMap, nil 229 | } 230 | 231 | type ( 232 | CharacterSpriteHubGroup struct { 233 | SpriteGroups []CharacterSpriteHub `json:"spriteGroups"` 234 | } 235 | CharacterSpriteHub struct { 236 | Sprites []CharacterSprite `json:"sprites"` 237 | FacePos struct { 238 | X float64 `json:"x"` 239 | Y float64 `json:"y"` 240 | } `json:"FacePos"` 241 | FaceSize struct { 242 | X float64 `json:"x"` 243 | Y float64 `json:"y"` 244 | } `json:"FaceSize"` 245 | } 246 | CharacterSprite struct { 247 | Sprite struct { 248 | MPathID int64 `json:"m_PathID"` 249 | } `json:"sprite"` 250 | AlphaTex struct { 251 | MPathID int64 `json:"m_PathID"` 252 | } `json:"alphaTex"` 253 | IsWholeBody int `json:"isWholeBody"` 254 | } 255 | SpriteTypeTree struct { 256 | PixelsToUnits float64 `json:"m_PixelsToUnits"` 257 | } 258 | ) 259 | 260 | func (c *CharacterSpriteHubGroup) toArt(id string, pathIDMap map[int64]string, ttMap map[int64]SpriteTypeTree) (a CharacterArt) { 261 | convertSpriteHubToArt := func(i CharacterSpriteHub) (o CharacterArtBodyVariation) { return i.toArt(pathIDMap, ttMap) } 262 | a = CharacterArt{ 263 | ID: id, 264 | Kind: "character", 265 | BodyVariations: cols.Map(c.SpriteGroups, convertSpriteHubToArt), 266 | } 267 | return 268 | } 269 | func (c *CharacterSpriteHub) toArt(pathIDMap map[int64]string, ttMap map[int64]SpriteTypeTree) (a CharacterArtBodyVariation) { 270 | convertSpriteToArt := func(i CharacterSprite) (o CharacterArtFaceVariation) { return i.toArt(pathIDMap, ttMap) } 271 | a = CharacterArtBodyVariation{ 272 | BodySprite: "", 273 | BodySpriteAlpha: "", 274 | FaceRectangle: image.Rect( 275 | int(math.Round(c.FacePos.X)), 276 | int(math.Round(c.FacePos.Y)), 277 | int(math.Round(c.FacePos.X+c.FaceSize.X)), 278 | int(math.Round(c.FacePos.Y+c.FaceSize.Y)), 279 | ), 280 | FaceVariations: cols.Map(c.Sprites, convertSpriteToArt), 281 | PixelToUnits: 1.0, 282 | } 283 | // If the face pos is valid, then extract the last face variation as the body. 284 | // Otherwise, all variations contain the whole body. 285 | if c.FacePos.X >= 0 && c.FacePos.Y >= 0 { 286 | lastVariation := a.FaceVariations[len(a.FaceVariations)-1] 287 | a.BodySprite = lastVariation.FaceSprite 288 | a.BodySpriteAlpha = lastVariation.FaceSpriteAlpha 289 | a.PixelToUnits = lastVariation.PixelToUnits 290 | a.FaceVariations = a.FaceVariations[:len(a.FaceVariations)-1] 291 | } else { 292 | for i := range a.FaceVariations { 293 | a.FaceVariations[i].WholeBody = true 294 | } 295 | } 296 | return 297 | } 298 | func (c *CharacterSprite) toArt(pathIDMap map[int64]string, ttMap map[int64]SpriteTypeTree) (a CharacterArtFaceVariation) { 299 | a = CharacterArtFaceVariation{ 300 | FaceSprite: pathIDMap[c.Sprite.MPathID], 301 | FaceSpriteAlpha: pathIDMap[c.AlphaTex.MPathID], 302 | WholeBody: c.IsWholeBody != 0, 303 | PixelToUnits: ttMap[c.Sprite.MPathID].PixelsToUnits / 100.0, 304 | } 305 | return 306 | } 307 | --------------------------------------------------------------------------------