├── default.pgo ├── testdata ├── images │ └── result │ │ └── .gitkeep ├── hath.txt ├── ehdl.txt └── x.json ├── docs ├── images │ ├── admin.png │ ├── catalogue.png │ ├── settings.png │ ├── editing_gallery.png │ ├── series_listing.png │ ├── catalogue_grouped.png │ └── thumbnails │ │ ├── admin.png │ │ ├── catalogue.png │ │ ├── settings.png │ │ ├── series_listing.png │ │ ├── editing_gallery.png │ │ └── catalogue_grouped.png ├── logo-large.png ├── logo-small.png ├── CONTRIBUTING.md ├── nginx.conf ├── docker-compose.rclone.yml ├── LIBRARY.md ├── docker-compose.example.yml └── ENVIRONMENTALS.md ├── .gitignore ├── .dockerignore ├── pkg ├── constants │ ├── regex.go │ └── language.go ├── types │ └── sqlite │ │ ├── model │ │ ├── gallery_tag.go │ │ ├── library.go │ │ ├── tag.go │ │ ├── goose_db_version.go │ │ ├── session.go │ │ ├── user.go │ │ ├── gallery_pref.go │ │ ├── reference.go │ │ └── gallery.go │ │ └── table │ │ ├── tag.go │ │ ├── library.go │ │ ├── gallery_tag.go │ │ ├── session.go │ │ ├── goose_db_version.go │ │ ├── gallery_pref.go │ │ ├── user.go │ │ ├── reference.go │ │ └── gallery.go ├── db │ ├── migrations │ │ ├── 20240121071505_add_salt_col_to_user.sql │ │ ├── 20230713160822_add_deleted_col_to_gallery.sql │ │ ├── 20240109011130_add_page_thumbnails_col_to_gallery.sql │ │ ├── 20240322225051_add_meta_title_hash_column.sql │ │ ├── 20240121073330_add_bcrypt_pw_col_to_user.sql │ │ ├── 20220227131049_modify_gallery_columns.sql │ │ └── 20220106011520_create_tables.sql │ ├── db.go │ ├── library.go │ ├── validations.go │ └── user.go ├── library │ ├── library.go │ └── scan.go ├── utils │ ├── orderedMap.go │ ├── validations.go │ ├── utils.go │ ├── archive.go │ ├── password.go │ └── image.go ├── log │ └── log.go ├── metadata │ ├── hath.go │ ├── ehdl.go │ ├── metadata_test.go │ ├── title.go │ ├── x.go │ └── scan.go ├── api │ ├── task.go │ ├── jwt.go │ ├── helpers.go │ ├── api.go │ └── gallery.go └── cache │ ├── disk.go │ ├── processing_status.go │ └── gallery.go ├── internal └── config │ ├── db.go │ ├── path.go │ └── config.go ├── .github ├── workflows │ ├── go.yml │ ├── container-image.yml │ └── codeql-analysis.yml ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md └── FUNDING.yml ├── Dockerfile ├── cmd └── mangatsu-server │ └── main.go ├── go.mod ├── example.env └── README.md /default.pgo: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /testdata/images/result/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/images/admin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mangatsu/server/HEAD/docs/images/admin.png -------------------------------------------------------------------------------- /docs/logo-large.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mangatsu/server/HEAD/docs/logo-large.png -------------------------------------------------------------------------------- /docs/logo-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mangatsu/server/HEAD/docs/logo-small.png -------------------------------------------------------------------------------- /docs/images/catalogue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mangatsu/server/HEAD/docs/images/catalogue.png -------------------------------------------------------------------------------- /docs/images/settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mangatsu/server/HEAD/docs/images/settings.png -------------------------------------------------------------------------------- /docs/images/editing_gallery.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mangatsu/server/HEAD/docs/images/editing_gallery.png -------------------------------------------------------------------------------- /docs/images/series_listing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mangatsu/server/HEAD/docs/images/series_listing.png -------------------------------------------------------------------------------- /docs/images/catalogue_grouped.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mangatsu/server/HEAD/docs/images/catalogue_grouped.png -------------------------------------------------------------------------------- /docs/images/thumbnails/admin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mangatsu/server/HEAD/docs/images/thumbnails/admin.png -------------------------------------------------------------------------------- /docs/images/thumbnails/catalogue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mangatsu/server/HEAD/docs/images/thumbnails/catalogue.png -------------------------------------------------------------------------------- /docs/images/thumbnails/settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mangatsu/server/HEAD/docs/images/thumbnails/settings.png -------------------------------------------------------------------------------- /docs/images/thumbnails/series_listing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mangatsu/server/HEAD/docs/images/thumbnails/series_listing.png -------------------------------------------------------------------------------- /docs/images/thumbnails/editing_gallery.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mangatsu/server/HEAD/docs/images/thumbnails/editing_gallery.png -------------------------------------------------------------------------------- /docs/images/thumbnails/catalogue_grouped.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mangatsu/server/HEAD/docs/images/thumbnails/catalogue_grouped.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IDE, editors 2 | /.idea 3 | 4 | # data 5 | /data 6 | mangatsu.sqlite 7 | /devarchive 8 | /.env 9 | docker-compose.yml 10 | /mangatsu-server.exe 11 | 12 | # testing 13 | test.http 14 | coverage.out 15 | testdata/images/result/* 16 | !testdata/images/result/.gitkeep -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # IDE, editors 2 | /.idea 3 | 4 | # data 5 | /devarchive 6 | /data 7 | /.env 8 | 9 | # testing 10 | /test.http 11 | /coverage.out 12 | 13 | # other 14 | /.github 15 | /Dockerfile 16 | /.dockerignore 17 | /.gitignore 18 | /docs 19 | /example.env 20 | /README.md 21 | -------------------------------------------------------------------------------- /pkg/constants/regex.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | import "regexp" 4 | 5 | var ArchiveExtensions = regexp.MustCompile(`\.(?:zip|cbz|rar|cbr|7z)$`) 6 | var MetaExtensions = regexp.MustCompile(`\.(?:json|txt)$`) 7 | var ImageExtensions = regexp.MustCompile(`\.(?:jpe?g|png|webp|avif|bmp|gif|tiff?|heif)$`) 8 | -------------------------------------------------------------------------------- /testdata/hath.txt: -------------------------------------------------------------------------------- 1 | Title: (C88) [hサークル] とてもエッチなタイトル (魔法少女) 2 | Upload Time: 2022-01-01 12:01 3 | Uploaded By: good uploader 4 | Downloaded: 2022-01-03 14:22 5 | Tags: parody:mahou shoujo, group:hcircle, female:group, female:thigh high boots, artbook 6 | Downloaded from E-Hentai Galleries by the Hentai@Home Downloader <3 7 | -------------------------------------------------------------------------------- /pkg/types/sqlite/model/gallery_tag.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package model 9 | 10 | type GalleryTag struct { 11 | GalleryUUID string 12 | TagID int32 13 | } 14 | -------------------------------------------------------------------------------- /pkg/types/sqlite/model/library.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package model 9 | 10 | type Library struct { 11 | ID int32 `sql:"primary_key"` 12 | Path string 13 | Layout string 14 | } 15 | -------------------------------------------------------------------------------- /pkg/types/sqlite/model/tag.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package model 9 | 10 | type Tag struct { 11 | ID int32 `sql:"primary_key"` 12 | Namespace string 13 | Name string 14 | } 15 | -------------------------------------------------------------------------------- /pkg/db/migrations/20240121071505_add_salt_col_to_user.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | SELECT 'up SQL query'; 4 | ALTER TABLE user 5 | ADD COLUMN salt BLOB NOT NULL default ''; 6 | -- +goose StatementEnd 7 | 8 | -- +goose Down 9 | -- +goose StatementBegin 10 | SELECT 'down SQL query'; 11 | ALTER TABLE user 12 | DROP COLUMN salt; 13 | -- +goose StatementEnd 14 | -------------------------------------------------------------------------------- /pkg/db/migrations/20230713160822_add_deleted_col_to_gallery.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | SELECT 'up SQL query'; 4 | ALTER TABLE gallery 5 | ADD COLUMN deleted boolean NOT NULL DEFAULT false; 6 | -- +goose StatementEnd 7 | 8 | -- +goose Down 9 | -- +goose StatementBegin 10 | SELECT 'down SQL query'; 11 | ALTER TABLE gallery 12 | DROP COLUMN deleted; 13 | -- +goose StatementEnd 14 | -------------------------------------------------------------------------------- /pkg/db/migrations/20240109011130_add_page_thumbnails_col_to_gallery.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | SELECT 'up SQL query'; 4 | ALTER TABLE gallery 5 | ADD COLUMN page_thumbnails int; 6 | -- +goose StatementEnd 7 | 8 | -- +goose Down 9 | -- +goose StatementBegin 10 | SELECT 'down SQL query'; 11 | ALTER TABLE gallery 12 | DROP COLUMN page_thumbnails; 13 | -- +goose StatementEnd 14 | -------------------------------------------------------------------------------- /pkg/db/migrations/20240322225051_add_meta_title_hash_column.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | SELECT 'up SQL query'; 4 | ALTER TABLE reference 5 | ADD COLUMN meta_title_hash TEXT; 6 | -- +goose StatementEnd 7 | 8 | -- +goose Down 9 | -- +goose StatementBegin 10 | SELECT 'down SQL query'; 11 | ALTER TABLE reference 12 | DROP COLUMN meta_title_hash; 13 | -- +goose StatementEnd 14 | -------------------------------------------------------------------------------- /pkg/types/sqlite/model/goose_db_version.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package model 9 | 10 | import ( 11 | "time" 12 | ) 13 | 14 | type GooseDbVersion struct { 15 | ID *int32 `sql:"primary_key"` 16 | VersionID int32 17 | IsApplied int32 18 | Tstamp *time.Time 19 | } 20 | -------------------------------------------------------------------------------- /pkg/types/sqlite/model/session.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package model 9 | 10 | import ( 11 | "time" 12 | ) 13 | 14 | type Session struct { 15 | ID string `sql:"primary_key"` 16 | UserUUID string `sql:"primary_key"` 17 | Name *string 18 | ExpiresAt *time.Time 19 | } 20 | -------------------------------------------------------------------------------- /internal/config/db.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | type DBOptions struct { 8 | Name string 9 | Migrations bool 10 | } 11 | 12 | func dbMigrationsEnabled() bool { 13 | // Disabled only when explicitly set to false. 14 | return os.Getenv("MTSU_DB_MIGRATIONS") != "false" 15 | } 16 | 17 | func dbName() string { 18 | value := os.Getenv("MTSU_DB_NAME") 19 | if value == "" { 20 | return "mangatsu" 21 | } 22 | return value 23 | } 24 | -------------------------------------------------------------------------------- /pkg/types/sqlite/model/user.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package model 9 | 10 | import ( 11 | "time" 12 | ) 13 | 14 | type User struct { 15 | UUID string 16 | Username string 17 | Password []byte 18 | Salt []byte 19 | Role int32 20 | BcryptPw *string 21 | CreatedAt time.Time 22 | UpdatedAt time.Time 23 | } 24 | -------------------------------------------------------------------------------- /pkg/types/sqlite/model/gallery_pref.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package model 9 | 10 | import ( 11 | "time" 12 | ) 13 | 14 | type GalleryPref struct { 15 | UserUUID string `sql:"primary_key"` 16 | GalleryUUID string `sql:"primary_key"` 17 | Progress int32 18 | FavoriteGroup *string 19 | UpdatedAt time.Time 20 | } 21 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v4 18 | with: 19 | go-version-file: go.mod 20 | 21 | - name: Build 22 | run: go build -v github.com/Mangatsu/server/cmd/mangatsu-server 23 | 24 | - name: Test 25 | run: go test -v ./... 26 | -------------------------------------------------------------------------------- /pkg/types/sqlite/model/reference.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package model 9 | 10 | type Reference struct { 11 | GalleryUUID string `sql:"primary_key"` 12 | MetaInternal bool 13 | MetaPath *string 14 | MetaMatch *int32 15 | Urls *string 16 | ExhGid *int32 17 | ExhToken *string 18 | AnilistID *int32 19 | MetaTitleHash *string 20 | } 21 | -------------------------------------------------------------------------------- /pkg/library/library.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "github.com/Mangatsu/server/pkg/log" 5 | "go.uber.org/zap" 6 | "io" 7 | "io/fs" 8 | ) 9 | 10 | func closeFile(f interface{ Close() error }) { 11 | err := f.Close() 12 | if err != nil { 13 | log.Z.Debug("failed to close file", zap.String("err", err.Error())) 14 | return 15 | } 16 | } 17 | 18 | func ReadAll(filesystem fs.FS, filename string) ([]byte, error) { 19 | archive, err := filesystem.Open(filename) 20 | if err != nil { 21 | return nil, err 22 | } 23 | 24 | defer closeFile(archive) 25 | 26 | return io.ReadAll(archive) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/utils/orderedMap.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | type OrderedMap struct { 4 | Keys []string 5 | Map map[string]interface{} 6 | } 7 | 8 | func NewOrderedMap() *OrderedMap { 9 | return &OrderedMap{ 10 | Keys: make([]string, 0), 11 | Map: make(map[string]interface{}), 12 | } 13 | } 14 | 15 | func (om *OrderedMap) Set(key string, value interface{}) { 16 | if _, exists := om.Map[key]; !exists { 17 | om.Keys = append(om.Keys, key) 18 | } 19 | om.Map[key] = value 20 | } 21 | 22 | func (om *OrderedMap) Get(key string) (interface{}, bool) { 23 | val, exists := om.Map[key] 24 | return val, exists 25 | } 26 | 27 | func (om *OrderedMap) OrderedKeys() []string { 28 | return om.Keys 29 | } 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.22-alpine as build 2 | 3 | RUN mkdir /usr/app/ 4 | WORKDIR /usr/app/ 5 | COPY . . 6 | 7 | # Unit tests 8 | RUN apk add build-base && go test -buildvcs=false ./... 9 | 10 | RUN GOOS=linux GOARCH=amd64 go build -buildvcs=false -ldflags="-w -s" -o /go/bin/mangatsu-server github.com/Mangatsu/server/cmd/mangatsu-server 11 | 12 | FROM alpine 13 | 14 | RUN adduser -D mangatsu && mkdir /home/mangatsu/app && mkdir /home/mangatsu/data 15 | USER mangatsu 16 | WORKDIR /home/mangatsu/app 17 | 18 | COPY --from=build /go/bin/mangatsu-server /home/mangatsu/app/mangatsu-server 19 | COPY --from=build /usr/app/pkg/db/migrations /home/mangatsu/app/pkg/db/migrations 20 | 21 | EXPOSE 5050 22 | CMD [ "sh", "-c", "/home/mangatsu/app/mangatsu-server" ] 23 | -------------------------------------------------------------------------------- /pkg/log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "log" 5 | 6 | "go.uber.org/zap" 7 | "go.uber.org/zap/zapcore" 8 | ) 9 | 10 | type Environment string 11 | 12 | const ( 13 | Production Environment = "production" 14 | Development = "development" 15 | ) 16 | 17 | var Z *zap.Logger 18 | var S *zap.SugaredLogger 19 | 20 | func InitializeLogger(env Environment, logLevel zapcore.Level) { 21 | var loggerErr error 22 | if env == Production { 23 | Z, loggerErr = zap.NewProduction(zap.IncreaseLevel(logLevel)) 24 | } else { 25 | Z, loggerErr = zap.NewDevelopment(zap.IncreaseLevel(logLevel)) 26 | } 27 | 28 | if loggerErr != nil { 29 | log.Fatal("Failed to initialize zap logger: ", loggerErr) 30 | } 31 | 32 | defer Z.Sync() 33 | S = Z.Sugar() 34 | } 35 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information if applicable):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. Chrome, Safari] 29 | - Version [e.g. 22] 30 | 31 | **Additional context** 32 | Add any other context about the problem here. 33 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [CrescentKohana] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 13 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 14 | -------------------------------------------------------------------------------- /pkg/types/sqlite/model/gallery.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package model 9 | 10 | import ( 11 | "time" 12 | ) 13 | 14 | type Gallery struct { 15 | UUID string `sql:"primary_key"` 16 | LibraryID int32 17 | ArchivePath string 18 | Title string 19 | TitleNative *string 20 | TitleTranslated *string 21 | Category *string 22 | Series *string 23 | Released *string 24 | Language *string 25 | Translated *bool 26 | Nsfw bool 27 | Hidden bool 28 | ImageCount *int32 29 | ArchiveSize *int32 30 | ArchiveHash *string 31 | Thumbnail *string 32 | CreatedAt time.Time 33 | UpdatedAt time.Time 34 | Deleted bool 35 | PageThumbnails *int32 36 | } 37 | -------------------------------------------------------------------------------- /docs/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## git 4 | 5 | Use [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) when making commits. 6 | 7 | ## Setup 8 | 9 | ### 🚧 Building and running 10 | 11 | - Copy example.env as .env and change the values according to your needs. 12 | - Build `go build` 13 | - (Optional) Manually initialize development database: `goose -dir pkg/db/migrations sqlite3 ./data/mangatsu.sqlite up` 14 | - Run `backend` (`backend.exe` on Windows) 15 | 16 | ### 💾 Database migrations 17 | 18 | - Automatically run when the server is launched. Can be disabled by setting `MTSU_DB_MIGRATIONS=false` in `.env`. 19 | - Manually: `goose -dir pkg/db/migrations sqlite3 ./PATH/TO/mangatsu.sqlite ` 20 | - To use goose as a tool: `go install github.com/pressly/goose/v3/cmd/goose@latest` 21 | - Automatic models and types: `jet -dsn="file:///full/path/to/data.sqlite" -path=types` based on the db schema 22 | - To use install jet as a tool: `go install github.com/go-jet/jet/v2/cmd/jet@latest` 23 | 24 | ### 🔬 Testing 25 | 26 | - Test: `go test ./... -v -coverprofile "coverage.out"` 27 | - Show coverage report: `go tool cover -html "coverage.out"` 28 | 29 | ### 📝 Generating docs 30 | 31 | - Run `godoc -http=localhost:8080` 32 | - Go to `http://localhost:8080/pkg/#thirdparty` 33 | 34 | ## Requirements 35 | 36 | - Go 1.21+ 37 | - SQLite3 38 | - Docker (optional) 39 | -------------------------------------------------------------------------------- /pkg/db/migrations/20240121073330_add_bcrypt_pw_col_to_user.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | SELECT 'up SQL query'; 4 | CREATE TABLE user2 5 | ( 6 | uuid text UNIQUE NOT NULL, 7 | username text UNIQUE NOT NULL, 8 | password blob NOT NULL, 9 | salt blob NOT NULL, 10 | role integer NOT NULL DEFAULT 10, 11 | bcrypt_pw text, 12 | created_at datetime NOT NULL, 13 | updated_at datetime NOT NULL 14 | ); 15 | INSERT INTO user2 16 | SELECT uuid, 17 | username, 18 | '' AS password, 19 | '' AS salt, 20 | role, 21 | password AS bcrypt_pw, 22 | created_at, 23 | updated_at 24 | FROM user; 25 | 26 | DROP TABLE user; 27 | 28 | ALTER TABLE user2 29 | RENAME TO user; 30 | -- +goose StatementEnd 31 | 32 | -- +goose Down 33 | -- +goose StatementBegin 34 | SELECT 'down SQL query'; 35 | CREATE TABLE user2 36 | ( 37 | uuid text UNIQUE NOT NULL, 38 | username text UNIQUE NOT NULL, 39 | password blob NOT NULL, 40 | role integer NOT NULL DEFAULT 10, 41 | created_at datetime NOT NULL, 42 | updated_at datetime NOT NULL 43 | ); 44 | 45 | INSERT INTO user2 46 | SELECT uuid, 47 | username, 48 | bcrypt_pw AS password, 49 | role, 50 | created_at, 51 | updated_at 52 | FROM user; 53 | 54 | DROP TABLE user; 55 | 56 | ALTER TABLE user2 57 | RENAME TO user; 58 | -- +goose StatementEnd 59 | -------------------------------------------------------------------------------- /cmd/mangatsu-server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/Mangatsu/server/internal/config" 7 | "github.com/Mangatsu/server/pkg/api" 8 | "github.com/Mangatsu/server/pkg/cache" 9 | "github.com/Mangatsu/server/pkg/db" 10 | "github.com/Mangatsu/server/pkg/log" 11 | "github.com/Mangatsu/server/pkg/utils" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | func main() { 16 | config.LoadEnv() 17 | log.InitializeLogger(config.AppEnvironment, config.LogLevel) 18 | config.SetEnv() 19 | cache.InitPhysicalCache() 20 | db.InitDB() 21 | db.EnsureLatestVersion() 22 | 23 | username, password := config.GetInitialAdmin() 24 | users, err := db.GetUser(username) 25 | if err != nil { 26 | log.Z.Error("error fetching initial admin", zap.String("error", err.Error())) 27 | } 28 | 29 | if users == nil || len(users) == 0 { 30 | if err := db.Register(username, password, db.SuperAdmin); err != nil { 31 | log.Z.Fatal("error registering initial admin: ", zap.String("err", err.Error())) 32 | } 33 | } 34 | 35 | // Parse libraries from the environmental and insert/update to the db. 36 | libraries := config.ParseBasePaths() 37 | if err = db.StorePaths(libraries); err != nil { 38 | log.Z.Fatal("error saving library to db: ", zap.String("err", err.Error())) 39 | } 40 | 41 | cache.InitGalleryCache() 42 | cache.InitProcessingStatusCache() 43 | 44 | // Tasks 45 | utils.PeriodicTask(time.Minute, cache.PruneCache) 46 | utils.PeriodicTask(time.Minute, db.PruneExpiredSessions) 47 | 48 | api.LaunchAPI() 49 | } 50 | -------------------------------------------------------------------------------- /docs/nginx.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; listen [::]:80; 3 | server_name mangatsu.example.com; 4 | return 301 https://$server_name$request_uri; 5 | } 6 | 7 | server { 8 | listen 443 ssl http2; listen [::]:443 ssl http2; 9 | 10 | server_name mangatsu.example.com; 11 | 12 | ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem; 13 | ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem; 14 | 15 | location / { 16 | proxy_pass http://localhost:3003; 17 | proxy_http_version 1.1; 18 | proxy_cache_bypass $http_upgrade; 19 | # Proxy headers 20 | proxy_set_header Upgrade $http_upgrade; 21 | proxy_set_header Connection $connection_upgrade; 22 | proxy_set_header Host $host; 23 | proxy_set_header X-Real-IP $remote_addr; 24 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 25 | proxy_set_header X-Forwarded-Proto $scheme; 26 | proxy_set_header X-Forwarded-Host $host; 27 | proxy_set_header X-Forwarded-Port $server_port; 28 | # Proxy timeouts 29 | proxy_connect_timeout 60s; 30 | proxy_send_timeout 60s; 31 | proxy_read_timeout 60s; 32 | } 33 | 34 | # gzip 35 | gzip on; 36 | gzip_vary on; 37 | gzip_proxied any; 38 | gzip_comp_level 6; 39 | gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml; 40 | } 41 | 42 | -------------------------------------------------------------------------------- /pkg/db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "database/sql" 5 | "embed" 6 | "fmt" 7 | 8 | "github.com/Mangatsu/server/internal/config" 9 | "github.com/Mangatsu/server/pkg/log" 10 | _ "github.com/mattn/go-sqlite3" 11 | "github.com/pressly/goose/v3" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | var database *sql.DB 16 | 17 | //go:embed migrations 18 | var embedMigrations embed.FS 19 | 20 | // InitDB initializes the database 21 | func InitDB() { 22 | var dbErr error 23 | database, dbErr = sql.Open("sqlite3", config.BuildDataPath(config.Options.DB.Name+".sqlite")) 24 | if dbErr != nil { 25 | log.Z.Fatal(dbErr.Error()) 26 | } 27 | } 28 | 29 | func db() *sql.DB { 30 | return database 31 | } 32 | 33 | // EnsureLatestVersion ensures that the database is at the latest version by running all migrations. 34 | func EnsureLatestVersion() { 35 | if !config.Options.DB.Migrations { 36 | log.Z.Warn("database migrations are disabled.") 37 | return 38 | } 39 | 40 | // For embedding the migrations in the binary. 41 | goose.SetBaseFS(embedMigrations) 42 | 43 | err := goose.SetDialect("sqlite3") 44 | if err != nil { 45 | log.Z.Fatal("failed setting DB dialect", zap.String("err", err.Error())) 46 | } 47 | 48 | err = goose.Up(db(), "migrations") 49 | fmt.Println("") 50 | if err != nil { 51 | log.Z.Fatal("failed to apply new migrations", zap.String("err", err.Error())) 52 | } 53 | } 54 | 55 | func rollbackTx(tx *sql.Tx) { 56 | err := tx.Rollback() 57 | if err != nil { 58 | log.Z.Debug("failed to rollback transaction", zap.String("err", err.Error())) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /pkg/metadata/hath.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "strings" 7 | 8 | "github.com/Mangatsu/server/pkg/types/sqlite/model" 9 | ) 10 | 11 | // ParseHath parses given text file. Input file is expected to be in the H@H (Hath) format (galleryinfo.txt). 12 | func ParseHath(metaPath string, metaData []byte, internal bool) (model.Gallery, []model.Tag, model.Reference, error) { 13 | gallery := model.Gallery{} 14 | reference := model.Reference{ 15 | MetaPath: &metaPath, 16 | MetaInternal: internal, 17 | Urls: nil, 18 | } 19 | var tags []model.Tag 20 | 21 | buffer := bytes.NewBuffer(metaData) 22 | scanner := bufio.NewScanner(buffer) 23 | 24 | for scanner.Scan() { 25 | line := scanner.Text() 26 | if len(line) == 0 { 27 | continue 28 | } 29 | 30 | if strings.HasPrefix(line, "Title:") { 31 | title := strings.TrimSpace(strings.TrimPrefix(line, "Title:")) 32 | gallery.TitleNative = &title 33 | continue 34 | } 35 | 36 | if strings.HasPrefix(line, "Tags:") { 37 | tagsList := strings.Split(strings.TrimSpace(strings.TrimPrefix(line, "Tags:")), ",") 38 | for _, tag := range tagsList { 39 | splitTag := strings.Split(tag, ":") 40 | if len(splitTag) == 2 { 41 | tags = append(tags, model.Tag{ 42 | Namespace: strings.TrimSpace(splitTag[0]), 43 | Name: strings.TrimSpace(splitTag[1]), 44 | }) 45 | } else if len(splitTag) == 1 { 46 | tags = append(tags, model.Tag{ 47 | Namespace: "other", 48 | Name: strings.TrimSpace(splitTag[0]), 49 | }) 50 | } 51 | 52 | } 53 | } 54 | } 55 | 56 | return gallery, tags, reference, nil 57 | } 58 | -------------------------------------------------------------------------------- /testdata/ehdl.txt: -------------------------------------------------------------------------------- 1 | [CRAZY CIRCLE (Hana)] Oppai Oppai Oppai 2 | [CRAZY CIRCLE (はな)] おっぱいおっぱいおっぱい 3 | https://example.org/g/111111/ffffffffff/ 4 | 5 | Category: Doujinshi 6 | Uploader: good uploader 7 | Posted: 2020-06-01 10:02 8 | Parent: None 9 | Visible: Yes 10 | Language: Japanese   11 | File Size: 69.69 MB 12 | Length: 12 pages 13 | Favorited: 1231 times 14 | Rating: 4.55 15 | 16 | Tags: 17 | > group: crazy circle 18 | > artist: hana 19 | > female: fft threesome, group, stockings 20 | 21 | 22 | 23 | Page 1: https://example.org/s/0000000000/111111-1 24 | Image 1: 01：001.jpg 25 | 26 | Page 2: https://example.org/s/0000000000/111111-2 27 | Image 2: 02：002.jpg 28 | 29 | Page 3: https://example.org/s/0000000000/111111-3 30 | Image 3: 03：003.jpg 31 | 32 | Page 4: https://example.org/s/0000000000/111111-4 33 | Image 4: 04：004.jpg 34 | 35 | Page 5: https://example.org/s/0000000000/111111-5 36 | Image 5: 05：005.jpg 37 | 38 | Page 6: https://example.org/s/0000000000/111111-6 39 | Image 6: 06：006.jpg 40 | 41 | Page 7: https://example.org/s/0000000000/111111-7 42 | Image 7: 07：007.jpg 43 | 44 | Page 8: https://example.org/s/0000000000/111111-8 45 | Image 8: 08：008.jpg 46 | 47 | Page 9: https://example.org/s/0000000000/111111-9 48 | Image 9: 09：009.jpg 49 | 50 | Page 10: https://example.org/s/0000000000/111111-10 51 | Image 10: 10：010.jpg 52 | 53 | Page 11: https://example.org/s/0000000000/111111-11 54 | Image 11: 11：011.jpg 55 | 56 | Page 12: https://example.org/s/0000000000/111111-12 57 | Image 12: 12：012.jpg 58 | 59 | Downloaded at Thu Jul 16 2021 15:00:37 GMT+0300 (Eastern European Summer Time) 60 | 61 | Generated by E-Hentai Downloader. https://github.com/dnsev-h/E-Hentai-Downloader -------------------------------------------------------------------------------- /.github/workflows/container-image.yml: -------------------------------------------------------------------------------- 1 | name: Container Image 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'main' 7 | tags: 8 | - 'v*' 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - name: 'Checkout repo' 16 | uses: actions/checkout@v3 17 | 18 | - name: 'Docker meta' 19 | id: meta 20 | uses: docker/metadata-action@v4 21 | with: 22 | images: | 23 | luukuton/mangatsu-server 24 | ghcr.io/Mangatsu/server 25 | tags: | 26 | type=ref,event=branch 27 | type=semver,pattern={{version}} 28 | type=semver,pattern={{major}}.{{minor}} 29 | type=semver,pattern={{major}} 30 | 31 | - name: Set up Docker Buildx 32 | id: buildx 33 | uses: docker/setup-buildx-action@v2 34 | 35 | - name: 'Login to Docker Hub' 36 | id: login_dockerhub 37 | uses: docker/login-action@v2 38 | with: 39 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 40 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 41 | 42 | - name: 'Login to GitHub Container Registry' 43 | id: login_ghcr 44 | uses: docker/login-action@v2 45 | with: 46 | registry: ghcr.io 47 | username: ${{github.actor}} 48 | password: ${{secrets.GITHUB_TOKEN}} 49 | 50 | - name: 'Build and push container' 51 | id: build_push_container 52 | uses: docker/build-push-action@v4 53 | with: 54 | context: ./ 55 | file: ./Dockerfile 56 | push: true 57 | tags: ${{ steps.meta.outputs.tags }} 58 | 59 | - name: Image digests 60 | run: | 61 | echo ${{ steps.build_push_container.outputs.digest }} 62 | -------------------------------------------------------------------------------- /docs/docker-compose.rclone.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | backend: 5 | hostname: mtsuserver 6 | image: ghcr.io/mangatsu/server:latest 7 | user: 1000:1000 8 | ports: 9 | - '5050:5050' # container:host 10 | restart: always 11 | environment: 12 | MTSU_ENV: production 13 | MTSU_LOG_LEVEL: info 14 | MTSU_INITIAL_ADMIN_NAME: admin 15 | MTSU_INITIAL_ADMIN_PW: admin321 16 | MTSU_BASE_PATHS: freeform1;/archive 17 | MTSU_CACHE_SIZE: 10000 18 | MTSU_CACHE_TTL: 604800 19 | MTSU_DISABLE_CACHE_SERVER: 'false' 20 | MTSU_VISIBILITY: private 21 | MTSU_RESTRICTED_PASSPHRASE: secretpassword 22 | MTSU_REGISTRATIONS: 'false' 23 | MTSU_JWT_SECRET: 9Wag7sMvKl3aF6K5lwIg6TI42ia2f6BstZAVrdJIq8Mp38lnl7UzQMC1qjKyZCBzHFGbbqsA0gKcHqDuyXQAhWoJ0lcx4K5q 24 | MTSU_DOMAIN: example.org 25 | MTSU_STRICT_ACAO: 'false' 26 | MTSU_SECURE: 'true' 27 | MTSU_THUMBNAIL_FORMAT: webp 28 | MTSU_FUZZY_SEARCH_SIMILARITY: 0.7 29 | MTSU_LTR: 'true' 30 | MTSU_HOSTNAME: mtsuserver 31 | MTSU_PORT: 5050 32 | MTSU_DATA_PATH: /data 33 | volumes: 34 | - "/path/to/data:/data" 35 | - "rclonevol:/archive:ro" 36 | 37 | frontend: 38 | hostname: mtsuweb 39 | image: ghcr.io/mangatsu/web:latest 40 | ports: 41 | - '3030:3030' # container:host 42 | restart: always 43 | environment: 44 | NODE_ENV: production 45 | NEXT_PUBLIC_MANGATSU_API_URL: https://mangatsu-api.example.com 46 | NEXT_MANGATSU_IMAGE_HOSTNAME: mangatsu-api.example.com 47 | PORT: 3030 48 | NEXT_PUBLIC_INTERNAL_MANGATSU_API_URL: http://mtsuserver:5050 49 | 50 | volumes: 51 | rclonevol: 52 | driver: rclone 53 | driver_opts: 54 | remote: "manga:/home/user/archive" 55 | allow_other: "true" 56 | vfs_cache_mode: full 57 | vfs_cache_max_age: "720h" 58 | vfs-cache-max-size: "50G" 59 | read_only: "true" -------------------------------------------------------------------------------- /pkg/utils/validations.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | // Alphanumeric characters of any language, dashes, underscores, spaces, and special characters in the session name. 8 | var wideRe = regexp.MustCompile(`^[\p{L}\p{N}\p{Pd}\p{Pc}\p{Zs}\p{Sc}\p{Sk}!?@#$%^&*+]+$`) 9 | 10 | // The username can contain alphanumeric characters, dashes and underscores. 11 | var usernameRe = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`) 12 | 13 | // The password can contain almost all characters except control characters, whitespace and quotes. 14 | var passwordRe = regexp.MustCompile(`^[^\x00-\x1F\x7F\s'"]+$`) 15 | 16 | const minUsernameLength = 2 17 | const maxUsernameLength = 32 18 | const minPasswordLength = 8 19 | const maxPasswordLength = 512 20 | const maxSessionNameLength = 128 21 | const minCookieAge = 60 22 | const maxCookieAge = 365 * 24 * 60 * 60 // year in seconds 23 | 24 | // ClampCookieAge returns a valid cookie age in seconds. 25 | func ClampCookieAge(seconds *int64) int64 { 26 | if seconds == nil { 27 | return maxCookieAge 28 | } 29 | 30 | return Clamp(*seconds, minCookieAge, maxCookieAge) 31 | } 32 | 33 | // IsValidSessionName checks if the session name is valid. 34 | func IsValidSessionName(sessionName *string) bool { 35 | if sessionName == nil { 36 | return true 37 | } 38 | 39 | if !wideRe.MatchString(*sessionName) { 40 | return false 41 | } 42 | 43 | return len(*sessionName) <= maxSessionNameLength 44 | } 45 | 46 | // IsValidUsername checks if the username is valid. 47 | func IsValidUsername(username string) bool { 48 | if len(username) < minUsernameLength || len(username) > maxUsernameLength { 49 | return false 50 | } 51 | 52 | return usernameRe.MatchString(username) 53 | } 54 | 55 | // IsValidPassword checks if the password is valid. 56 | func IsValidPassword(password string) bool { 57 | if len(password) < minPasswordLength || len(password) > maxPasswordLength { 58 | return false 59 | } 60 | 61 | return passwordRe.MatchString(password) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/types/sqlite/table/tag.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package table 9 | 10 | import ( 11 | "github.com/go-jet/jet/v2/sqlite" 12 | ) 13 | 14 | var Tag = newTagTable("", "tag", "") 15 | 16 | type tagTable struct { 17 | sqlite.Table 18 | 19 | //Columns 20 | ID sqlite.ColumnInteger 21 | Namespace sqlite.ColumnString 22 | Name sqlite.ColumnString 23 | 24 | AllColumns sqlite.ColumnList 25 | MutableColumns sqlite.ColumnList 26 | } 27 | 28 | type TagTable struct { 29 | tagTable 30 | 31 | EXCLUDED tagTable 32 | } 33 | 34 | // AS creates new TagTable with assigned alias 35 | func (a TagTable) AS(alias string) *TagTable { 36 | return newTagTable(a.SchemaName(), a.TableName(), alias) 37 | } 38 | 39 | // Schema creates new TagTable with assigned schema name 40 | func (a TagTable) FromSchema(schemaName string) *TagTable { 41 | return newTagTable(schemaName, a.TableName(), a.Alias()) 42 | } 43 | 44 | func newTagTable(schemaName, tableName, alias string) *TagTable { 45 | return &TagTable{ 46 | tagTable: newTagTableImpl(schemaName, tableName, alias), 47 | EXCLUDED: newTagTableImpl("", "excluded", ""), 48 | } 49 | } 50 | 51 | func newTagTableImpl(schemaName, tableName, alias string) tagTable { 52 | var ( 53 | IDColumn = sqlite.IntegerColumn("id") 54 | NamespaceColumn = sqlite.StringColumn("namespace") 55 | NameColumn = sqlite.StringColumn("name") 56 | allColumns = sqlite.ColumnList{IDColumn, NamespaceColumn, NameColumn} 57 | mutableColumns = sqlite.ColumnList{NamespaceColumn, NameColumn} 58 | ) 59 | 60 | return tagTable{ 61 | Table: sqlite.NewTable(schemaName, tableName, alias, allColumns...), 62 | 63 | //Columns 64 | ID: IDColumn, 65 | Namespace: NamespaceColumn, 66 | Name: NameColumn, 67 | 68 | AllColumns: allColumns, 69 | MutableColumns: mutableColumns, 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /pkg/types/sqlite/table/library.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package table 9 | 10 | import ( 11 | "github.com/go-jet/jet/v2/sqlite" 12 | ) 13 | 14 | var Library = newLibraryTable("", "library", "") 15 | 16 | type libraryTable struct { 17 | sqlite.Table 18 | 19 | //Columns 20 | ID sqlite.ColumnInteger 21 | Path sqlite.ColumnString 22 | Layout sqlite.ColumnString 23 | 24 | AllColumns sqlite.ColumnList 25 | MutableColumns sqlite.ColumnList 26 | } 27 | 28 | type LibraryTable struct { 29 | libraryTable 30 | 31 | EXCLUDED libraryTable 32 | } 33 | 34 | // AS creates new LibraryTable with assigned alias 35 | func (a LibraryTable) AS(alias string) *LibraryTable { 36 | return newLibraryTable(a.SchemaName(), a.TableName(), alias) 37 | } 38 | 39 | // Schema creates new LibraryTable with assigned schema name 40 | func (a LibraryTable) FromSchema(schemaName string) *LibraryTable { 41 | return newLibraryTable(schemaName, a.TableName(), a.Alias()) 42 | } 43 | 44 | func newLibraryTable(schemaName, tableName, alias string) *LibraryTable { 45 | return &LibraryTable{ 46 | libraryTable: newLibraryTableImpl(schemaName, tableName, alias), 47 | EXCLUDED: newLibraryTableImpl("", "excluded", ""), 48 | } 49 | } 50 | 51 | func newLibraryTableImpl(schemaName, tableName, alias string) libraryTable { 52 | var ( 53 | IDColumn = sqlite.IntegerColumn("id") 54 | PathColumn = sqlite.StringColumn("path") 55 | LayoutColumn = sqlite.StringColumn("layout") 56 | allColumns = sqlite.ColumnList{IDColumn, PathColumn, LayoutColumn} 57 | mutableColumns = sqlite.ColumnList{PathColumn, LayoutColumn} 58 | ) 59 | 60 | return libraryTable{ 61 | Table: sqlite.NewTable(schemaName, tableName, alias, allColumns...), 62 | 63 | //Columns 64 | ID: IDColumn, 65 | Path: PathColumn, 66 | Layout: LayoutColumn, 67 | 68 | AllColumns: allColumns, 69 | MutableColumns: mutableColumns, 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /pkg/types/sqlite/table/gallery_tag.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package table 9 | 10 | import ( 11 | "github.com/go-jet/jet/v2/sqlite" 12 | ) 13 | 14 | var GalleryTag = newGalleryTagTable("", "gallery_tag", "") 15 | 16 | type galleryTagTable struct { 17 | sqlite.Table 18 | 19 | //Columns 20 | GalleryUUID sqlite.ColumnString 21 | TagID sqlite.ColumnInteger 22 | 23 | AllColumns sqlite.ColumnList 24 | MutableColumns sqlite.ColumnList 25 | } 26 | 27 | type GalleryTagTable struct { 28 | galleryTagTable 29 | 30 | EXCLUDED galleryTagTable 31 | } 32 | 33 | // AS creates new GalleryTagTable with assigned alias 34 | func (a GalleryTagTable) AS(alias string) *GalleryTagTable { 35 | return newGalleryTagTable(a.SchemaName(), a.TableName(), alias) 36 | } 37 | 38 | // Schema creates new GalleryTagTable with assigned schema name 39 | func (a GalleryTagTable) FromSchema(schemaName string) *GalleryTagTable { 40 | return newGalleryTagTable(schemaName, a.TableName(), a.Alias()) 41 | } 42 | 43 | func newGalleryTagTable(schemaName, tableName, alias string) *GalleryTagTable { 44 | return &GalleryTagTable{ 45 | galleryTagTable: newGalleryTagTableImpl(schemaName, tableName, alias), 46 | EXCLUDED: newGalleryTagTableImpl("", "excluded", ""), 47 | } 48 | } 49 | 50 | func newGalleryTagTableImpl(schemaName, tableName, alias string) galleryTagTable { 51 | var ( 52 | GalleryUUIDColumn = sqlite.StringColumn("gallery_uuid") 53 | TagIDColumn = sqlite.IntegerColumn("tag_id") 54 | allColumns = sqlite.ColumnList{GalleryUUIDColumn, TagIDColumn} 55 | mutableColumns = sqlite.ColumnList{GalleryUUIDColumn, TagIDColumn} 56 | ) 57 | 58 | return galleryTagTable{ 59 | Table: sqlite.NewTable(schemaName, tableName, alias, allColumns...), 60 | 61 | //Columns 62 | GalleryUUID: GalleryUUIDColumn, 63 | TagID: TagIDColumn, 64 | 65 | AllColumns: allColumns, 66 | MutableColumns: mutableColumns, 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /testdata/x.json: -------------------------------------------------------------------------------- 1 | { 2 | "gallery_info": { 3 | "title": "(C99) [doujin circle (some artist)] very lewd title (Magical Girls) [DL]", 4 | "title_original": "(C99) [同人サークル (とあるアーティスト)] とてもエッチなタイトル (魔法少女) [DL版]", 5 | "link": "https://exh.org/g/1/abc", 6 | "category": "doujinshi", 7 | "tags": { 8 | "parody": [ 9 | "Magical Girls" 10 | ], 11 | "female": [ 12 | "swimsuit", 13 | "yuri" 14 | ] 15 | }, 16 | "language": "Japanese", 17 | "translated": false, 18 | "upload_date": [ 19 | 2021, 20 | 12, 21 | 31, 22 | 12, 23 | 0, 24 | 0 25 | ], 26 | "source": { 27 | "site": "exh", 28 | "gid": 1, 29 | "token": "abc", 30 | "parent_gallery": null, 31 | "newer_versions": [] 32 | } 33 | }, 34 | "gallery_info_full": { 35 | "gallery": { 36 | "gid": 1, 37 | "token": "abc" 38 | }, 39 | "title": "(C99) [doujin circle (some artist)] very lewd title (Magical Girls) [DL]", 40 | "title_original": "(C99) [同人サークル (とあるアーティスト)] とてもエッチなタイトル (魔法少女) [DL版]", 41 | "date_uploaded": 1640944800000, 42 | "category": "doujinshi", 43 | "uploader": "example", 44 | "rating": { 45 | "average": 4.91, 46 | "count": 300 47 | }, 48 | "parent": null, 49 | "newer_versions": [], 50 | "image_count": 30, 51 | "images_resized": false, 52 | "total_file_size_approx": 11639011, 53 | "visible": true, 54 | "visible_reason": "", 55 | "language": "Japanese", 56 | "translated": false, 57 | "tags": { 58 | "parody": [ 59 | "Magical Girls" 60 | ], 61 | "character": [ 62 | "magical girl" 63 | ], 64 | "group": [ 65 | "doujin circle" 66 | ], 67 | "artist": [ 68 | "some artist" 69 | ], 70 | "female": [ 71 | "swimsuit", 72 | "tanlines", 73 | "yuri" 74 | ], 75 | "misc": [ 76 | "group" 77 | ] 78 | }, 79 | "tags_have_namespace": true, 80 | "source": "html", 81 | "source_site": "exh", 82 | "date_generated": 1622908495000 83 | } 84 | } -------------------------------------------------------------------------------- /pkg/types/sqlite/table/session.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package table 9 | 10 | import ( 11 | "github.com/go-jet/jet/v2/sqlite" 12 | ) 13 | 14 | var Session = newSessionTable("", "session", "") 15 | 16 | type sessionTable struct { 17 | sqlite.Table 18 | 19 | //Columns 20 | ID sqlite.ColumnString 21 | UserUUID sqlite.ColumnString 22 | Name sqlite.ColumnString 23 | ExpiresAt sqlite.ColumnTimestamp 24 | 25 | AllColumns sqlite.ColumnList 26 | MutableColumns sqlite.ColumnList 27 | } 28 | 29 | type SessionTable struct { 30 | sessionTable 31 | 32 | EXCLUDED sessionTable 33 | } 34 | 35 | // AS creates new SessionTable with assigned alias 36 | func (a SessionTable) AS(alias string) *SessionTable { 37 | return newSessionTable(a.SchemaName(), a.TableName(), alias) 38 | } 39 | 40 | // Schema creates new SessionTable with assigned schema name 41 | func (a SessionTable) FromSchema(schemaName string) *SessionTable { 42 | return newSessionTable(schemaName, a.TableName(), a.Alias()) 43 | } 44 | 45 | func newSessionTable(schemaName, tableName, alias string) *SessionTable { 46 | return &SessionTable{ 47 | sessionTable: newSessionTableImpl(schemaName, tableName, alias), 48 | EXCLUDED: newSessionTableImpl("", "excluded", ""), 49 | } 50 | } 51 | 52 | func newSessionTableImpl(schemaName, tableName, alias string) sessionTable { 53 | var ( 54 | IDColumn = sqlite.StringColumn("id") 55 | UserUUIDColumn = sqlite.StringColumn("user_uuid") 56 | NameColumn = sqlite.StringColumn("name") 57 | ExpiresAtColumn = sqlite.TimestampColumn("expires_at") 58 | allColumns = sqlite.ColumnList{IDColumn, UserUUIDColumn, NameColumn, ExpiresAtColumn} 59 | mutableColumns = sqlite.ColumnList{NameColumn, ExpiresAtColumn} 60 | ) 61 | 62 | return sessionTable{ 63 | Table: sqlite.NewTable(schemaName, tableName, alias, allColumns...), 64 | 65 | //Columns 66 | ID: IDColumn, 67 | UserUUID: UserUUIDColumn, 68 | Name: NameColumn, 69 | ExpiresAt: ExpiresAtColumn, 70 | 71 | AllColumns: allColumns, 72 | MutableColumns: mutableColumns, 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /docs/LIBRARY.md: -------------------------------------------------------------------------------- 1 | # 📂 Directory structure 2 | **Multiple root directories are supported.** I suggest creating a structured format for proper long-running manga, and a freeform structure for doujinshi and other art collections. Examples follow: 3 | 4 | - **Freeform**: galleries can be up to three levels deep. Good for doujinshi, one-shots and other more unstructured collections. 5 | - External JSON metadata files have to be placed in the same level as the gallery archive. Preferably having the same name as the gallery archive. If no exact match is found, filename close enough will be used instead. 6 | 7 | ``` 8 | 📂 freeform 9 | ├── 📂 doujinshi 10 | │ ├──── 📂 oppai 11 | │ │ ├──── 📦 [Group (Artist)] Ecchi Doujinshi.cbz 12 | │ │ └──── 📄 [Group (Artist)] Ecchi Doujinshi.json 13 | │ ├──── 📦 (C99) [Group (Artist)] elfs.zip 14 | │ ├──── 📄 (C99) [Group (Artist)] elfs.json 15 | │ └──── 📦 (C88) [kroup (author, another author)] Tankoubon [DL].zip (JSON or TXT metafile inside) 16 | ├── 📂 art 17 | │ ├──── 📂 [Artist] Pixiv collection 18 | │ │ ├──── 🖼️ 0001.jpg 19 | │ │ ├────... 20 | │ │ └──── 🖼️ 0300.jpg 21 | │ ├──── 📦 art collection y.rar 22 | │ └──── 📄 art collection y.json 23 | └── 📦 (C93) [group (artist)] Lonely doujinshi (Magical Girls).cbz 24 | ``` 25 | 26 | - **Structured**: galleries follow a strict structure. Good for long-running manga (shounen, seinen etc). 27 | - `Manga -> Volumes -> Chapters`, `Manga -> Volumes` or `Manga -> Chapters` 28 | - 'Series' will be set to the name of the 1st level directory except for galleries in the root directory. 29 | 30 | ``` 31 | 📂 structured 32 | ├── 📕 Manga 1 33 | │ ├── 📦 Volume 1.cbz 34 | │ ├── 📦 Volume 2.cbz 35 | │ ├── 📦 Volume 3.cbz 36 | │ └── 📦 Volume 4.zip 37 | ├── 📘 Manga 2 38 | │ └── 📂 Vol. 1 39 | │ ├── 🖼️ 0001.jpg 40 | │ ├── ... 41 | │ └── 🖼️ 0140.jpg 42 | ├── 📘 Manga 3 43 | │ └── 📂 Vol. 1 44 | │ ├── 📦 Chapter 1.zip 45 | │ ├── 📦 Chapter 2.zip 46 | │ └── 📦 Chapter 3.rar 47 | ├── 📗 Manga 4 48 | │ ├── 📦 Chapter 1.zip 49 | │ ├── ... 50 | │ └── 📦 Chapter 30.rar 51 | └── 📦 One Shot Manga.rar 52 | ``` 53 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Mangatsu/server 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/adrg/strutil v0.3.1 7 | github.com/chai2010/webp v1.1.1 8 | github.com/disintegration/imaging v1.6.2 9 | github.com/djherbis/atime v1.1.0 10 | github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb 11 | github.com/go-jet/jet/v2 v2.11.0 12 | github.com/golang-jwt/jwt/v5 v5.2.1 13 | github.com/google/uuid v1.6.0 14 | github.com/gorilla/mux v1.8.1 15 | github.com/joho/godotenv v1.5.1 16 | github.com/mattn/go-sqlite3 v1.14.22 17 | github.com/mholt/archiver/v4 v4.0.0-alpha.8 18 | github.com/pressly/goose/v3 v3.19.2 19 | github.com/rs/cors v1.11.0 20 | github.com/weppos/publicsuffix-go v0.30.2 21 | go.uber.org/zap v1.27.0 22 | golang.org/x/crypto v0.31.0 23 | ) 24 | 25 | require ( 26 | github.com/andybalholm/brotli v1.1.0 // indirect 27 | github.com/bodgit/plumbing v1.3.0 // indirect 28 | github.com/bodgit/sevenzip v1.5.0 // indirect 29 | github.com/bodgit/windows v1.0.1 // indirect 30 | github.com/davecgh/go-spew v1.1.1 // indirect 31 | github.com/dsnet/compress v0.0.1 // indirect 32 | github.com/golang/snappy v0.0.4 // indirect 33 | github.com/hashicorp/errwrap v1.1.0 // indirect 34 | github.com/hashicorp/go-multierror v1.1.1 // indirect 35 | github.com/klauspost/compress v1.17.7 // indirect 36 | github.com/klauspost/pgzip v1.2.6 // indirect 37 | github.com/mfridman/interpolate v0.0.2 // indirect 38 | github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect 39 | github.com/pierrec/lz4/v4 v4.1.21 // indirect 40 | github.com/pmezard/go-difflib v1.0.0 // indirect 41 | github.com/sethvargo/go-retry v0.2.4 // indirect 42 | github.com/stretchr/testify v1.9.0 // indirect 43 | github.com/therootcompany/xz v1.0.1 // indirect 44 | github.com/ulikunitz/xz v0.5.11 // indirect 45 | go.uber.org/multierr v1.11.0 // indirect 46 | go4.org v0.0.0-20230225012048-214862532bf5 // indirect 47 | golang.org/x/image v0.18.0 // indirect 48 | golang.org/x/net v0.23.0 // indirect 49 | golang.org/x/sync v0.10.0 // indirect 50 | golang.org/x/sys v0.28.0 // indirect 51 | golang.org/x/text v0.21.0 // indirect 52 | gopkg.in/yaml.v3 v3.0.1 // indirect 53 | ) 54 | 55 | // fix ambiguous import 56 | replace google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240318140521-94a12d6c2237 57 | -------------------------------------------------------------------------------- /pkg/types/sqlite/table/goose_db_version.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package table 9 | 10 | import ( 11 | "github.com/go-jet/jet/v2/sqlite" 12 | ) 13 | 14 | var GooseDbVersion = newGooseDbVersionTable("", "goose_db_version", "") 15 | 16 | type gooseDbVersionTable struct { 17 | sqlite.Table 18 | 19 | //Columns 20 | ID sqlite.ColumnInteger 21 | VersionID sqlite.ColumnInteger 22 | IsApplied sqlite.ColumnInteger 23 | Tstamp sqlite.ColumnTimestamp 24 | 25 | AllColumns sqlite.ColumnList 26 | MutableColumns sqlite.ColumnList 27 | } 28 | 29 | type GooseDbVersionTable struct { 30 | gooseDbVersionTable 31 | 32 | EXCLUDED gooseDbVersionTable 33 | } 34 | 35 | // AS creates new GooseDbVersionTable with assigned alias 36 | func (a GooseDbVersionTable) AS(alias string) *GooseDbVersionTable { 37 | return newGooseDbVersionTable(a.SchemaName(), a.TableName(), alias) 38 | } 39 | 40 | // Schema creates new GooseDbVersionTable with assigned schema name 41 | func (a GooseDbVersionTable) FromSchema(schemaName string) *GooseDbVersionTable { 42 | return newGooseDbVersionTable(schemaName, a.TableName(), a.Alias()) 43 | } 44 | 45 | func newGooseDbVersionTable(schemaName, tableName, alias string) *GooseDbVersionTable { 46 | return &GooseDbVersionTable{ 47 | gooseDbVersionTable: newGooseDbVersionTableImpl(schemaName, tableName, alias), 48 | EXCLUDED: newGooseDbVersionTableImpl("", "excluded", ""), 49 | } 50 | } 51 | 52 | func newGooseDbVersionTableImpl(schemaName, tableName, alias string) gooseDbVersionTable { 53 | var ( 54 | IDColumn = sqlite.IntegerColumn("id") 55 | VersionIDColumn = sqlite.IntegerColumn("version_id") 56 | IsAppliedColumn = sqlite.IntegerColumn("is_applied") 57 | TstampColumn = sqlite.TimestampColumn("tstamp") 58 | allColumns = sqlite.ColumnList{IDColumn, VersionIDColumn, IsAppliedColumn, TstampColumn} 59 | mutableColumns = sqlite.ColumnList{VersionIDColumn, IsAppliedColumn, TstampColumn} 60 | ) 61 | 62 | return gooseDbVersionTable{ 63 | Table: sqlite.NewTable(schemaName, tableName, alias, allColumns...), 64 | 65 | //Columns 66 | ID: IDColumn, 67 | VersionID: VersionIDColumn, 68 | IsApplied: IsAppliedColumn, 69 | Tstamp: TstampColumn, 70 | 71 | AllColumns: allColumns, 72 | MutableColumns: mutableColumns, 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /pkg/api/task.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "github.com/Mangatsu/server/pkg/cache" 6 | "github.com/Mangatsu/server/pkg/db" 7 | "github.com/Mangatsu/server/pkg/library" 8 | "github.com/Mangatsu/server/pkg/metadata" 9 | "net/http" 10 | ) 11 | 12 | func scanLibraries(w http.ResponseWriter, r *http.Request) { 13 | access, _ := hasAccess(w, r, db.Admin) 14 | if !access { 15 | return 16 | } 17 | 18 | // fullScan := r.URL.Query().Get("full") 19 | go library.ScanArchives() 20 | 21 | w.Header().Set("Content-Type", "application/json;charset=UTF-8") 22 | fmt.Fprintf(w, `{ "Message": "started scanning for new archives." }`) 23 | } 24 | 25 | func returnProcessingStatus(w http.ResponseWriter, r *http.Request) { 26 | access, _ := hasAccess(w, r, db.Admin) 27 | if !access { 28 | return 29 | } 30 | 31 | status := cache.ProcessingStatusCache 32 | resultToJSON(w, status, r.URL.Path) 33 | } 34 | 35 | func generateThumbnails(w http.ResponseWriter, r *http.Request) { 36 | access, _ := hasAccess(w, r, db.Admin) 37 | if !access { 38 | return 39 | } 40 | 41 | pages := r.URL.Query().Get("pages") 42 | force := r.URL.Query().Get("force") 43 | go library.GenerateThumbnails(pages == "true", force == "true") 44 | 45 | w.Header().Set("Content-Type", "application/json;charset=UTF-8") 46 | fmt.Fprintf(w, `{ "Message": "started generateting thumbnails. Prioritizing covers." }`) 47 | } 48 | 49 | func findMetadata(w http.ResponseWriter, r *http.Request) { 50 | access, _ := hasAccess(w, r, db.Admin) 51 | if !access { 52 | return 53 | } 54 | 55 | title := r.URL.Query().Get("title") 56 | x := r.URL.Query().Get("x") 57 | ehdl := r.URL.Query().Get("ehdl") 58 | hath := r.URL.Query().Get("hath") 59 | fuzzy := r.URL.Query().Get("fuzzy") 60 | 61 | metaTypes := make(map[metadata.MetaType]bool) 62 | metaTypes[metadata.XMeta] = x == "true" 63 | metaTypes[metadata.EHDLMeta] = ehdl == "true" 64 | metaTypes[metadata.HathMeta] = hath == "true" 65 | metaTypes[metadata.FuzzyMatch] = fuzzy == "true" 66 | go metadata.ParseMetadata(metaTypes) 67 | 68 | if title == "true" { 69 | go metadata.ParseTitles(true, false) 70 | } 71 | 72 | w.Header().Set("Content-Type", "application/json;charset=UTF-8") 73 | if metaTypes[metadata.XMeta] || metaTypes[metadata.EHDLMeta] || metaTypes[metadata.HathMeta] || title == "true" { 74 | fmt.Fprintf(w, `{ "Message": "started parsing given sources" }`) 75 | return 76 | } 77 | 78 | errorHandler(w, http.StatusBadRequest, "no sources specified", r.URL.Path) 79 | } 80 | -------------------------------------------------------------------------------- /pkg/constants/language.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | // Languages is a map of languages that are supported by the application. 4 | // TODO: Add more, and convert to international language codes. 5 | var Languages = map[string]bool{ 6 | "afrikaans": true, 7 | "albanian": true, 8 | "arabic": true, 9 | "aramaic": true, 10 | "armenian": true, 11 | "bengali": true, 12 | "bosnian": true, 13 | "bulgarian": true, 14 | "burmese": true, 15 | "catalan": true, 16 | "cebuano": true, 17 | "chinese": true, 18 | "cree": true, 19 | "creole": true, 20 | "croatian": true, 21 | "czech": true, 22 | "danish": true, 23 | "dutch": true, 24 | "english": true, 25 | "esperanto": true, 26 | "estonian": true, 27 | "finnish": true, 28 | "french": true, 29 | "georgian": true, 30 | "german": true, 31 | "greek": true, 32 | "gujarati": true, 33 | "hebrew": true, 34 | "hindi": true, 35 | "hmong": true, 36 | "hungarian": true, 37 | "icelandic": true, 38 | "indonesian": true, 39 | "irish": true, 40 | "italian": true, 41 | "japanese": true, 42 | "javanese": true, 43 | "kannada": true, 44 | "kazakh": true, 45 | "khmer": true, 46 | "korean": true, 47 | "kurdish": true, 48 | "ladino": true, 49 | "lao": true, 50 | "latin": true, 51 | "latvian": true, 52 | "marathi": true, 53 | "mongolian": true, 54 | "ndebele": true, 55 | "nepali": true, 56 | "norwegian": true, 57 | "oromo": true, 58 | "papiamento": true, 59 | "pashto": true, 60 | "persian": true, 61 | "polish": true, 62 | "portuguese": true, 63 | "punjabi": true, 64 | "romanian": true, 65 | "russian": true, 66 | "sango": true, 67 | "sanskrit": true, 68 | "serbian": true, 69 | "shona": true, 70 | "slovak": true, 71 | "slovenian": true, 72 | "somali": true, 73 | "spanish": true, 74 | "swahili": true, 75 | "swedish": true, 76 | "tagalog": true, 77 | "tamil": true, 78 | "telugu": true, 79 | "thai": true, 80 | "tibetan": true, 81 | "tigrinya": true, 82 | "turkish": true, 83 | "ukrainian": true, 84 | "urdu": true, 85 | "vietnamese": true, 86 | "welsh": true, 87 | "yiddish": true, 88 | "zulu": true, 89 | } 90 | 91 | // LTRLanguages is a list of languages that are written from left to right. TODO: Add more. 92 | var LTRLanguages = []string{ 93 | "arabic", 94 | "hebrew", 95 | "japanese", 96 | "persian", 97 | "urdu", 98 | } 99 | -------------------------------------------------------------------------------- /pkg/api/jwt.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | 7 | "github.com/Mangatsu/server/internal/config" 8 | "github.com/Mangatsu/server/pkg/db" 9 | "github.com/Mangatsu/server/pkg/utils" 10 | "github.com/golang-jwt/jwt/v5" 11 | ) 12 | 13 | type CustomClaims struct { 14 | jwt.RegisteredClaims 15 | ID string 16 | Subject string 17 | Name string 18 | Roles *int32 19 | } 20 | 21 | // readJWT parses the JWT from an HTTP request's Cookie or Authorization header. 22 | func readJWT(r *http.Request) string { 23 | jwtCookie, err := r.Cookie("mtsu.jwt") // Mostly for web browsers 24 | jwtAuth := r.Header.Get("Authorization") // Others such as mobile apps 25 | 26 | token := "" 27 | if err == nil { 28 | token = jwtCookie.Value 29 | } else { 30 | token = jwtAuth 31 | } 32 | 33 | splitToken := strings.Fields(token) 34 | if len(splitToken) == 2 { 35 | return splitToken[1] 36 | } 37 | 38 | return "" 39 | } 40 | 41 | func newJWT(userUUID string, expiresIn *int64, sessionName *string, role *int32) (string, error) { 42 | if expiresIn != nil { 43 | *expiresIn = utils.Clamp(*expiresIn, 30, 60*60*24*365) 44 | } 45 | 46 | sessionID, err := db.NewSession(userUUID, expiresIn, sessionName) 47 | if err != nil { 48 | return "", err 49 | } 50 | 51 | claims := jwt.NewWithClaims(jwt.SigningMethodHS256, CustomClaims{ 52 | ID: sessionID, 53 | Subject: userUUID, 54 | Roles: role, 55 | }) 56 | 57 | token, err := claims.SignedString([]byte(config.Credentials.JWTSecret)) 58 | if err != nil { 59 | return "", err 60 | } 61 | 62 | return token, err 63 | } 64 | 65 | func verifyJWT(tokenString string, role db.Role) (bool, *string) { 66 | claims, ok, token, err := parseJWT(tokenString) 67 | 68 | claimedRole := 0 69 | if claims.Roles != nil { 70 | claimedRole = int(*claims.Roles) 71 | } 72 | 73 | if err == nil && ok && token.Valid && db.VerifySession(claims.ID, claims.Subject) && claimedRole >= int(role) { 74 | return true, &claims.Subject 75 | } 76 | 77 | return false, nil 78 | } 79 | 80 | func parseJWT(tokenString string) (CustomClaims, bool, *jwt.Token, error) { 81 | token, err := jwt.ParseWithClaims( 82 | tokenString, &CustomClaims{}, 83 | func(token *jwt.Token) (interface{}, error) { 84 | return []byte(config.Credentials.JWTSecret), nil 85 | }, 86 | jwt.WithValidMethods([]string{jwt.SigningMethodHS256.Name}), 87 | ) 88 | 89 | if err != nil { 90 | return CustomClaims{}, false, nil, err 91 | } 92 | 93 | claims, ok := token.Claims.(*CustomClaims) 94 | 95 | return *claims, ok, token, err 96 | } 97 | -------------------------------------------------------------------------------- /internal/config/path.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "io/fs" 6 | "os" 7 | "path" 8 | "path/filepath" 9 | "regexp" 10 | "strconv" 11 | "strings" 12 | 13 | "github.com/Mangatsu/server/pkg/log" 14 | ) 15 | 16 | type Library struct { 17 | ID int32 18 | Path string 19 | Layout string 20 | } 21 | 22 | var libraryOptionsR = regexp.MustCompile(`^(freeform|structured)(\d+)$`) 23 | 24 | func ParseBasePaths() []Library { 25 | basePaths := os.Getenv("MTSU_BASE_PATHS") 26 | if basePaths == "" { 27 | log.Z.Fatal("MTSU_BASE_PATHS is not set") 28 | } 29 | 30 | basePathsSlice := strings.Split(basePaths, ";;") 31 | var libraryPaths []Library 32 | 33 | for _, basePath := range basePathsSlice { 34 | layoutAndPath := strings.SplitN(basePath, ";", 2) 35 | if len(layoutAndPath) != 2 { 36 | log.Z.Fatal("MTSU_BASE_PATHS is not set correctly") 37 | } 38 | 39 | libraryOptionsMatch := libraryOptionsR.MatchString(layoutAndPath[0]) 40 | if !libraryOptionsMatch { 41 | log.Z.Fatal(layoutAndPath[0] + " is not a valid layout in BASE_PATHS. Valid: freeform1, structured2, freeform3 ...") 42 | } 43 | if layoutAndPath[1] == "" { 44 | log.Z.Fatal("Paths in MTSU_BASE_PATHS cannot be empty") 45 | } 46 | if _, err := os.Stat(layoutAndPath[1]); errors.Is(err, fs.ErrNotExist) { 47 | log.Z.Fatal("Path in MTSU_BASE_PATHS not found: " + layoutAndPath[1]) 48 | } 49 | 50 | libraryOptions := libraryOptionsR.FindStringSubmatch(layoutAndPath[0]) 51 | id, _ := strconv.ParseInt(libraryOptions[2], 10, 32) 52 | 53 | libraryPaths = append(libraryPaths, Library{ 54 | ID: int32(id), 55 | Path: layoutAndPath[1], 56 | Layout: libraryOptions[1], 57 | }) 58 | } 59 | 60 | return libraryPaths 61 | } 62 | 63 | func RelativePath(basePath string, fullPath string) string { 64 | return strings.Replace(filepath.ToSlash(fullPath), filepath.ToSlash(basePath)+"/", "", 1) 65 | } 66 | 67 | func BuildPath(base string, pathParts ...string) string { 68 | if len(pathParts) == 0 { 69 | return base 70 | } 71 | basePath := []string{base} 72 | pathSlice := append(basePath, pathParts...) 73 | return filepath.ToSlash(path.Join(pathSlice...)) 74 | } 75 | 76 | func BuildLibraryPath(libraryPath string, pathParts ...string) string { 77 | return BuildPath(libraryPath, pathParts...) 78 | } 79 | 80 | func BuildDataPath(pathParts ...string) string { 81 | return BuildPath(os.Getenv("MTSU_DATA_PATH"), pathParts...) 82 | } 83 | 84 | func BuildCachePath(pathParts ...string) string { 85 | return BuildPath(os.Getenv("MTSU_DATA_PATH"), append([]string{"cache"}, pathParts...)...) 86 | } 87 | -------------------------------------------------------------------------------- /pkg/types/sqlite/table/gallery_pref.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package table 9 | 10 | import ( 11 | "github.com/go-jet/jet/v2/sqlite" 12 | ) 13 | 14 | var GalleryPref = newGalleryPrefTable("", "gallery_pref", "") 15 | 16 | type galleryPrefTable struct { 17 | sqlite.Table 18 | 19 | //Columns 20 | UserUUID sqlite.ColumnString 21 | GalleryUUID sqlite.ColumnString 22 | Progress sqlite.ColumnInteger 23 | FavoriteGroup sqlite.ColumnString 24 | UpdatedAt sqlite.ColumnTimestamp 25 | 26 | AllColumns sqlite.ColumnList 27 | MutableColumns sqlite.ColumnList 28 | } 29 | 30 | type GalleryPrefTable struct { 31 | galleryPrefTable 32 | 33 | EXCLUDED galleryPrefTable 34 | } 35 | 36 | // AS creates new GalleryPrefTable with assigned alias 37 | func (a GalleryPrefTable) AS(alias string) *GalleryPrefTable { 38 | return newGalleryPrefTable(a.SchemaName(), a.TableName(), alias) 39 | } 40 | 41 | // Schema creates new GalleryPrefTable with assigned schema name 42 | func (a GalleryPrefTable) FromSchema(schemaName string) *GalleryPrefTable { 43 | return newGalleryPrefTable(schemaName, a.TableName(), a.Alias()) 44 | } 45 | 46 | func newGalleryPrefTable(schemaName, tableName, alias string) *GalleryPrefTable { 47 | return &GalleryPrefTable{ 48 | galleryPrefTable: newGalleryPrefTableImpl(schemaName, tableName, alias), 49 | EXCLUDED: newGalleryPrefTableImpl("", "excluded", ""), 50 | } 51 | } 52 | 53 | func newGalleryPrefTableImpl(schemaName, tableName, alias string) galleryPrefTable { 54 | var ( 55 | UserUUIDColumn = sqlite.StringColumn("user_uuid") 56 | GalleryUUIDColumn = sqlite.StringColumn("gallery_uuid") 57 | ProgressColumn = sqlite.IntegerColumn("progress") 58 | FavoriteGroupColumn = sqlite.StringColumn("favorite_group") 59 | UpdatedAtColumn = sqlite.TimestampColumn("updated_at") 60 | allColumns = sqlite.ColumnList{UserUUIDColumn, GalleryUUIDColumn, ProgressColumn, FavoriteGroupColumn, UpdatedAtColumn} 61 | mutableColumns = sqlite.ColumnList{ProgressColumn, FavoriteGroupColumn, UpdatedAtColumn} 62 | ) 63 | 64 | return galleryPrefTable{ 65 | Table: sqlite.NewTable(schemaName, tableName, alias, allColumns...), 66 | 67 | //Columns 68 | UserUUID: UserUUIDColumn, 69 | GalleryUUID: GalleryUUIDColumn, 70 | Progress: ProgressColumn, 71 | FavoriteGroup: FavoriteGroupColumn, 72 | UpdatedAt: UpdatedAtColumn, 73 | 74 | AllColumns: allColumns, 75 | MutableColumns: mutableColumns, 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ main ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ main ] 20 | schedule: 21 | - cron: '18 23 * * 0' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v2 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 52 | 53 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 54 | # If this step fails, then you should remove it and run the build manually (see below) 55 | #- name: Autobuild 56 | # uses: github/codeql-action/autobuild@v2 57 | 58 | # ℹ️ Command-line programs to run using the OS shell. 59 | # 📚 https://git.io/JvXDl 60 | 61 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 62 | # and modify them (or add more) to build your code if your project 63 | # uses a compiled language 64 | 65 | - run: | 66 | /usr/bin/env GOTOOLCHAIN=go1.22.1+auto go build github.com/Mangatsu/server/cmd/mangatsu-server 67 | 68 | - name: Perform CodeQL Analysis 69 | uses: github/codeql-action/analyze@v2 70 | -------------------------------------------------------------------------------- /pkg/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "crypto/sha1" 5 | "encoding/hex" 6 | "errors" 7 | "github.com/Mangatsu/server/pkg/log" 8 | "github.com/adrg/strutil" 9 | "github.com/adrg/strutil/metrics" 10 | "go.uber.org/zap" 11 | "io/fs" 12 | "os" 13 | "path/filepath" 14 | "time" 15 | ) 16 | 17 | // ReadJSON returns the given JSON file as bytes. 18 | func ReadJSON(jsonFile string) ([]byte, error) { 19 | jsonFileBytes, err := os.ReadFile(jsonFile) 20 | 21 | if err != nil { 22 | log.Z.Debug("failed to read JSON file", zap.String("err", err.Error())) 23 | return nil, err 24 | } 25 | 26 | return jsonFileBytes, nil 27 | } 28 | 29 | // Clamp clamps the given value to the given range. 30 | func Clamp(value, min, max int64) int64 { 31 | if value < min { 32 | return min 33 | } 34 | if value > max { 35 | return max 36 | } 37 | return value 38 | } 39 | 40 | // ClampU clamps the given unsigned value to the given range. 41 | func ClampU(value, min, max uint64) uint64 { 42 | if value < min { 43 | return min 44 | } 45 | if value > max { 46 | return max 47 | } 48 | return value 49 | } 50 | 51 | // PeriodicTask loops the given function in separate thread between the given interval. 52 | func PeriodicTask(d time.Duration, f func()) { 53 | go func() { 54 | for { 55 | f() 56 | time.Sleep(d) 57 | } 58 | }() 59 | } 60 | 61 | // PathExists checks if the given path exists. 62 | func PathExists(pathTo string) bool { 63 | _, err := os.Stat(pathTo) 64 | if errors.Is(err, fs.ErrNotExist) { 65 | return false 66 | } 67 | 68 | return err == nil 69 | } 70 | 71 | func FileSize(filePath string) (int64, error) { 72 | stat, err := os.Stat(filePath) 73 | if err != nil { 74 | log.Z.Error("could not get file size", zap.String("path", filePath), zap.String("err", err.Error())) 75 | return 0, err 76 | } 77 | 78 | return stat.Size(), nil 79 | } 80 | 81 | func DirSize(dirPath string) (int64, error) { 82 | var size int64 83 | err := filepath.Walk(dirPath, func(_ string, info os.FileInfo, err error) error { 84 | if err != nil { 85 | return err 86 | } 87 | if !info.IsDir() { 88 | size += info.Size() 89 | } 90 | return err 91 | }) 92 | return size, err 93 | } 94 | 95 | // Similarity calculates the similarity between two strings. 96 | func Similarity(a string, b string) float64 { 97 | sd := metrics.NewSorensenDice() 98 | sd.CaseSensitive = false 99 | sd.NgramSize = 4 100 | similarity := strutil.Similarity(a, b, sd) 101 | 102 | return similarity 103 | } 104 | 105 | func HashStringSHA1(s string) string { 106 | h := sha1.New() 107 | h.Write([]byte(s)) 108 | 109 | return hex.EncodeToString(h.Sum(nil)) 110 | } 111 | -------------------------------------------------------------------------------- /pkg/types/sqlite/table/user.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package table 9 | 10 | import ( 11 | "github.com/go-jet/jet/v2/sqlite" 12 | ) 13 | 14 | var User = newUserTable("", "user", "") 15 | 16 | type userTable struct { 17 | sqlite.Table 18 | 19 | //Columns 20 | UUID sqlite.ColumnString 21 | Username sqlite.ColumnString 22 | Password sqlite.ColumnString 23 | Salt sqlite.ColumnString 24 | Role sqlite.ColumnInteger 25 | BcryptPw sqlite.ColumnString 26 | CreatedAt sqlite.ColumnTimestamp 27 | UpdatedAt sqlite.ColumnTimestamp 28 | 29 | AllColumns sqlite.ColumnList 30 | MutableColumns sqlite.ColumnList 31 | } 32 | 33 | type UserTable struct { 34 | userTable 35 | 36 | EXCLUDED userTable 37 | } 38 | 39 | // AS creates new UserTable with assigned alias 40 | func (a UserTable) AS(alias string) *UserTable { 41 | return newUserTable(a.SchemaName(), a.TableName(), alias) 42 | } 43 | 44 | // Schema creates new UserTable with assigned schema name 45 | func (a UserTable) FromSchema(schemaName string) *UserTable { 46 | return newUserTable(schemaName, a.TableName(), a.Alias()) 47 | } 48 | 49 | func newUserTable(schemaName, tableName, alias string) *UserTable { 50 | return &UserTable{ 51 | userTable: newUserTableImpl(schemaName, tableName, alias), 52 | EXCLUDED: newUserTableImpl("", "excluded", ""), 53 | } 54 | } 55 | 56 | func newUserTableImpl(schemaName, tableName, alias string) userTable { 57 | var ( 58 | UUIDColumn = sqlite.StringColumn("uuid") 59 | UsernameColumn = sqlite.StringColumn("username") 60 | PasswordColumn = sqlite.StringColumn("password") 61 | SaltColumn = sqlite.StringColumn("salt") 62 | RoleColumn = sqlite.IntegerColumn("role") 63 | BcryptPwColumn = sqlite.StringColumn("bcrypt_pw") 64 | CreatedAtColumn = sqlite.TimestampColumn("created_at") 65 | UpdatedAtColumn = sqlite.TimestampColumn("updated_at") 66 | allColumns = sqlite.ColumnList{UUIDColumn, UsernameColumn, PasswordColumn, SaltColumn, RoleColumn, BcryptPwColumn, CreatedAtColumn, UpdatedAtColumn} 67 | mutableColumns = sqlite.ColumnList{UUIDColumn, UsernameColumn, PasswordColumn, SaltColumn, RoleColumn, BcryptPwColumn, CreatedAtColumn, UpdatedAtColumn} 68 | ) 69 | 70 | return userTable{ 71 | Table: sqlite.NewTable(schemaName, tableName, alias, allColumns...), 72 | 73 | //Columns 74 | UUID: UUIDColumn, 75 | Username: UsernameColumn, 76 | Password: PasswordColumn, 77 | Salt: SaltColumn, 78 | Role: RoleColumn, 79 | BcryptPw: BcryptPwColumn, 80 | CreatedAt: CreatedAtColumn, 81 | UpdatedAt: UpdatedAtColumn, 82 | 83 | AllColumns: allColumns, 84 | MutableColumns: mutableColumns, 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /pkg/cache/disk.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "errors" 5 | "io/fs" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | 10 | "github.com/Mangatsu/server/internal/config" 11 | "github.com/Mangatsu/server/pkg/log" 12 | "github.com/Mangatsu/server/pkg/utils" 13 | "github.com/facette/natsort" 14 | "go.uber.org/zap" 15 | ) 16 | 17 | // InitPhysicalCache initializes the physical cache directories. 18 | func InitPhysicalCache() { 19 | cachePath := config.BuildCachePath() 20 | if !utils.PathExists(cachePath) { 21 | err := os.Mkdir(cachePath, os.ModePerm) 22 | if err != nil { 23 | log.Z.Error("failed to create cache dir", 24 | zap.String("path", cachePath), 25 | zap.String("err", err.Error())) 26 | } 27 | } 28 | 29 | thumbnailsPath := config.BuildCachePath("thumbnails") 30 | if !utils.PathExists(thumbnailsPath) { 31 | err := os.Mkdir(thumbnailsPath, os.ModePerm) 32 | if err != nil { 33 | log.Z.Error("failed to create thumbnails cache dir", 34 | zap.String("path", thumbnailsPath), 35 | zap.String("err", err.Error())) 36 | } 37 | } 38 | } 39 | 40 | // readPhysicalCache reads the physical cache from the disk and returns the list of files and the number of files. 41 | func readPhysicalCache(dst string, uuid string) ([]string, int) { 42 | var files []string 43 | count := 0 44 | 45 | cacheWalk := func(s string, d fs.DirEntry, err error) error { 46 | if err != nil { 47 | log.Z.Error("failed to walk cache dir", 48 | zap.String("name", d.Name()), 49 | zap.String("err", err.Error())) 50 | return err 51 | } 52 | if d.IsDir() { 53 | return nil 54 | } 55 | 56 | // ReplaceAll ensures that the path is correct: cache/uuid/ 57 | files = append(files, strings.ReplaceAll(filepath.ToSlash(s), config.BuildCachePath(uuid)+"/", "")) 58 | count += 1 59 | return nil 60 | } 61 | 62 | err := filepath.WalkDir(dst, cacheWalk) 63 | if err != nil { 64 | log.Z.Error("failed to walk cache dir", 65 | zap.String("dst", dst), 66 | zap.String("err", err.Error())) 67 | return nil, 0 68 | } 69 | 70 | return files, count 71 | } 72 | 73 | // extractGallery extracts the gallery from the archive and returns the list of files and the number of files. 74 | func extractGallery(archivePath string, uuid string) ([]string, int) { 75 | dst := config.BuildCachePath(uuid) 76 | if _, err := os.Stat(dst); errors.Is(err, fs.ErrNotExist) { 77 | return utils.UniversalExtract(dst, archivePath) 78 | } 79 | 80 | files, count := readPhysicalCache(dst, uuid) 81 | if count == 0 { 82 | err := os.Remove(dst) 83 | if err != nil { 84 | log.Z.Debug("removing empty cache dir failed", 85 | zap.String("dst", dst), 86 | zap.String("err", err.Error())) 87 | return nil, 0 88 | } 89 | 90 | return utils.UniversalExtract(dst, archivePath) 91 | } 92 | natsort.Sort(files) 93 | 94 | return files, count 95 | } 96 | -------------------------------------------------------------------------------- /pkg/utils/archive.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "errors" 5 | "github.com/Mangatsu/server/pkg/constants" 6 | "github.com/Mangatsu/server/pkg/log" 7 | "github.com/mholt/archiver/v4" 8 | "go.uber.org/zap" 9 | "io" 10 | "io/fs" 11 | "os" 12 | "path/filepath" 13 | ) 14 | 15 | // UniversalExtract extracts media files from zip, cbz, rar, cbr, tar (all its variants) archives. 16 | // Plain directories without compression are also supported. For PDF files, use ExtractPDF. 17 | func UniversalExtract(dst string, archivePath string) ([]string, int) { 18 | fsys, err := archiver.FileSystem(nil, archivePath) 19 | if err != nil { 20 | log.Z.Error("failed to open an archive", 21 | zap.String("path", archivePath), 22 | zap.String("err", err.Error())) 23 | return nil, 0 24 | } 25 | 26 | if err = os.Mkdir(dst, os.ModePerm); err != nil { 27 | if !errors.Is(err, fs.ErrNotExist) { 28 | log.Z.Error("failed to create a dir for gallery", 29 | zap.String("path", dst), 30 | zap.String("err", err.Error())) 31 | return nil, 0 32 | 33 | } 34 | } 35 | 36 | var files []string 37 | count := 0 38 | 39 | err = fs.WalkDir(fsys, ".", func(s string, d fs.DirEntry, err error) error { 40 | if err != nil { 41 | return err 42 | } 43 | dstPath := filepath.Join(dst, s) 44 | 45 | if s == "." || s == ".." { 46 | return nil 47 | } 48 | 49 | if d.IsDir() { 50 | err = os.Mkdir(dstPath, os.ModePerm) 51 | if errors.Is(err, fs.ErrNotExist) { 52 | return nil 53 | } 54 | return err 55 | } 56 | 57 | if !constants.ImageExtensions.MatchString(d.Name()) { 58 | return nil 59 | } 60 | 61 | //if !strings.HasPrefix(dstPath, filepath.Clean(dst)+string(os.PathSeparator)) { 62 | // log.Error("Invalid file path: ", dstPath) 63 | // return nil 64 | //} 65 | 66 | fileInArchive, err := fsys.Open(s) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | if _, err := io.Copy(dstFile, fileInArchive); err != nil { 77 | log.Z.Error("failed to copy file", 78 | zap.String("dstFile", dstFile.Name()), 79 | zap.String("err", err.Error())) 80 | } 81 | 82 | if err = dstFile.Close(); err != nil { 83 | return err 84 | } 85 | if err = fileInArchive.Close(); err != nil { 86 | return err 87 | } 88 | 89 | files = append(files, s) 90 | count += 1 91 | return nil 92 | }) 93 | if err != nil { 94 | log.Z.Debug("failed to walk dir when copying archive", zap.String("err", err.Error())) 95 | return nil, 0 96 | } 97 | 98 | return files, count 99 | } 100 | 101 | func ExtractPDF() { 102 | // TODO: Add support for PDF files, Probably with https://github.com/gen2brain/go-fitz 103 | } 104 | -------------------------------------------------------------------------------- /pkg/utils/password.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "crypto/rand" 5 | "crypto/subtle" 6 | "github.com/Mangatsu/server/pkg/log" 7 | "go.uber.org/zap" 8 | "golang.org/x/crypto/argon2" 9 | ) 10 | 11 | // Argon2idHash is a wrapper for the argon2id hashing algorithm. 12 | // https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#password-hashing-algorithms 13 | type Argon2idHash struct { 14 | // time represents the number of passed over the specified memory 15 | time uint32 16 | // CPU memory cost 17 | memory uint32 18 | // threads for parallelism 19 | threads uint8 20 | // keyLen of the generate hash key 21 | keyLen uint32 22 | // saltLen the length of the salt used 23 | saltLen uint32 24 | } 25 | 26 | // HashSalt is a wrapper for the hash and salt values. 27 | type HashSalt struct { 28 | Hash []byte 29 | Salt []byte 30 | } 31 | 32 | // NewArgon2idHash constructor function for Argon2idHash. 33 | func NewArgon2idHash(time, saltLen, memory uint32, threads uint8, keyLen uint32) *Argon2idHash { 34 | return &Argon2idHash{ 35 | // time represents the number of passed over the specified memory 36 | time: time, 37 | // salt length 38 | saltLen: saltLen, 39 | // CPU memory cost (in KiB) 40 | memory: memory, 41 | // threads for parallelism 42 | threads: threads, 43 | // hash key length 44 | keyLen: keyLen, 45 | } 46 | } 47 | 48 | // DefaultArgon2idHash constructor function for Argon2idHash. 49 | // https://tobtu.com/minimum-password-settings/ 50 | func DefaultArgon2idHash() *Argon2idHash { 51 | return &Argon2idHash{ 52 | time: 2, 53 | saltLen: 16, 54 | memory: 19456, 55 | threads: 2, 56 | keyLen: 32, 57 | } 58 | } 59 | 60 | // randomSecret generates a random secret of a given length. Used for salt generation. 61 | func randomSecret(length uint32) ([]byte, error) { 62 | secret := make([]byte, length) 63 | 64 | _, err := rand.Read(secret) 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | return secret, nil 70 | } 71 | 72 | // GenerateHash using the password and provided salt. 73 | // If no salt value was provided, fallbacks to a random value of a given length. 74 | func (a *Argon2idHash) GenerateHash(password, salt []byte) (*HashSalt, error) { 75 | if len(salt) == 0 { 76 | var err error 77 | if salt, err = randomSecret(a.saltLen); err != nil { 78 | return nil, err 79 | } 80 | } 81 | 82 | hash := argon2.IDKey(password, salt, a.time, a.memory, a.threads, a.keyLen) 83 | 84 | return &HashSalt{Hash: hash, Salt: salt}, nil 85 | } 86 | 87 | // Compare generated hash with stored hash. 88 | func (a *Argon2idHash) Compare(hash, salt, password []byte) bool { 89 | hashSalt, err := a.GenerateHash(password, salt) 90 | if err != nil { 91 | log.Z.Debug("failed to generate hash", zap.Error(err)) 92 | return false 93 | } 94 | 95 | // ConstantTimeCompare used to prevent timing attacks. 96 | return subtle.ConstantTimeCompare(hash, hashSalt.Hash) == 1 97 | } 98 | -------------------------------------------------------------------------------- /pkg/db/migrations/20220227131049_modify_gallery_columns.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | SELECT 'up SQL query'; 4 | CREATE TABLE gallery2 5 | ( 6 | uuid text UNIQUE NOT NULL, 7 | library_id integer NOT NULL, 8 | archive_path text UNIQUE NOT NULL, 9 | title text NOT NULL, 10 | title_native text, 11 | title_translated text, 12 | category text, 13 | series text, 14 | released text, 15 | language text, 16 | translated boolean, 17 | nsfw boolean NOT NULL DEFAULT false, 18 | hidden boolean NOT NULL DEFAULT false, 19 | image_count int, 20 | archive_size int, 21 | archive_hash text, 22 | thumbnail text, 23 | created_at datetime NOT NULL, 24 | updated_at datetime NOT NULL, 25 | PRIMARY KEY (uuid), 26 | FOREIGN KEY (library_id) 27 | REFERENCES library (id) 28 | ); 29 | INSERT INTO gallery2 30 | SELECT uuid, 31 | library_id, 32 | archive_path, 33 | title, 34 | title_native, 35 | title_short AS title_translated, 36 | category, 37 | series, 38 | released, language, translated, nsfw, hidden, image_count, archive_size, archive_hash, thumbnail, created_at, updated_at 39 | FROM gallery; 40 | DROP TABLE gallery; 41 | ALTER TABLE gallery2 RENAME TO gallery; 42 | -- +goose StatementEnd 43 | 44 | -- +goose Down 45 | -- +goose StatementBegin 46 | SELECT 'down SQL query'; 47 | CREATE TABLE gallery2 48 | ( 49 | uuid text UNIQUE NOT NULL, 50 | library_id integer NOT NULL, 51 | archive_path text UNIQUE NOT NULL, 52 | title text NOT NULL, 53 | title_native text, 54 | title_short text, 55 | released text, 56 | circle text, 57 | artists text, 58 | series text, 59 | category text, 60 | language text, 61 | translated boolean, 62 | image_count int, 63 | archive_size int, 64 | archive_hash text, 65 | thumbnail text, 66 | nsfw boolean NOT NULL DEFAULT false, 67 | hidden boolean NOT NULL DEFAULT false, 68 | created_at datetime NOT NULL, 69 | updated_at datetime NOT NULL, 70 | PRIMARY KEY (uuid), 71 | FOREIGN KEY (library_id) 72 | REFERENCES library (id) 73 | ); 74 | INSERT INTO gallery2 75 | SELECT uuid, 76 | library_id, 77 | archive_path, 78 | title, 79 | title_native, 80 | title_translated AS title_short, 81 | released, 82 | null AS circle, 83 | null AS artists, 84 | series, 85 | category, language, translated, image_count, archive_size, archive_hash, thumbnail, nsfw, hidden, created_at, updated_at 86 | FROM gallery; 87 | DROP TABLE gallery; 88 | ALTER TABLE gallery2 RENAME TO gallery; 89 | -- +goose StatementEnd 90 | -------------------------------------------------------------------------------- /pkg/db/library.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "github.com/Mangatsu/server/internal/config" 5 | "github.com/Mangatsu/server/pkg/log" 6 | "github.com/Mangatsu/server/pkg/types/sqlite/model" 7 | . "github.com/Mangatsu/server/pkg/types/sqlite/table" 8 | . "github.com/go-jet/jet/v2/sqlite" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | type CombinedLibrary struct { 13 | model.Library 14 | Galleries []model.Gallery 15 | } 16 | 17 | func StorePaths(givenLibraries []config.Library) error { 18 | for _, library := range givenLibraries { 19 | libraries, err := getLibrary(library.ID, "") 20 | if err != nil { 21 | log.Z.Debug("failed to get libraries when storing paths", 22 | zap.Int32("id", library.ID), 23 | zap.String("err", err.Error())) 24 | continue 25 | } 26 | 27 | if len(libraries) == 0 { 28 | if err := newLibrary(library.ID, library.Path, library.Layout); err != nil { 29 | return err 30 | } 31 | continue 32 | } 33 | 34 | if libraries[0].Path != library.Path || libraries[0].Layout != library.Layout { 35 | if err := updateLibrary(library.ID, library.Path, library.Layout); err != nil { 36 | return err 37 | } 38 | } 39 | } 40 | 41 | return nil 42 | } 43 | 44 | func GetOnlyLibraries() ([]model.Library, error) { 45 | stmt := SELECT(Library.AllColumns).FROM(Library.Table) 46 | var libraries []model.Library 47 | 48 | err := stmt.Query(db(), &libraries) 49 | return libraries, err 50 | } 51 | 52 | func GetLibraries() ([]CombinedLibrary, error) { 53 | stmt := SELECT(Library.AllColumns, Gallery.AllColumns). 54 | FROM(Library.LEFT_JOIN(Gallery, Gallery.LibraryID.EQ(Library.ID))) 55 | var libraries []CombinedLibrary 56 | 57 | err := stmt.Query(db(), &libraries) 58 | return libraries, err 59 | } 60 | 61 | // getLibrary returns the library from the database based on the ID or path. 62 | func getLibrary(id int32, path string) ([]model.Library, error) { 63 | stmt := SELECT( 64 | Library.AllColumns, 65 | ).FROM( 66 | Library.Table, 67 | ) 68 | 69 | if path == "" { 70 | stmt = stmt.WHERE(Library.ID.EQ(Int32(id))) 71 | } else { 72 | stmt = stmt.WHERE(Library.ID.EQ(Int32(id)).OR(Library.Path.EQ(String(path)))) 73 | } 74 | 75 | var libraries []model.Library 76 | err := stmt.Query(db(), &libraries) 77 | return libraries, err 78 | } 79 | 80 | // newLibrary creates a new library to the database. 81 | func newLibrary(id int32, path string, layout string) error { 82 | stmt := Library.INSERT(Library.ID, Library.Path, Library.Layout).VALUES(id, path, layout). 83 | ON_CONFLICT(Library.ID). 84 | DO_UPDATE(SET(Library.Path.SET(String(path)), Library.Layout.SET(String(layout)))) 85 | 86 | _, err := stmt.Exec(db()) 87 | if err != nil { 88 | log.Z.Error("failed to create a new library", 89 | zap.Int32("id", id), 90 | zap.String("path", path), 91 | zap.String("err", err.Error())) 92 | } 93 | 94 | return err 95 | } 96 | 97 | // updateLibrary updates the library in the database. 98 | func updateLibrary(id int32, path string, layout string) error { 99 | stmt := Library.UPDATE(Library.Path, Library.Layout).SET(path, layout).WHERE(Library.ID.EQ(Int32(id))) 100 | _, err := stmt.Exec(db()) 101 | return err 102 | } 103 | -------------------------------------------------------------------------------- /pkg/types/sqlite/table/reference.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package table 9 | 10 | import ( 11 | "github.com/go-jet/jet/v2/sqlite" 12 | ) 13 | 14 | var Reference = newReferenceTable("", "reference", "") 15 | 16 | type referenceTable struct { 17 | sqlite.Table 18 | 19 | //Columns 20 | GalleryUUID sqlite.ColumnString 21 | MetaInternal sqlite.ColumnBool 22 | MetaPath sqlite.ColumnString 23 | MetaMatch sqlite.ColumnInteger 24 | Urls sqlite.ColumnString 25 | ExhGid sqlite.ColumnInteger 26 | ExhToken sqlite.ColumnString 27 | AnilistID sqlite.ColumnInteger 28 | MetaTitleHash sqlite.ColumnString 29 | 30 | AllColumns sqlite.ColumnList 31 | MutableColumns sqlite.ColumnList 32 | } 33 | 34 | type ReferenceTable struct { 35 | referenceTable 36 | 37 | EXCLUDED referenceTable 38 | } 39 | 40 | // AS creates new ReferenceTable with assigned alias 41 | func (a ReferenceTable) AS(alias string) *ReferenceTable { 42 | return newReferenceTable(a.SchemaName(), a.TableName(), alias) 43 | } 44 | 45 | // Schema creates new ReferenceTable with assigned schema name 46 | func (a ReferenceTable) FromSchema(schemaName string) *ReferenceTable { 47 | return newReferenceTable(schemaName, a.TableName(), a.Alias()) 48 | } 49 | 50 | func newReferenceTable(schemaName, tableName, alias string) *ReferenceTable { 51 | return &ReferenceTable{ 52 | referenceTable: newReferenceTableImpl(schemaName, tableName, alias), 53 | EXCLUDED: newReferenceTableImpl("", "excluded", ""), 54 | } 55 | } 56 | 57 | func newReferenceTableImpl(schemaName, tableName, alias string) referenceTable { 58 | var ( 59 | GalleryUUIDColumn = sqlite.StringColumn("gallery_uuid") 60 | MetaInternalColumn = sqlite.BoolColumn("meta_internal") 61 | MetaPathColumn = sqlite.StringColumn("meta_path") 62 | MetaMatchColumn = sqlite.IntegerColumn("meta_match") 63 | UrlsColumn = sqlite.StringColumn("urls") 64 | ExhGidColumn = sqlite.IntegerColumn("exh_gid") 65 | ExhTokenColumn = sqlite.StringColumn("exh_token") 66 | AnilistIDColumn = sqlite.IntegerColumn("anilist_id") 67 | MetaTitleHashColumn = sqlite.StringColumn("meta_title_hash") 68 | allColumns = sqlite.ColumnList{GalleryUUIDColumn, MetaInternalColumn, MetaPathColumn, MetaMatchColumn, UrlsColumn, ExhGidColumn, ExhTokenColumn, AnilistIDColumn, MetaTitleHashColumn} 69 | mutableColumns = sqlite.ColumnList{MetaInternalColumn, MetaPathColumn, MetaMatchColumn, UrlsColumn, ExhGidColumn, ExhTokenColumn, AnilistIDColumn, MetaTitleHashColumn} 70 | ) 71 | 72 | return referenceTable{ 73 | Table: sqlite.NewTable(schemaName, tableName, alias, allColumns...), 74 | 75 | //Columns 76 | GalleryUUID: GalleryUUIDColumn, 77 | MetaInternal: MetaInternalColumn, 78 | MetaPath: MetaPathColumn, 79 | MetaMatch: MetaMatchColumn, 80 | Urls: UrlsColumn, 81 | ExhGid: ExhGidColumn, 82 | ExhToken: ExhTokenColumn, 83 | AnilistID: AnilistIDColumn, 84 | MetaTitleHash: MetaTitleHashColumn, 85 | 86 | AllColumns: allColumns, 87 | MutableColumns: mutableColumns, 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /pkg/metadata/ehdl.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "regexp" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/Mangatsu/server/pkg/log" 11 | "github.com/Mangatsu/server/pkg/types/sqlite/model" 12 | ) 13 | 14 | var exhURLRegex = regexp.MustCompile(`https://\w+\.\w+/g/(\d+)/[a-z0-9]+`) 15 | var lengthRegex = regexp.MustCompile(`Length:\s*(\d+)`) 16 | var sizeRegex = regexp.MustCompile(`File Size:\s*(\d+(?:\.\d+)?)`) 17 | 18 | // ParseEHDL parses given text file. Input file is expected to be in the H@H (Hath) format (galleryinfo.txt). 19 | // Input file is expected to be in the E-Hentai-Downloader format (info.txt). 20 | func ParseEHDL(metaPath string, metaData []byte, internal bool) (model.Gallery, []model.Tag, model.Reference, error) { 21 | gallery := model.Gallery{} 22 | reference := model.Reference{ 23 | MetaPath: &metaPath, 24 | MetaInternal: internal, 25 | Urls: nil, 26 | } 27 | var tags []model.Tag 28 | 29 | buffer := bytes.NewBuffer(metaData) 30 | scanner := bufio.NewScanner(buffer) 31 | lineNumber := -1 32 | 33 | for scanner.Scan() { 34 | line := scanner.Text() 35 | lineNumber++ 36 | if len(line) == 0 { 37 | continue 38 | } 39 | 40 | switch lineNumber { 41 | case 0: 42 | gallery.Title = strings.TrimSpace(line) 43 | continue 44 | case 1: 45 | titleNative := strings.TrimSpace(line) 46 | gallery.TitleNative = &titleNative 47 | continue 48 | case 2: 49 | // https://example.org/g/999999/f2f2f2f2f2/ 50 | capture := exhURLRegex.FindStringSubmatch(line) 51 | if len(capture) < 3 { 52 | continue 53 | } 54 | exhGid, err := strconv.ParseInt(capture[2], 10, 32) 55 | if err != nil { 56 | log.S.Debug("failed to parse exhGid", "err", err.Error(), "exhGid", exhGid) 57 | continue 58 | } 59 | exhGidInt32 := int32(exhGid) 60 | 61 | reference.ExhGid = &exhGidInt32 62 | reference.ExhToken = &capture[1] 63 | continue 64 | } 65 | 66 | if strings.HasPrefix(line, "Category:") { 67 | category := strings.ToLower(strings.TrimSpace(strings.TrimPrefix(line, "Category:"))) 68 | gallery.Category = &category 69 | continue 70 | } 71 | 72 | if strings.HasPrefix(line, "Language:") { 73 | language := strings.TrimSpace(strings.TrimPrefix(line, "Language:")) 74 | gallery.Language = &language 75 | continue 76 | } 77 | 78 | if strings.HasPrefix(line, "Length:") { 79 | capture := lengthRegex.FindStringSubmatch(line) 80 | if len(capture) < 2 { 81 | continue 82 | } 83 | length, err := strconv.ParseInt(capture[1], 10, 32) 84 | if err != nil { 85 | log.S.Debug("failed to parse length", "err", err.Error(), "length", length) 86 | continue 87 | } 88 | lengthInt32 := int32(length) 89 | gallery.ImageCount = &lengthInt32 90 | continue 91 | } 92 | 93 | if strings.HasPrefix(line, "File Size:") { 94 | capture := sizeRegex.FindStringSubmatch(line) 95 | if len(capture) < 2 { 96 | continue 97 | } 98 | size, err := strconv.ParseFloat(capture[1], 32) 99 | if err != nil { 100 | log.S.Debug("failed to parse size", "err", err.Error(), "size", size) 101 | continue 102 | } 103 | sizeInt32 := int32(size * 1000 * 1000) 104 | gallery.ArchiveSize = &sizeInt32 105 | continue 106 | } 107 | 108 | if strings.HasPrefix(line, "> ") { 109 | tagsList := strings.Split(strings.TrimSpace(strings.TrimPrefix(line, "> ")), ":") 110 | if len(tagsList) != 2 { 111 | names := strings.Split(tagsList[1], ",") 112 | for _, name := range names { 113 | tags = append(tags, model.Tag{ 114 | Namespace: tagsList[0], 115 | Name: strings.TrimSpace(name), 116 | }) 117 | } 118 | } 119 | } 120 | } 121 | 122 | return gallery, tags, reference, nil 123 | } 124 | -------------------------------------------------------------------------------- /example.env: -------------------------------------------------------------------------------- 1 | # App environment: production or development 2 | MTSU_ENV=production 3 | # Log level: debug, info, warn, error 4 | MTSU_LOG_LEVEL=info 5 | 6 | # Credentials for the initial admin user. Recommended to change. 7 | MTSU_INITIAL_ADMIN_NAME=admin 8 | MTSU_INITIAL_ADMIN_PW=admin321 9 | 10 | # Hostname and port for the server. Use mtsuserver as the hostname if using Docker Compose. 11 | MTSU_HOSTNAME=localhost 12 | MTSU_PORT=5050 13 | 14 | # Domain for the server. Used in cookies. 15 | # For example, if the address for the server is "api.example.org", and for the frontend "read.example.org", 16 | # the value here should be "example.org" for the cookies to work properly between subdomains. 17 | # https://developer.mozilla.org/en-US/docs/Web/HTTP/Cookies#define_where_cookies_are_sent 18 | MTSU_DOMAIN=localhost 19 | 20 | # When true, the server only allows authenticated connections from the MTSU_DOMAIN and its subdomains (eg .*.example.org). 21 | # When false, connections from every origin is allowed. 22 | # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin 23 | MTSU_STRICT_ACAO=false 24 | 25 | # When true, Mangatsu can be accessed only through HTTPS or localhost domains. 26 | MTSU_SECURE=true 27 | 28 | # Paths to the archive directories. Relative or absolute paths are accepted. 29 | # First specify the type of the directory and a numerical ID (e.g. freeform1 or structured2) and then the path separated by a semicolon: `;`. 30 | # Multiple paths can be separated by a double-semicolon: `;;`. 31 | # Format: `;;;;`... 32 | # If using Docker Compose, make sure that the paths match the containerpaths in the volumes section. 33 | MTSU_BASE_PATHS=freeform1;/home/user/doujinshi;;structured2;/home/user/manga 34 | 35 | # Location of the data dir which includes the SQLite db and the cache for gallery images and thumbnails. Relative or absolute paths are accepted. 36 | # Doesn't need changing if using Docker Compose. 37 | MTSU_DATA_PATH=../data 38 | 39 | # Database 40 | MTSU_DB_NAME=mangatsu # The filename (without extension) of the SQLite. 41 | MTSU_DB_MIGRATIONS=true # For development. Keep as true. 42 | 43 | # Set true to disable the internal cache server (serves media files and thumbnails). Useful if one wants to use the web server such as NGINX to serve the files. 44 | MTSU_DISABLE_CACHE_SERVER=false 45 | 46 | # Cache time to live (for example 336h (2 weeks), 8h30m). If a gallery is not viewed for this time, it will be purged from the cache. 47 | MTSU_CACHE_TTL=336h 48 | 49 | # public: anyone can access the collection and its galleries. 50 | # restricted: users need a global passphrase to access collection and its galleries. 51 | # private: only logged-in users can access the collection and its galleries. 52 | # In all modes, user accounts are supported and have more privileges than anonymous users (e.g. favorite galleries). 53 | MTSU_VISIBILITY=public 54 | 55 | # Passphrase to access the collection and its galleries. 56 | # Only used when VISIBILITY is set to restricted. 57 | MTSU_RESTRICTED_PASSPHRASE=secretpassword 58 | 59 | # Whether to allow user registrations. If set to false, only admins can create new users. 60 | # Currently, only affects the API path /register. Has no effect in the frontend. 61 | MTSU_REGISTRATIONS=false 62 | 63 | # Secret to sign JWTs for login sessions in the backend. Recommended to change. 64 | MTSU_JWT_SECRET=9Wag7sMvKl3aF6K5lwIg6TI42ia2f6BstZAVrdJIq8Mp38lnl7UzQMC1qjKyZCBzHFGbbqsA0gKcHqDuyXQAhWoJ0lcx4K5q 65 | 66 | # Thumbnail image format: webp 67 | # AVIF support is planned. AVIF is said to take 20% longer to encode, but it compresses to 20% smaller size compared to WebP. 68 | MTSU_THUMBNAIL_FORMAT=webp 69 | 70 | # Similarity threshold for the fuzzy match for gallery and metadata filenames. 71 | # The higher the value, the more similar the results has to be to match. 0.1 - 1.0. 72 | MTSU_FUZZY_SEARCH_SIMILARITY=0.7 73 | 74 | # Set to false to use right-to-left (RTL) default for galleries. Otherwise, defaults to left-to-right (like Japanese manga). 75 | MTSU_LTR=true 76 | -------------------------------------------------------------------------------- /pkg/cache/processing_status.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type processingError struct { 4 | UUIDOrPath string 5 | Error string 6 | Details map[string]string 7 | } 8 | 9 | type scanResult struct { 10 | Running bool 11 | FoundGalleries []string 12 | SkippedGalleries []string 13 | Errors []processingError 14 | } 15 | 16 | type thumbnailResult struct { 17 | Running bool 18 | TotalCovers int 19 | TotalPages int 20 | GeneratedCovers int 21 | GeneratedPages int 22 | Errors []processingError 23 | } 24 | 25 | type metadataResult struct { 26 | // TODO: more information like progress and which sources are being used 27 | Running bool 28 | Errors []processingError 29 | } 30 | 31 | type ProcessingStatus struct { 32 | Scan scanResult 33 | Thumbnails thumbnailResult 34 | Metadata metadataResult 35 | } 36 | 37 | var ProcessingStatusCache *ProcessingStatus 38 | 39 | func InitProcessingStatusCache() { 40 | ProcessingStatusCache = &ProcessingStatus{ 41 | Scan: scanResult{ 42 | Running: false, 43 | FoundGalleries: make([]string, 0), 44 | SkippedGalleries: make([]string, 0), 45 | Errors: make([]processingError, 0), 46 | }, 47 | Thumbnails: thumbnailResult{ 48 | Running: false, 49 | GeneratedCovers: 0, 50 | GeneratedPages: 0, 51 | Errors: make([]processingError, 0), 52 | }, 53 | Metadata: metadataResult{ 54 | Running: false, 55 | Errors: make([]processingError, 0), 56 | }, 57 | } 58 | } 59 | 60 | func (s *ProcessingStatus) SetScanRunning(running bool) { 61 | s.Scan.Running = running 62 | } 63 | 64 | func (s *ProcessingStatus) AddScanFoundGallery(galleryUUID string) { 65 | s.Scan.FoundGalleries = append(s.Scan.FoundGalleries, galleryUUID) 66 | } 67 | 68 | func (s *ProcessingStatus) AddScanSkippedGallery(galleryUUID string) { 69 | s.Scan.SkippedGalleries = append(s.Scan.SkippedGalleries, galleryUUID) 70 | } 71 | 72 | func (s *ProcessingStatus) AddScanError(uuidOrPath string, err string, details map[string]string) { 73 | s.Scan.Errors = append(s.Scan.Errors, processingError{ 74 | UUIDOrPath: uuidOrPath, 75 | Error: err, 76 | Details: details, 77 | }) 78 | } 79 | 80 | func (s *ProcessingStatus) SetThumbnailsRunning(running bool) { 81 | s.Thumbnails.Running = running 82 | } 83 | 84 | func (s *ProcessingStatus) SetTotalCoversAndPages(coverCount int, pageCount int) { 85 | ProcessingStatusCache.Thumbnails.TotalCovers = coverCount 86 | ProcessingStatusCache.Thumbnails.TotalPages = pageCount 87 | } 88 | 89 | func (s *ProcessingStatus) AddThumbnailGeneratedCover() { 90 | s.Thumbnails.GeneratedCovers++ 91 | } 92 | 93 | func (s *ProcessingStatus) AddThumbnailGeneratedPage() { 94 | s.Thumbnails.GeneratedPages++ 95 | } 96 | 97 | func (s *ProcessingStatus) AddThumbnailError(uuidOrPath string, err string, details map[string]string) { 98 | s.Thumbnails.Errors = append(s.Thumbnails.Errors, processingError{ 99 | UUIDOrPath: uuidOrPath, 100 | Error: err, 101 | Details: details, 102 | }) 103 | } 104 | 105 | func (s *ProcessingStatus) SetMetadataRunning(running bool) { 106 | s.Metadata.Running = running 107 | 108 | } 109 | 110 | func (s *ProcessingStatus) AddMetadataError(uuidOrPath string, err string, details map[string]string) { 111 | s.Metadata.Errors = append(s.Metadata.Errors, processingError{ 112 | UUIDOrPath: uuidOrPath, 113 | Error: err, 114 | Details: details, 115 | }) 116 | } 117 | 118 | func (s *ProcessingStatus) Reset() { 119 | s.Scan = scanResult{ 120 | Running: false, 121 | FoundGalleries: make([]string, 0), 122 | SkippedGalleries: make([]string, 0), 123 | Errors: make([]processingError, 0), 124 | } 125 | s.Thumbnails = thumbnailResult{ 126 | Running: false, 127 | TotalCovers: 0, 128 | TotalPages: 0, 129 | GeneratedCovers: 0, 130 | GeneratedPages: 0, 131 | Errors: make([]processingError, 0), 132 | } 133 | s.Metadata = metadataResult{ 134 | Running: false, 135 | Errors: make([]processingError, 0), 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /pkg/utils/image.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "github.com/Mangatsu/server/internal/config" 8 | "github.com/chai2010/webp" 9 | "github.com/disintegration/imaging" 10 | "image" 11 | "image/draw" 12 | ) 13 | 14 | // TODO: test how long it takes to generate webp thumbnails compared to jpg + size differences 15 | //newImage, _ := os.Create("../cache/thumbnails/" + galleryUUID + "/" + name) 16 | //defer func(newImage *os.File) { 17 | // err := newImage.Close() 18 | // if err != nil { 19 | // log.Error("Closing thumbnail: ", err) 20 | // } 21 | //}(newImage) 22 | //err = png.Encode(newImage, dstImage) 23 | //if err != nil { 24 | // log.Error("Writing thumbnail: ", err) 25 | // return 26 | //} 27 | 28 | const DEFAULT_ASPECT_RATIO = 1.5 29 | const DEFAULT_THUMB_COVER_WIDTH = 512 30 | const DEFAULT_THUMB_PAGE_WIDTH = 256 31 | const DEFAULT_COVER_WIDTH_SHIFT = 0.025 32 | 33 | // EncodeImage encodes the given image to the format specified in the config. 34 | func EncodeImage(dstImage *image.NRGBA) (*bytes.Buffer, error) { 35 | var buf bytes.Buffer 36 | var err error 37 | 38 | switch config.Options.GalleryOptions.ThumbnailFormat { 39 | case config.WebP: 40 | err = webp.Encode(&buf, dstImage, &webp.Options{Lossless: false, Quality: 75}) 41 | case config.AVIF: 42 | return nil, errors.New("avif not supported yet") 43 | default: 44 | return nil, errors.New("unknown image format") 45 | } 46 | 47 | return &buf, err 48 | } 49 | 50 | // TransformImage resizes the given image to the specified width and returns it. 51 | // If cover is true, the image will be tried to be cropped to the specified width as some covers include the back cover as well. 52 | func TransformImage(srcImage *image.NRGBA, width int, cropLandscape bool, ltr bool) *image.NRGBA { 53 | var dstImage *image.NRGBA 54 | 55 | // For covers. If the image is wider than it is tall, crops the image to the specified width 56 | if cropLandscape { 57 | 58 | shift := int(float64(srcImage.Bounds().Dx()) * DEFAULT_COVER_WIDTH_SHIFT) 59 | width := int(float64(srcImage.Bounds().Dy()) / DEFAULT_ASPECT_RATIO) 60 | if srcImage.Bounds().Dx() < width { 61 | width = srcImage.Bounds().Dx() 62 | } 63 | 64 | var startX int 65 | var endX int 66 | 67 | if ltr { 68 | endX = srcImage.Bounds().Dx() / 2 69 | startX = endX - width 70 | 71 | startX -= shift 72 | endX -= shift 73 | } else { 74 | startX = srcImage.Bounds().Dx() / 2 75 | endX = startX + width 76 | 77 | startX += shift 78 | endX += shift 79 | } 80 | 81 | // Ensures the cropping window is within the image bounds 82 | if startX < 0 { 83 | startX = 0 84 | } 85 | if endX > srcImage.Bounds().Dx() { 86 | endX = srcImage.Bounds().Dx() 87 | } 88 | 89 | // Crops the image 90 | dstImage = imaging.Crop(srcImage, image.Rect(startX, 0, endX, srcImage.Bounds().Dy())) 91 | } 92 | 93 | if dstImage == nil { 94 | dstImage = imaging.Resize(srcImage, width, 0, imaging.Lanczos) 95 | } else { 96 | dstImage = imaging.Resize(dstImage, width, 0, imaging.Lanczos) 97 | } 98 | 99 | if dstImage.Bounds().Dy() > (int(float64(dstImage.Bounds().Dx()) * DEFAULT_ASPECT_RATIO)) { 100 | // If height is more than the aspect ratio allows, crops the top of the image to match the aspect ratio 101 | dstImage = imaging.Crop( 102 | dstImage, 103 | image.Rect(0, 0, dstImage.Bounds().Dx(), int(float64(dstImage.Bounds().Dx())*DEFAULT_ASPECT_RATIO)), 104 | ) 105 | } 106 | 107 | return dstImage 108 | } 109 | 110 | func ConvertImageToNRGBA(srcImage image.Image) (*image.NRGBA, error) { 111 | var nrgba *image.NRGBA 112 | 113 | switch img := srcImage.(type) { 114 | case *image.NRGBA: 115 | nrgba = img 116 | case 117 | *image.NRGBA64, 118 | *image.YCbCr, 119 | *image.NYCbCrA, 120 | *image.RGBA, 121 | *image.RGBA64, 122 | *image.Gray, 123 | *image.Gray16, 124 | *image.CMYK, 125 | *image.Paletted: 126 | 127 | b := img.Bounds() 128 | nrgba = image.NewNRGBA(b) 129 | draw.Draw(nrgba, b, img, b.Min, draw.Src) 130 | default: 131 | return nil, errors.New(fmt.Sprintf("unsupported image type: %T", img)) 132 | } 133 | 134 | return nrgba, nil 135 | } 136 | -------------------------------------------------------------------------------- /pkg/db/migrations/20220106011520_create_tables.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | PRAGMA foreign_keys = TRUE; 4 | 5 | CREATE TABLE IF NOT EXISTS library 6 | ( 7 | id integer UNIQUE NOT NULL, 8 | path text UNIQUE NOT NULL, 9 | layout text NOT NULL, 10 | PRIMARY KEY (id) 11 | ); 12 | 13 | CREATE TABLE IF NOT EXISTS gallery 14 | ( 15 | uuid text UNIQUE NOT NULL, 16 | library_id integer NOT NULL, 17 | archive_path text UNIQUE NOT NULL, 18 | title text NOT NULL, 19 | title_native text, 20 | title_short text, 21 | released text, 22 | circle text, 23 | artists text, 24 | series text, 25 | category text, 26 | language text, 27 | translated boolean, 28 | image_count int, 29 | archive_size int, 30 | archive_hash text, 31 | thumbnail text, 32 | nsfw boolean NOT NULL DEFAULT false, 33 | hidden boolean NOT NULL DEFAULT false, 34 | created_at datetime NOT NULL, 35 | updated_at datetime NOT NULL, 36 | PRIMARY KEY (uuid), 37 | FOREIGN KEY (library_id) 38 | REFERENCES library (id) 39 | ); 40 | 41 | CREATE TABLE IF NOT EXISTS tag 42 | ( 43 | id integer PRIMARY KEY AUTOINCREMENT NOT NULL, 44 | namespace text NOT NULL, 45 | name text NOT NULL, 46 | CONSTRAINT unique_tag UNIQUE (namespace, name) 47 | ); 48 | 49 | CREATE TABLE IF NOT EXISTS gallery_tag 50 | ( 51 | gallery_uuid text NOT NULL, 52 | tag_id integer NOT NULL, 53 | CONSTRAINT unique_tag UNIQUE (gallery_uuid, tag_id), 54 | CONSTRAINT gallery 55 | FOREIGN KEY (gallery_uuid) 56 | REFERENCES gallery (uuid) 57 | ON DELETE CASCADE, 58 | CONSTRAINT tag 59 | FOREIGN KEY (tag_id) 60 | REFERENCES tag (id) 61 | ON DELETE CASCADE 62 | ); 63 | 64 | CREATE TABLE IF NOT EXISTS reference 65 | ( 66 | gallery_uuid text UNIQUE NOT NULL, 67 | meta_internal boolean NOT NULL DEFAULT false, 68 | meta_path text, 69 | meta_match integer, 70 | urls text, 71 | exh_gid int, 72 | exh_token text, 73 | anilist_id int, 74 | PRIMARY KEY (gallery_uuid), 75 | CONSTRAINT gallery 76 | FOREIGN KEY (gallery_uuid) 77 | REFERENCES gallery (uuid) 78 | ON DELETE CASCADE 79 | ); 80 | 81 | CREATE TABLE IF NOT EXISTS user 82 | ( 83 | uuid text UNIQUE NOT NULL, 84 | username text UNIQUE NOT NULL, 85 | password text NOT NULL, 86 | role integer NOT NULL DEFAULT 10, 87 | created_at datetime NOT NULL, 88 | updated_at datetime NOT NULL, 89 | PRIMARY KEY (uuid) 90 | ); 91 | 92 | CREATE TABLE IF NOT EXISTS session 93 | ( 94 | id text NOT NULL, 95 | user_uuid text NOT NULL, 96 | name text, 97 | expires_at datetime, 98 | PRIMARY KEY (id, user_uuid), 99 | CONSTRAINT user 100 | FOREIGN KEY (user_uuid) 101 | REFERENCES user (uuid) 102 | ON DELETE CASCADE 103 | ); 104 | 105 | CREATE TABLE IF NOT EXISTS gallery_pref 106 | ( 107 | user_uuid text NOT NULL, 108 | gallery_uuid text NOT NULL, 109 | progress integer NOT NULL DEFAULT 0, 110 | favorite_group text, 111 | updated_at datetime NOT NULL, 112 | PRIMARY KEY (user_uuid, gallery_uuid), 113 | FOREIGN KEY (gallery_uuid) 114 | REFERENCES gallery (uuid), 115 | CONSTRAINT user 116 | FOREIGN KEY (user_uuid) 117 | REFERENCES user (uuid) 118 | ON DELETE CASCADE 119 | ); 120 | 121 | CREATE INDEX idx_title ON gallery (title); 122 | CREATE INDEX idx_title_native ON gallery (title_native); 123 | CREATE INDEX idx_series ON gallery (series); 124 | CREATE INDEX idx_category ON gallery (category); 125 | CREATE INDEX idx_archive_path ON gallery (archive_path); 126 | CREATE INDEX idx_updated_at ON gallery (updated_at); 127 | CREATE INDEX idx_tag ON tag (namespace, name); 128 | CREATE INDEX idx_gallery_pref ON gallery_pref (user_uuid, gallery_uuid); 129 | CREATE INDEX idx_favorite ON gallery_pref (favorite_group); 130 | CREATE INDEX idx_session ON session (id, user_uuid); 131 | -- +goose StatementEnd 132 | 133 | -- +goose Down 134 | -- +goose StatementBegin 135 | DROP TABLE IF EXISTS reference; 136 | DROP TABLE IF EXISTS tag; 137 | DROP TABLE IF EXISTS gallery_pref; 138 | DROP TABLE IF EXISTS gallery; 139 | DROP TABLE IF EXISTS library; 140 | DROP TABLE IF EXISTS session; 141 | DROP TABLE IF EXISTS user; 142 | -- +goose StatementEnd 143 | -------------------------------------------------------------------------------- /pkg/cache/gallery.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "errors" 5 | "io/fs" 6 | "os" 7 | "path" 8 | "sync" 9 | "time" 10 | 11 | "github.com/Mangatsu/server/internal/config" 12 | "github.com/Mangatsu/server/pkg/log" 13 | "github.com/djherbis/atime" 14 | "github.com/google/uuid" 15 | "go.uber.org/zap" 16 | ) 17 | 18 | type cacheValue struct { 19 | Accessed time.Time 20 | Mu *sync.Mutex 21 | } 22 | 23 | type GalleryCache struct { 24 | Path string 25 | Store map[string]cacheValue 26 | } 27 | 28 | var galleryCache *GalleryCache 29 | 30 | // InitGalleryCache initializes the abstraction layer for the gallery cache. 31 | func InitGalleryCache() { 32 | galleryCache = &GalleryCache{ 33 | Path: config.BuildCachePath(), 34 | Store: make(map[string]cacheValue), 35 | } 36 | 37 | iterateCacheEntries(func(pathToEntry string, accessTime time.Time) { 38 | maybeUUID := path.Base(pathToEntry) 39 | if _, err := uuid.Parse(maybeUUID); err != nil { 40 | return 41 | } 42 | 43 | galleryCache.Store[path.Base(pathToEntry)] = cacheValue{ 44 | Accessed: accessTime, 45 | Mu: &sync.Mutex{}, 46 | } 47 | }) 48 | } 49 | 50 | // PruneCache removes entries not accessed (internal timestamp in mem) in the last x time in a thread-safe manner. 51 | func PruneCache() { 52 | now := time.Now() 53 | for galleryUUID, value := range galleryCache.Store { 54 | value.Mu.Lock() 55 | if value.Accessed.Add(config.Options.Cache.TTL).Before(now) { 56 | if err := remove(galleryUUID); err != nil { 57 | log.Z.Error("failed to delete a cache entry", 58 | zap.Bool("thread-safe", true), 59 | zap.String("uuid", galleryUUID), 60 | zap.String("err", err.Error())) 61 | } 62 | } 63 | value.Mu.Unlock() 64 | } 65 | } 66 | 67 | // PruneCacheFS removes entries not accessed (filesystem timestamp) in the last x time. Not thread-safe. 68 | func PruneCacheFS() { 69 | now := time.Now() 70 | iterateCacheEntries(func(pathToEntry string, accessTime time.Time) { 71 | if accessTime.Add(config.Options.Cache.TTL).Before(now) { 72 | if err := os.Remove(pathToEntry); err != nil { 73 | log.Z.Error("failed to delete a cache entry", 74 | zap.Bool("thread-safe", false), 75 | zap.String("path", pathToEntry), 76 | zap.String("err", err.Error())) 77 | } 78 | } 79 | }) 80 | } 81 | 82 | // Read reads the cached gallery from the disk. If it doesn't exist, it will be created and then read. 83 | func Read(archivePath string, galleryUUID string) ([]string, int) { 84 | galleryCache.Store[galleryUUID] = cacheValue{ 85 | Accessed: time.Now(), 86 | Mu: &sync.Mutex{}, 87 | } 88 | 89 | galleryCache.Store[galleryUUID].Mu.Lock() 90 | defer galleryCache.Store[galleryUUID].Mu.Unlock() 91 | 92 | files, count := extractGallery(archivePath, galleryUUID) 93 | if count == 0 { 94 | return files, count 95 | } 96 | 97 | return files, count 98 | } 99 | 100 | // remove wipes the cached gallery from the disk. 101 | func remove(galleryUUID string) error { 102 | // Paranoid check to make sure that the base is a real UUID, since we don't want to delete anything else. 103 | maybeUUID := path.Base(galleryUUID) 104 | if _, err := uuid.Parse(maybeUUID); err != nil { 105 | delete(galleryCache.Store, galleryUUID) 106 | return err 107 | } 108 | 109 | galleryPath := config.BuildCachePath(galleryUUID) 110 | if err := os.RemoveAll(galleryPath); err != nil { 111 | if errors.Is(err, fs.ErrNotExist) { 112 | delete(galleryCache.Store, galleryUUID) 113 | } 114 | return err 115 | } 116 | 117 | delete(galleryCache.Store, galleryUUID) 118 | 119 | return nil 120 | } 121 | 122 | // iterateCacheEntries iterates over all cache entries and calls the callback function for each entry. 123 | func iterateCacheEntries(callback func(pathToEntry string, accessTime time.Time)) { 124 | cachePath := config.BuildCachePath() 125 | cacheEntries, err := os.ReadDir(cachePath) 126 | if err != nil { 127 | log.Z.Error("could not read cache dir", 128 | zap.String("path", cachePath), 129 | zap.String("err", err.Error())) 130 | return 131 | } 132 | 133 | for _, entry := range cacheEntries { 134 | info, err := entry.Info() 135 | if err != nil { 136 | log.Z.Error("could to read cache entry info", 137 | zap.String("path", cachePath), 138 | zap.String("err", err.Error())) 139 | return 140 | } 141 | 142 | pathToEntry := path.Join(cachePath, entry.Name()) 143 | accessTime, err := atime.Stat(pathToEntry) 144 | if err != nil { 145 | log.Z.Debug("could to read the access time", 146 | zap.String("name", entry.Name()), 147 | zap.String("path", cachePath), 148 | zap.String("err", err.Error())) 149 | accessTime = info.ModTime() 150 | } 151 | 152 | callback(pathToEntry, accessTime) 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /docs/docker-compose.example.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | backend: 5 | hostname: mtsuserver 6 | image: ghcr.io/mangatsu/server:latest 7 | user: 1000:1000 # should have access to the volumes. For data dir, both read and write is required. For libraries only read is required. 8 | ports: 9 | - '5050:5050' # container:host 10 | restart: always 11 | environment: 12 | # Environment: production or development 13 | MTSU_ENV: production 14 | # Log level: debug, info, warn, error 15 | MTSU_LOG_LEVEL: info 16 | # Credentials for the initial admin user. Changing them is recommended. 17 | MTSU_INITIAL_ADMIN_NAME: admin 18 | MTSU_INITIAL_ADMIN_PW: admin321 19 | # Make sure that the paths match containerpaths below in the volumes section! 20 | # Format: ;;;; ... 21 | MTSU_BASE_PATHS: freeform1;/library1;;structured2;/library2 22 | # Disable internal cache server? Remember apostrophes around the value. 23 | MTSU_DISABLE_CACHE_SERVER: 'false' 24 | # Max size of the cache where galleries are extracted from the library in MB. Can overflow a bit especially if set too low. 25 | MTSU_CACHE_SIZE: 10000 26 | # Time to live for the cache in seconds (604800 = 1 week). 27 | MTSU_CACHE_TTL: 604800 28 | # Modes: public, restricted, private 29 | MTSU_VISIBILITY: private 30 | # Password for restricted mode. 31 | MTSU_RESTRICTED_PASSPHRASE: secretpassword 32 | # Allow registrations? Remember apostrophes around the value. 33 | MTSU_REGISTRATIONS: 'false' 34 | # Secret to sign JWTs for login sessions in the backend. Recommended to change. 35 | MTSU_JWT_SECRET: 9Wag7sMvKl3aF6K5lwIg6TI42ia2f6BstZAVrdJIq8Mp38lnl7UzQMC1qjKyZCBzHFGbbqsA0gKcHqDuyXQAhWoJ0lcx4K5q 36 | # Domain for the server. Used in cookies. 37 | # For example, if the address for the server is "api.example.com", and for the frontend "read.example.com", 38 | # the value here should be "example.com" for the cookies to work properly between subdomains. 39 | # https://developer.mozilla.org/en-US/docs/Web/HTTP/Cookies#define_where_cookies_are_sent 40 | MTSU_DOMAIN: example.org 41 | # When true, the server only allows authenticated connections from the MTSU_DOMAIN and its subdomains (eg .*.example.org). 42 | # When false, connections from every origin is allowed. 43 | # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin 44 | MTSU_STRICT_ACAO: 'false' 45 | # When true, Mangatsu can be accessed only through HTTPS or localhost domains. 46 | MTSU_SECURE: 'true' 47 | # Thumbnail image format: webp 48 | MTSU_THUMBNAIL_FORMAT: webp 49 | # Similarity threshold for the fuzzy match for gallery and metadata filenames. 50 | # The higher the value, the more similar the results has to be to match. 0.1 - 1.0. 51 | MTSU_FUZZY_SEARCH_SIMILARITY: 0.7 52 | # Set to false to use right-to-left (RTL) default for galleries. Otherwise, defaults to left-to-right (like Japanese manga). 53 | MTSU_LTR: 'true' 54 | # Usually the following three envs don't need changing as they are the container's internal hostname, port and data path. 55 | # In case of port conflict, change the port here and the first one above in the ports section. 56 | MTSU_HOSTNAME: mtsuserver 57 | MTSU_PORT: 5050 58 | MTSU_DATA_PATH: /data 59 | # Above environmental variables can also be loaded from a file: 60 | #env_file: 61 | # - ./.env 62 | volumes: 63 | # Change paths below. Add as many volumes as you need. Format: hostpath:containerpath:ro (ro = read-only) 64 | # Make sure that the containerpaths match the paths above or the ones in the .env file. 65 | - "/path/to/data:/data" 66 | - "/path/to/your/library/1:/library1:ro" 67 | - "/path/to/your/library/2:/library2:ro" 68 | 69 | frontend: 70 | hostname: mtsuweb 71 | image: ghcr.io/mangatsu/web:latest 72 | ports: 73 | - '3030:3030' # container:host 74 | restart: always 75 | environment: 76 | NODE_ENV: production 77 | # URL to the backend server above. If running locally, localhost can be used in the following way: http://localhost:5050 78 | NEXT_PUBLIC_MANGATSU_API_URL: https://mangatsu-api.example.com 79 | # Internal URL to the backend server. Required when running both containers locally without external network. 80 | # If not specified, some calls such as image URLs will be made through the public URL. 81 | NEXT_PUBLIC_INTERNAL_MANGATSU_API_URL: http://mtsuserver:5050 82 | # Hostname or the domain where images are hosted. 83 | # Usually the same as the domain in the internal (if specified) API URL, or otherwise the public API URL. 84 | NEXT_MANGATSU_IMAGE_HOSTNAME: mtsuserver 85 | # If some other container is already using the same port, you can use this to avoid conflicts. Remember to change the first port in 3030:3030 too. 86 | PORT: 3030 87 | 88 | # Above environmental variables can also be loaded from a file: 89 | #env_file: 90 | # - ./.env.web 91 | -------------------------------------------------------------------------------- /docs/ENVIRONMENTALS.md: -------------------------------------------------------------------------------- 1 | ## 📝 Mangatsu Server - Configuration 2 | Usable options inside the **.env** or **docker-compose.yml**: 3 | 4 | _~~Struck out~~ values have no effect yet._ 5 | 6 | - **MTSU_ENV**=production 7 | - Environment: production, development 8 | - **MTSU_LOG_LEVEL**=info 9 | - Log level: debug, info, warn, error 10 | - **MTSU_INITIAL_ADMIN_NAME**=admin 11 | - **MTSU_INITIAL_ADMIN_PW**=admin321 12 | - Credentials for the initial admin user. Recommended to change. 13 | - **MTSU_DOMAIN**: example.org. 14 | - For example, if the address for the server is "api.example.com", and for the frontend "read.example.com", the value here should be "example.com" for the cookies to work properly between subdomains. 15 | - https://developer.mozilla.org/en-US/docs/Web/HTTP/Cookies#define_where_cookies_are_sent 16 | - **MTSU_STRICT_ACAO**: 'true' 17 | - When true, the server only allows authenticated connections from the MTSU_DOMAIN and its subdomains (eg .*.example.org). When false, connections from every origin is allowed. 18 | - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin 19 | - **MTSU_HOSTNAME**=localhost 20 | - **MTSU_PORT**=5050 21 | - Hostname and port for the server. Use **mtsuserver** as the hostname if using Docker Compose. 22 | - **MTSU_BASE_PATHS**=freeform1;/home/user/doujinshi;;structured2;/home/user/manga 23 | - Paths to the archive directories. Relative or absolute paths are accepted. 24 | - First specify the type of the directory and a numerical ID (e.g. freeform1 or structured2) and then the path separated by a semicolon: `;`. 25 | - Multiple paths can be separated by a double-semicolon: `;;`. 26 | - Format: `;;;;`... 27 | - If using Docker Compose, make sure that the paths match the containerpaths in the volumes section. 28 | - **MTSU_DATA_PATH**=../data 29 | - Location of the data dir which includes the SQLite db and the cache for gallery images and thumbnails. Relative or absolute paths are accepted. 30 | - Doesn't need changing if using Docker Compose. 31 | - **MTSU_DISABLE_CACHE_SERVER**=false 32 | - Set true to disable the internal cache server (serves media files and thumbnails). Useful if one wants to use the web server such as NGINX to serve the files. 33 | - **MTSU_CACHE_TTL**=336h 34 | - Cache time to live (for example `336h` (2 weeks), `8h30m`). If a gallery is not viewed for this time, it will be purged from the cache. 35 | - ~~**MTSU_CACHE_SIZE**~~=10000 36 | - Max size of the cache where galleries are extracted from the library in MB. Can overflow a bit especially if set too low. 37 | - **MTSU_DB_NAME**=mangatsu 38 | - Name of the SQLite database file 39 | - ~~**MTSU_DB**~~=sqlite 40 | - Database type: `sqlite`, `postgres`, `mysql` or `mariadb` 41 | - ~~**MTSU_DB_HOST**~~=localhost 42 | - Hostname of the database server. 43 | - ~~**MTSU_DB_PORT**~~=5432 44 | - Usually 5432 for PostgreSQL and 3306 for MySQL and MariaDB. 45 | - ~~**MTSU_DB_USER**~~=mtsu-user 46 | - ~~**MTSU_DB_PASSWORD**~~=s3cr3t 47 | - **MTSU_VISIBILITY**=public 48 | - **public**: anyone can access the collection and its galleries. 49 | - **restricted**: users need a global passphrase to access collection and its galleries. 50 | - **private**: only logged-in users can access the collection and its galleries. 51 | - In all modes, user accounts are supported and have more privileges than anonymous users (e.g. favorite galleries). 52 | - **MTSU_RESTRICTED_PASSPHRASE**=secretpassword 53 | - Passphrase to access the collection and its galleries. 54 | - Only used when **VISIBILITY** is set to **restricted**. 55 | - **MTSU_REGISTRATIONS**=false 56 | - Whether to allow user registrations. If set to false, only admins can create new users. 57 | - **Currently, only affects the API path /register. Has no effect in the frontend.** 58 | - **MTSU_JWT_SECRET**=secret123 59 | - Secret to sign JWTs for login sessions in the backend. Recommended to change. 60 | - **MTSU_THUMBNAIL_FORMAT**=webp 61 | - Supported formats: webp 62 | - AVIF support is planned. AVIF is said to take 20% longer to encode, but it compresses to 20% smaller size compared to WebP. 63 | - **MTSU_LTR**=true 64 | - Set to false to use right-to-left (RTL) default for galleries. Otherwise, defaults to left-to-right (like Japanese manga). 65 | 66 | ## 📝 Mangatsu Web - Configuration 67 | 68 | - **NEXT_PUBLIC_MANGATSU_API_URL**=https://mangatsu-api.example.com 69 | - URL to the backend API server. 70 | - **NEXT_PUBLIC_INTERNAL_MANGATSU_API_URL**=http://mtsuserver:5050 71 | - Internal URL to the backend server. Required when running both containers locally without external network. 72 | - For example, if both containers are running on the same network, the value should probably be "http://mtsuserver:5050". 73 | - **NEXT_MANGATSU_IMAGE_HOSTNAME**=mangatsu-api.example.com 74 | - Hostname or the domain where images are hosted. Usually the same as the domain in the API URL. 75 | - **PORT**=3030 76 | - Port to run the web client on. If you change this and also use Docker Compose, remember to update the first port of frontend's ports in the `docker-compose.yml` file. 77 | -------------------------------------------------------------------------------- /pkg/library/scan.go: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import ( 4 | "github.com/Mangatsu/server/internal/config" 5 | "github.com/Mangatsu/server/pkg/cache" 6 | "github.com/Mangatsu/server/pkg/constants" 7 | "github.com/Mangatsu/server/pkg/db" 8 | "github.com/Mangatsu/server/pkg/log" 9 | "github.com/Mangatsu/server/pkg/utils" 10 | "github.com/mholt/archiver/v4" 11 | "go.uber.org/zap" 12 | "io/fs" 13 | "path" 14 | "path/filepath" 15 | "strings" 16 | ) 17 | 18 | func countImages(archivePath string) (uint64, error) { 19 | filesystem, err := archiver.FileSystem(nil, archivePath) 20 | if err != nil { 21 | log.Z.Error("could not open archive", 22 | zap.String("path", archivePath), 23 | zap.String("err", err.Error()), 24 | ) 25 | 26 | return 0, err 27 | } 28 | 29 | var fileCount uint64 30 | 31 | err = fs.WalkDir(filesystem, ".", func(s string, d fs.DirEntry, err error) error { 32 | if err != nil { 33 | return err 34 | } 35 | 36 | if !d.IsDir() && constants.ImageExtensions.MatchString(d.Name()) { 37 | fileCount++ 38 | } 39 | 40 | return nil 41 | }) 42 | 43 | if err != nil { 44 | log.Z.Error("could not count files in archive", zap.String("path", archivePath), zap.String("err", err.Error())) 45 | return 0, err 46 | } 47 | 48 | return fileCount, nil 49 | } 50 | 51 | func walk(libraryPath string, libraryID int32, libraryLayout config.Layout) fs.WalkDirFunc { 52 | return func(s string, d fs.DirEntry, err error) error { 53 | if err != nil { 54 | return err 55 | } 56 | 57 | if d.IsDir() { 58 | return nil 59 | } 60 | 61 | isImage := constants.ImageExtensions.MatchString(d.Name()) 62 | isArchive := constants.ArchiveExtensions.MatchString(d.Name()) 63 | if !isArchive && !isImage { 64 | return nil 65 | } 66 | 67 | s = filepath.ToSlash(s) 68 | relativePath := config.RelativePath(libraryPath, s) 69 | fullPath := config.BuildLibraryPath(libraryPath, relativePath) 70 | 71 | // If an image is found, the parent dir will be considered as a gallery. 72 | if isImage { 73 | relativePath = path.Dir(relativePath) 74 | } 75 | 76 | // Skip if already in database 77 | foundGallery := db.ArchivePathFound(relativePath) 78 | if foundGallery != nil { 79 | log.Z.Debug("skipping archive already in db", zap.String("name", d.Name())) 80 | 81 | cache.ProcessingStatusCache.AddScanSkippedGallery(foundGallery[0].UUID) 82 | return nil 83 | } 84 | 85 | // Series name from the dir name if Structured layout 86 | series := "" 87 | if libraryLayout == config.Structured { 88 | dirs := strings.SplitN(relativePath, "/", 2) 89 | series = dirs[0] 90 | } 91 | 92 | var title string 93 | var size int64 94 | 95 | if isImage { 96 | title = path.Base(relativePath) 97 | 98 | if size, err = utils.DirSize(fullPath); err != nil { 99 | log.Z.Error("failed to get dir size", zap.String("path", fullPath), zap.String("err", err.Error())) 100 | } 101 | } else { 102 | n := strings.LastIndex(d.Name(), path.Ext(d.Name())) 103 | title = d.Name()[:n] 104 | 105 | if size, err = utils.FileSize(fullPath); err != nil { 106 | log.Z.Error("failed to get file size", zap.String("path", fullPath), zap.String("err", err.Error())) 107 | } 108 | } 109 | 110 | imageCount, err := countImages(fullPath) 111 | if err != nil { 112 | log.Z.Error("failed to count images", zap.String("path", fullPath), zap.String("err", err.Error())) 113 | } 114 | 115 | uuid, err := db.NewGallery(relativePath, libraryID, title, series, size, imageCount) 116 | 117 | if err != nil { 118 | log.Z.Error("failed to add gallery to db", 119 | zap.String("path", relativePath), 120 | zap.String("err", err.Error())) 121 | 122 | cache.ProcessingStatusCache.AddScanError(relativePath, err.Error(), map[string]string{ 123 | "libraryID": string(libraryID), 124 | "title": title, 125 | }) 126 | 127 | } else { 128 | // Generates cover thumbnail 129 | go GenerateCoverThumbnail(fullPath, uuid) 130 | 131 | log.Z.Info("added gallery", zap.String("path", relativePath), zap.String("uuid", uuid)) 132 | 133 | cache.ProcessingStatusCache.AddScanFoundGallery(uuid) 134 | } 135 | 136 | if isImage { 137 | return fs.SkipDir 138 | } 139 | 140 | return err 141 | } 142 | } 143 | 144 | func ScanArchives() { 145 | // TODO: Quick scan by only checking directories that have been modified. 146 | // Not too important as current implementation is pretty fast already. 147 | libraries, err := db.GetOnlyLibraries() 148 | if err != nil { 149 | log.Z.Error("failed to find libraries to scan", zap.String("err", err.Error())) 150 | 151 | cache.ProcessingStatusCache.AddScanError("library scan fail", err.Error(), nil) 152 | return 153 | } 154 | 155 | cache.ProcessingStatusCache.SetScanRunning(true) 156 | defer cache.ProcessingStatusCache.SetScanRunning(false) 157 | 158 | for _, library := range libraries { 159 | err := filepath.WalkDir(library.Path, walk(library.Path, library.ID, config.Layout(library.Layout))) 160 | if err != nil { 161 | log.Z.Error("skipping library as an error occurred during scanning", 162 | zap.String("path", library.Path), 163 | zap.String("err", err.Error())) 164 | 165 | cache.ProcessingStatusCache.AddScanError(library.Path, err.Error(), nil) 166 | continue 167 | } 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /pkg/types/sqlite/table/gallery.go: -------------------------------------------------------------------------------- 1 | // 2 | // Code generated by go-jet DO NOT EDIT. 3 | // 4 | // WARNING: Changes to this file may cause incorrect behavior 5 | // and will be lost if the code is regenerated 6 | // 7 | 8 | package table 9 | 10 | import ( 11 | "github.com/go-jet/jet/v2/sqlite" 12 | ) 13 | 14 | var Gallery = newGalleryTable("", "gallery", "") 15 | 16 | type galleryTable struct { 17 | sqlite.Table 18 | 19 | //Columns 20 | UUID sqlite.ColumnString 21 | LibraryID sqlite.ColumnInteger 22 | ArchivePath sqlite.ColumnString 23 | Title sqlite.ColumnString 24 | TitleNative sqlite.ColumnString 25 | TitleTranslated sqlite.ColumnString 26 | Category sqlite.ColumnString 27 | Series sqlite.ColumnString 28 | Released sqlite.ColumnString 29 | Language sqlite.ColumnString 30 | Translated sqlite.ColumnBool 31 | Nsfw sqlite.ColumnBool 32 | Hidden sqlite.ColumnBool 33 | ImageCount sqlite.ColumnInteger 34 | ArchiveSize sqlite.ColumnInteger 35 | ArchiveHash sqlite.ColumnString 36 | Thumbnail sqlite.ColumnString 37 | CreatedAt sqlite.ColumnTimestamp 38 | UpdatedAt sqlite.ColumnTimestamp 39 | Deleted sqlite.ColumnBool 40 | PageThumbnails sqlite.ColumnInteger 41 | 42 | AllColumns sqlite.ColumnList 43 | MutableColumns sqlite.ColumnList 44 | } 45 | 46 | type GalleryTable struct { 47 | galleryTable 48 | 49 | EXCLUDED galleryTable 50 | } 51 | 52 | // AS creates new GalleryTable with assigned alias 53 | func (a GalleryTable) AS(alias string) *GalleryTable { 54 | return newGalleryTable(a.SchemaName(), a.TableName(), alias) 55 | } 56 | 57 | // Schema creates new GalleryTable with assigned schema name 58 | func (a GalleryTable) FromSchema(schemaName string) *GalleryTable { 59 | return newGalleryTable(schemaName, a.TableName(), a.Alias()) 60 | } 61 | 62 | func newGalleryTable(schemaName, tableName, alias string) *GalleryTable { 63 | return &GalleryTable{ 64 | galleryTable: newGalleryTableImpl(schemaName, tableName, alias), 65 | EXCLUDED: newGalleryTableImpl("", "excluded", ""), 66 | } 67 | } 68 | 69 | func newGalleryTableImpl(schemaName, tableName, alias string) galleryTable { 70 | var ( 71 | UUIDColumn = sqlite.StringColumn("uuid") 72 | LibraryIDColumn = sqlite.IntegerColumn("library_id") 73 | ArchivePathColumn = sqlite.StringColumn("archive_path") 74 | TitleColumn = sqlite.StringColumn("title") 75 | TitleNativeColumn = sqlite.StringColumn("title_native") 76 | TitleTranslatedColumn = sqlite.StringColumn("title_translated") 77 | CategoryColumn = sqlite.StringColumn("category") 78 | SeriesColumn = sqlite.StringColumn("series") 79 | ReleasedColumn = sqlite.StringColumn("released") 80 | LanguageColumn = sqlite.StringColumn("language") 81 | TranslatedColumn = sqlite.BoolColumn("translated") 82 | NsfwColumn = sqlite.BoolColumn("nsfw") 83 | HiddenColumn = sqlite.BoolColumn("hidden") 84 | ImageCountColumn = sqlite.IntegerColumn("image_count") 85 | ArchiveSizeColumn = sqlite.IntegerColumn("archive_size") 86 | ArchiveHashColumn = sqlite.StringColumn("archive_hash") 87 | ThumbnailColumn = sqlite.StringColumn("thumbnail") 88 | CreatedAtColumn = sqlite.TimestampColumn("created_at") 89 | UpdatedAtColumn = sqlite.TimestampColumn("updated_at") 90 | DeletedColumn = sqlite.BoolColumn("deleted") 91 | PageThumbnailsColumn = sqlite.IntegerColumn("page_thumbnails") 92 | allColumns = sqlite.ColumnList{UUIDColumn, LibraryIDColumn, ArchivePathColumn, TitleColumn, TitleNativeColumn, TitleTranslatedColumn, CategoryColumn, SeriesColumn, ReleasedColumn, LanguageColumn, TranslatedColumn, NsfwColumn, HiddenColumn, ImageCountColumn, ArchiveSizeColumn, ArchiveHashColumn, ThumbnailColumn, CreatedAtColumn, UpdatedAtColumn, DeletedColumn, PageThumbnailsColumn} 93 | mutableColumns = sqlite.ColumnList{LibraryIDColumn, ArchivePathColumn, TitleColumn, TitleNativeColumn, TitleTranslatedColumn, CategoryColumn, SeriesColumn, ReleasedColumn, LanguageColumn, TranslatedColumn, NsfwColumn, HiddenColumn, ImageCountColumn, ArchiveSizeColumn, ArchiveHashColumn, ThumbnailColumn, CreatedAtColumn, UpdatedAtColumn, DeletedColumn, PageThumbnailsColumn} 94 | ) 95 | 96 | return galleryTable{ 97 | Table: sqlite.NewTable(schemaName, tableName, alias, allColumns...), 98 | 99 | //Columns 100 | UUID: UUIDColumn, 101 | LibraryID: LibraryIDColumn, 102 | ArchivePath: ArchivePathColumn, 103 | Title: TitleColumn, 104 | TitleNative: TitleNativeColumn, 105 | TitleTranslated: TitleTranslatedColumn, 106 | Category: CategoryColumn, 107 | Series: SeriesColumn, 108 | Released: ReleasedColumn, 109 | Language: LanguageColumn, 110 | Translated: TranslatedColumn, 111 | Nsfw: NsfwColumn, 112 | Hidden: HiddenColumn, 113 | ImageCount: ImageCountColumn, 114 | ArchiveSize: ArchiveSizeColumn, 115 | ArchiveHash: ArchiveHashColumn, 116 | Thumbnail: ThumbnailColumn, 117 | CreatedAt: CreatedAtColumn, 118 | UpdatedAt: UpdatedAtColumn, 119 | Deleted: DeletedColumn, 120 | PageThumbnails: PageThumbnailsColumn, 121 | 122 | AllColumns: allColumns, 123 | MutableColumns: mutableColumns, 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /pkg/api/helpers.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | "errors" 7 | "github.com/Mangatsu/server/internal/config" 8 | "github.com/Mangatsu/server/pkg/db" 9 | "github.com/Mangatsu/server/pkg/types/sqlite/model" 10 | "github.com/Mangatsu/server/pkg/utils" 11 | "github.com/weppos/publicsuffix-go/publicsuffix" 12 | "math" 13 | "net/http" 14 | "strconv" 15 | "strings" 16 | ) 17 | 18 | func parseQueryParams(r *http.Request) db.Filters { 19 | order := db.Order(r.URL.Query().Get("order")) 20 | sortBy := db.SortBy(r.URL.Query().Get("sortby")) 21 | searchTerm := r.URL.Query().Get("search") 22 | category := r.URL.Query().Get("category") 23 | series := r.URL.Query().Get("series") 24 | favoriteGroup := r.URL.Query().Get("favorite") 25 | nsfw := r.URL.Query().Get("nsfw") 26 | rawTags := r.URL.Query()["tag"] // namespace:name 27 | grouped := r.URL.Query().Get("grouped") 28 | 29 | var tags []model.Tag 30 | for _, rawTag := range rawTags { 31 | tag := strings.Split(rawTag, ":") 32 | if len(tag) != 2 { 33 | continue 34 | } 35 | tags = append(tags, model.Tag{Namespace: tag[0], Name: tag[1]}) 36 | } 37 | 38 | limit, err := strconv.ParseUint(r.URL.Query().Get("limit"), 10, 64) 39 | if err != nil { 40 | limit = 50 41 | } else { 42 | limit = utils.ClampU(limit, 1, 100) 43 | } 44 | 45 | offset, err := strconv.ParseUint(r.URL.Query().Get("offset"), 10, 64) 46 | if err != nil { 47 | offset = 0 48 | } else { 49 | offset = utils.ClampU(offset, 0, math.MaxUint64) 50 | } 51 | 52 | seed, err := strconv.ParseUint(r.URL.Query().Get("seed"), 10, 64) 53 | if err != nil { 54 | seed = 0 55 | } 56 | 57 | return db.Filters{ 58 | SearchTerm: strings.TrimSpace(searchTerm), 59 | Order: order, 60 | SortBy: sortBy, 61 | Limit: limit, 62 | Offset: offset, 63 | Category: category, 64 | Series: series, 65 | FavoriteGroup: favoriteGroup, 66 | NSFW: nsfw, 67 | Tags: tags, 68 | Grouped: grouped, 69 | Seed: seed, 70 | } 71 | } 72 | 73 | func convertTagsToMap(tags []model.Tag) map[string][]string { 74 | tagMap := map[string][]string{} 75 | for _, tag := range tags { 76 | tagMap[tag.Namespace] = append(tagMap[tag.Namespace], tag.Name) 77 | } 78 | return tagMap 79 | } 80 | 81 | func convertMetadata(metadata db.CombinedMetadata) MetadataResult { 82 | return MetadataResult{ 83 | ArchivePath: metadata.ArchivePath, 84 | Hidden: metadata.Hidden, 85 | Gallery: metadata.Gallery, 86 | Tags: convertTagsToMap(metadata.Tags), 87 | Reference: metadata.Reference, 88 | GalleryPref: metadata.GalleryPref, 89 | Library: metadata.Library, 90 | } 91 | } 92 | 93 | // hasAccess handles access based on the Visibility option. Role restricts access to the specified role. 94 | // NoRole (0) allows access to anonymous users if the Visibility is Public or Restricted (passphrase required). 95 | func hasAccess(w http.ResponseWriter, r *http.Request, role db.Role) (bool, *string) { 96 | publicAccess := config.Options.Visibility == config.Public && role == db.NoRole 97 | 98 | token := readJWT(r) 99 | if token != "" { 100 | access, userUUID := verifyJWT(token, role) 101 | if access { 102 | return access, userUUID 103 | } 104 | 105 | if publicAccess { 106 | return true, nil 107 | } 108 | 109 | restrictedAccess := config.Options.Visibility == config.Restricted && role == db.NoRole 110 | if restrictedAccess && token == config.Credentials.Passphrase { 111 | return true, nil 112 | } 113 | 114 | errorHandler(w, http.StatusUnauthorized, "", r.URL.Path) 115 | return false, nil 116 | } 117 | 118 | // Username & password auth 119 | if r.Body != nil { 120 | credentials := &Credentials{} 121 | err := json.NewDecoder(r.Body).Decode(credentials) 122 | if err == nil && credentials.Username != "" && credentials.Password != "" { 123 | access, userUUID, _ := loginHelper(w, *credentials, role) 124 | if !access { 125 | errorHandler(w, http.StatusUnauthorized, "", r.URL.Path) 126 | return false, nil 127 | } 128 | return access, userUUID 129 | } 130 | } 131 | 132 | // If public, anonymous access without passphrase is allowed 133 | if publicAccess { 134 | return true, nil 135 | } 136 | 137 | errorHandler(w, http.StatusUnauthorized, "", r.URL.Path) 138 | return false, nil 139 | } 140 | 141 | // loginHelper handles login 142 | func loginHelper(w http.ResponseWriter, credentials Credentials, requiredRole db.Role) (bool, *string, *int32) { 143 | err := db.MigratePassword(credentials.Username, credentials.Password) 144 | if err != nil { 145 | if errors.Is(err, sql.ErrNoRows) { 146 | errorHandler(w, http.StatusUnauthorized, "", "") 147 | } else { 148 | errorHandler(w, http.StatusInternalServerError, err.Error(), "") 149 | } 150 | return false, nil, nil 151 | } 152 | 153 | userUUID, role, err := db.Login(credentials.Username, credentials.Password, requiredRole) 154 | if err != nil || userUUID == nil { 155 | errorHandler(w, http.StatusUnauthorized, "", "") 156 | return false, nil, nil 157 | } 158 | 159 | return true, userUUID, role 160 | } 161 | 162 | // originAllowed returns true if the origin is allowed. If MTSU_STRICT_ACAO is false, it will always return true. 163 | func originAllowed(origin string) bool { 164 | if !config.Options.StrictACAO { 165 | return true 166 | } 167 | 168 | domain, err := publicsuffix.Domain(origin) 169 | return err == nil && domain == config.Options.Domain 170 | } 171 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 |

Mangatsu

3 | 4 |

5 | 6 |

7 | 8 | > 🌕 Server application for **storing**, **tagging** and **reading** doujinshi, manga, art collections and other galleries with API access and user control. Written in Go. 9 | > The name, **Mangatsu**, is a play on Japanese words **mangetsu** (満月, full moon) and **manga** (漫画, comic). 10 | 11 | ### **[📰 CHANGELOG](docs/CHANGELOG.md)** | **[❤ CONTRIBUTING](docs/CONTRIBUTING.md)** 12 | 13 | ### At experimental stage until version 1.0.0. Expect breaking changes. 14 | 15 | ## 📌 Features 16 | - Organizing and tagging local (and remote, with tools like [rclone](https://rclone.org)) collections of manga, doujinshi and other art 17 | - **Mangatsu will never do any writes inside specified locations.** 18 | - Supports **ZIP** (or CBZ), **RAR** (or CBR), **7z** and plain image (png, jpg, jpeg, webp, avif, heif, gif, tiff, bmp) files. 19 | - PDF and video support is planned. 20 | - Metadata parsing from filenames and JSON files (inside or beside the archive) 21 | - Support for more sources is planned such as TXT files from EH/ExH 22 | - API-access to the collection and archives 23 | - Extensive filtering, sorting and searching capabilities 24 | - Additional features for registered users such as tracking reading progress and adding favorite groups. Currently only in API, not in UI. 25 | - User access control 26 | - **Private**: only logged-in users can access the collection and archives (public registration disabled by default). 27 | - **Restricted**: users need a global passphrase to access collection and its galleries 28 | - **Public**: anyone can access (only read) collection and its galleries 29 | - Assignable roles (admin, member, viewer), and login sessions (can be managed through web) 30 | - Local cache and thumbnail support. _File server can be disabled to allow web servers like NGINX to handle the files._ 31 | 32 | ## 🖼️ Preview 33 | 34 | | Main Catalogue | Main Catalogue (grouped galleries) | 35 | |---------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------| 36 | | [![catalogue](docs/images/thumbnails/catalogue.png)](docs/images/catalogue.png) | [![grouped catalogue](docs/images/thumbnails/catalogue_grouped.png)](docs/images/catalogue_grouped.png) | 37 | 38 | | Gallery Page + Editing | Series Listing | 39 | |---------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------| 40 | | [![editing gallery](docs/images/thumbnails/editing_gallery.png)](docs/images/editing_gallery.png) | [![series listing](docs/images/thumbnails/series_listing.png)](docs/images/series_listing.png) | 41 | 42 | 43 | | User Settings | Administration | 44 | |------------------------------------------------------------------------------|------------------------------------------------------------------------------| 45 | | [![settings](docs/images/thumbnails/settings.png)](docs/images/settings.png) | [![administration](docs/images/thumbnails/admin.png)](docs/images/admin.png) | 46 | 47 | ## 📌 Clients 48 | 49 | ### 🌏 Web client 50 | - Included in the Docker setup below 51 | - Source: [Mangatsu Web](http://github.com/Mangatsu/web) 52 | 53 | ### 📱 [Tachiyomi](https://tachiyomi.org) extension for Android 54 | - Coming soon 55 | 56 | ## 📌 Installation and usage 57 | 58 | ### 📖 Guides 59 | 60 | - **[📝 Configuration with environmentals](docs/ENVIRONMENTALS.md)** 61 | - **[📚 Library directory structure](docs/LIBRARY.md)** 62 | 63 | ### 🐳 Docker setup (recommended) 64 | 65 | #### GitHub Container Registry: [server](https://github.com/Mangatsu/server/pkgs/container/server) & [web](https://github.com/Mangatsu/server/pkgs/container/server) 66 | 67 | #### DockerHub: [server](https://hub.docker.com/r/luukuton/mangatsu-server/) & [web](https://hub.docker.com/r/luukuton/mangatsu-web) images 68 | 69 | - Set up a webserver of your choice. NGINX is recommended. 70 | - [Example config](docs/nginx.conf). The same config can be used for both the server and the web client. Just change the domains, SSL cert paths and ports. 71 | - Install [Docker](https://docs.docker.com/engine/install/) (Linux, Windows or MacOS) 72 | - Local archives 73 | - Download the [docker-compose.example.yml](docs/docker-compose.example.yml) and rename it to docker-compose.yml 74 | - Edit the docker-compose.yml file to your needs 75 | - Create data and archive directories 76 | - Network archives with [Rclone](https://rclone.org) 77 | - Follow the [guide on Rclone site](https://rclone.org/docker/) 78 | - Download the [docker-compose.rclone.yml](docs/docker-compose.rclone.yml) and rename it to docker-compose.yml 79 | - Run `docker-compose up -d` to start the server and web client 80 | - Update by running `docker-compose down`, `docker-compose pull` and then `docker-compose up -d` 81 | 82 | ### 💻 Local setup 83 | 84 | - Set up server 85 | - Copy example.env as .env and change the values according to your needs 86 | - Build `go build ./cmd/mangatsu-server` 87 | - Run `./mangatsu-server` (`mangatsu-server.exe` on Windows) 88 | - Set up web 89 | - [Guide on github.com/Mangatsu/web](https://github.com/Mangatsu/web) 90 | -------------------------------------------------------------------------------- /pkg/metadata/metadata_test.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | import ( 4 | "github.com/Mangatsu/server/pkg/utils" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func TestParseTitle(t *testing.T) { 10 | wantMap := map[string]TitleMeta{ 11 | "(C99) [doujin circle (some artist)] very lewd title (Magical Girls) [DL].zip": { 12 | Released: "C99", 13 | Circle: "doujin circle", 14 | Artists: []string{"some artist"}, 15 | Title: "very lewd title", 16 | Series: "Magical Girls", 17 | Language: "", 18 | }, 19 | "(C99) [doujin circle] very lewd title (Magical Girls) [DL].zip": { 20 | Released: "C99", 21 | Circle: "doujin circle", 22 | Artists: []string{""}, 23 | Title: "very lewd title", 24 | Series: "Magical Girls", 25 | Language: "", 26 | }, 27 | "[doujin circle] very lewd title (Magical Girls) [DL].zip": { 28 | Released: "", 29 | Circle: "doujin circle", 30 | Artists: []string{""}, 31 | Title: "very lewd title", 32 | Series: "Magical Girls", 33 | Language: "", 34 | }, 35 | "(C99) [doujin circle] very lewd title [DL].zip": { 36 | Released: "C99", 37 | Circle: "doujin circle", 38 | Artists: []string{""}, 39 | Title: "very lewd title", 40 | Series: "", 41 | Language: "", 42 | }, 43 | } 44 | 45 | for title, want := range wantMap { 46 | got := ParseTitle(title) 47 | if got.Released != want.Released && 48 | got.Circle != want.Circle && 49 | got.Title != want.Title && 50 | got.Series != want.Series && 51 | got.Language != want.Language { 52 | t.Errorf("Parsed title (%s) didn't match the expected result", title) 53 | } 54 | for i, artist := range got.Artists { 55 | if artist != want.Artists[i] { 56 | t.Errorf("Parsed title's (%s) artists didn't match the expected result", title) 57 | } 58 | } 59 | } 60 | } 61 | 62 | func TestParseX(t *testing.T) { 63 | json, err := utils.ReadJSON("../../testdata/x.json") 64 | if err != nil { 65 | t.Error("Reading x.json failed") 66 | return 67 | } 68 | 69 | exhGallery, err := unmarshalExhJSON(json) 70 | if err != nil { 71 | t.Error("Error unmarshalling JSON data:", err) 72 | return 73 | } 74 | 75 | archivePath := "(C99) [同人サークル (とあるアーティスト)] とてもエッチなタイトル (魔法少女) [DL版].zip" 76 | gotGallery, gotTags, gotReference := convertExh(exhGallery, archivePath, "info.json", true) 77 | 78 | if gotGallery.Title != "(C99) [doujin circle (some artist)] very lewd title (Magical Girls) [DL]" || 79 | *gotGallery.TitleNative != "(C99) [同人サークル (とあるアーティスト)] とてもエッチなタイトル (魔法少女) [DL版]" || 80 | *gotGallery.Category != "doujinshi" || 81 | *gotGallery.Language != "Japanese" || 82 | *gotGallery.Translated != false || 83 | *gotGallery.ImageCount != int32(30) || 84 | *gotGallery.ArchiveSize != int32(11639011) || 85 | gotGallery.ArchivePath != archivePath { 86 | t.Error("parsed gallery didn't match the expected result") 87 | } 88 | 89 | wantTags := map[string]string{} 90 | wantTags["Magical Girls"] = "parody" 91 | wantTags["swimsuit"] = "female" 92 | wantTags["yuri"] = "female" 93 | 94 | for _, gotTag := range gotTags { 95 | 96 | if wantTags[gotTag.Name] != gotTag.Namespace { 97 | t.Error("parsed tags didn't match expected results: ", wantTags[gotTag.Namespace], " - ", gotTag.Name) 98 | } 99 | } 100 | 101 | if *gotReference.MetaPath != "info.json" || *gotReference.ExhGid != int32(1) || *gotReference.ExhToken != "abc" { 102 | t.Error("parsed reference info didn't match the expected result") 103 | } 104 | } 105 | 106 | func TestParseHath(t *testing.T) { 107 | filepath := "../../testdata/hath.txt" 108 | 109 | buf, err := os.ReadFile(filepath) 110 | if err != nil { 111 | t.Error("Error reading hath.txt:", err) 112 | return 113 | } 114 | 115 | gotGallery, gotTags, gotReference, err := ParseHath(filepath, buf, false) 116 | if err != nil { 117 | t.Error("Error parsing hath.txt:", err) 118 | return 119 | } 120 | 121 | if *gotGallery.TitleNative != "(C88) [hサークル] とてもエッチなタイトル (魔法少女)" { 122 | t.Error("parsed gallery didn't match the expected result") 123 | } 124 | 125 | wantTags := map[string]string{} 126 | wantTags["mahou shoujo"] = "parody" 127 | wantTags["hcircle"] = "group" 128 | wantTags["group"] = "female" 129 | wantTags["thigh high boots"] = "female" 130 | wantTags["artbook"] = "other" 131 | 132 | for _, gotTag := range gotTags { 133 | if wantTags[gotTag.Name] != gotTag.Namespace { 134 | t.Error("parsed tags didn't match expected results: ", wantTags[gotTag.Namespace], " - ", gotTag.Name) 135 | } 136 | } 137 | 138 | if gotReference.GalleryUUID != "" || 139 | *gotReference.MetaPath != filepath || 140 | gotReference.MetaInternal != false || 141 | gotReference.MetaTitleHash != nil || 142 | gotReference.MetaMatch != nil || 143 | gotReference.ExhToken != nil || 144 | gotReference.ExhGid != nil || 145 | gotReference.AnilistID != nil || 146 | gotReference.Urls != nil { 147 | t.Error("parsed reference didn't match the expected result") 148 | } 149 | 150 | } 151 | 152 | func TestParseEHDL(t *testing.T) { 153 | filepath := "../../testdata/ehdl.txt" 154 | 155 | buf, err := os.ReadFile(filepath) 156 | if err != nil { 157 | t.Error("Error reading ehdl.txt:", err) 158 | return 159 | } 160 | 161 | gotGallery, gotTags, gotReference, err := ParseEHDL(filepath, buf, false) 162 | if err != nil { 163 | t.Error("Error parsing ehdl.txt:", err) 164 | return 165 | } 166 | 167 | if gotGallery.Title != "[CRAZY CIRCLE (Hana)] Oppai Oppai Oppai" || 168 | *gotGallery.TitleNative != "[CRAZY CIRCLE (はな)] おっぱいおっぱいおっぱい" || 169 | *gotGallery.Category != "doujinshi" || 170 | *gotGallery.Language != "Japanese" || 171 | *gotGallery.ImageCount != int32(12) || 172 | *gotGallery.ArchiveSize != int32(69690002) { 173 | t.Error("parsed gallery didn't match the expected result") 174 | } 175 | 176 | wantTags := map[string]string{} 177 | wantTags["crazy circle"] = "group" 178 | wantTags["artist"] = "hana" 179 | wantTags["group"] = "female" 180 | wantTags["fft threesome"] = "female" 181 | wantTags["stockings"] = "female" 182 | 183 | for _, gotTag := range gotTags { 184 | if wantTags[gotTag.Name] != gotTag.Namespace { 185 | t.Error("parsed tags didn't match expected results: ", wantTags[gotTag.Namespace], " - ", gotTag.Name) 186 | } 187 | } 188 | 189 | if gotReference.GalleryUUID != "" || 190 | *gotReference.MetaPath != filepath || 191 | gotReference.MetaInternal != false || 192 | gotReference.MetaTitleHash != nil || 193 | gotReference.MetaMatch != nil || 194 | gotReference.ExhToken != nil || 195 | gotReference.ExhGid != nil || 196 | gotReference.AnilistID != nil || 197 | gotReference.Urls != nil { 198 | t.Error("parsed reference didn't match the expected result") 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /pkg/metadata/title.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | import ( 4 | "github.com/Mangatsu/server/pkg/constants" 5 | "path" 6 | "path/filepath" 7 | "regexp" 8 | "strconv" 9 | "strings" 10 | 11 | "github.com/Mangatsu/server/internal/config" 12 | "github.com/Mangatsu/server/pkg/db" 13 | "github.com/Mangatsu/server/pkg/log" 14 | "github.com/Mangatsu/server/pkg/types/sqlite/model" 15 | "go.uber.org/zap" 16 | ) 17 | 18 | var nameRegex = regexp.MustCompile( 19 | "(?i)(?:\\(([^([]+)\\))?\\s*(?:\\[([^()[\\]]+)(?:\\(([^()[\\]]+)\\))?])?\\s*([^([]+)\\s*(?:\\(([^([)]+)\\))?\\s*(?:\\[(?:DL版?|digital)])?\\s*(?:\\[([^\\]]+)])?", 20 | ) 21 | 22 | type TitleMeta struct { 23 | Released string 24 | Circle string 25 | Artists []string 26 | Title string 27 | Series string 28 | Language string 29 | } 30 | 31 | // ParseTitles parses all filenames and titles of the saved galleries in db. 32 | // tryNative tries to preserve the native language (usually Japanese) text. 33 | // overwrite writes over the previous values. 34 | func ParseTitles(tryNative bool, overwrite bool) { 35 | libraries, err := db.GetLibraries() 36 | if err != nil { 37 | log.Z.Error("libraries could not be retrieved to parse titles", zap.String("err", err.Error())) 38 | return 39 | } 40 | 41 | for _, library := range libraries { 42 | for _, gallery := range library.Galleries { 43 | if db.TitleHashMatch(gallery.UUID) { 44 | continue 45 | } 46 | 47 | _, currentTags, err := db.GetTags(gallery.UUID, false) 48 | if err != nil { 49 | log.Z.Error("tags could not be retrieved when parsing titles", zap.String("err", err.Error())) 50 | continue 51 | } 52 | 53 | currentReference, err := db.GetReference(gallery.UUID) 54 | if err != nil { 55 | log.Z.Error("reference could not be retrieved when parsing titles", zap.String("err", err.Error())) 56 | continue 57 | } 58 | 59 | hasTitleTranslated := gallery.TitleTranslated != nil 60 | hasRelease := gallery.Released != nil 61 | hasSeries := gallery.Series != nil 62 | hasLanguage := gallery.Language != nil 63 | hasCircle := containsTag(currentTags, "circle", nil) 64 | hasArtists := containsTag(currentTags, "artist", nil) 65 | 66 | if !overwrite && hasRelease && hasSeries && hasLanguage && hasCircle && hasArtists { 67 | continue 68 | } 69 | 70 | title := gallery.Title 71 | titleNative := gallery.TitleNative 72 | 73 | filename := filepath.Base(gallery.ArchivePath) 74 | n := strings.LastIndex(filename, path.Ext(filename)) 75 | filename = filename[:n] 76 | 77 | titleMeta := ParseTitle(title) 78 | 79 | if tryNative && titleMeta == nil { 80 | titleMeta = ParseTitle(*titleNative) 81 | } 82 | 83 | if titleMeta == nil { 84 | titleMeta = ParseTitle(filename) 85 | } 86 | 87 | if titleMeta != nil { 88 | if titleMeta.Title != "" && (!hasTitleTranslated || overwrite) { 89 | if gallery.Translated != nil && *gallery.Translated { 90 | gallery.TitleTranslated = &titleMeta.Title 91 | } else { 92 | gallery.TitleNative = &titleMeta.Title 93 | } 94 | } 95 | if titleMeta.Released != "" && (!hasRelease || overwrite) { 96 | gallery.Released = &titleMeta.Released 97 | } 98 | if len(titleMeta.Artists) != 0 && titleMeta.Circle != "" && (!hasCircle || overwrite) { 99 | if !containsTag(currentTags, "circle", &titleMeta.Circle) { 100 | currentTags = append(currentTags, model.Tag{ 101 | Namespace: "circle", 102 | Name: titleMeta.Circle, 103 | }) 104 | } 105 | } 106 | if len(titleMeta.Artists) != 0 && (!hasArtists || overwrite) { 107 | if titleMeta.Circle != "" && len(titleMeta.Artists) == 1 { 108 | if !containsTag(currentTags, "circle", &titleMeta.Artists[0]) { 109 | currentTags = append(currentTags, model.Tag{ 110 | Namespace: "circle", 111 | Name: titleMeta.Artists[0], 112 | }) 113 | } 114 | } else { 115 | for _, artist := range titleMeta.Artists { 116 | if !containsTag(currentTags, "circle", &artist) { 117 | currentTags = append(currentTags, model.Tag{ 118 | Namespace: "artist", 119 | Name: artist, 120 | }) 121 | } 122 | } 123 | } 124 | } 125 | 126 | // If structured, no need to set the series again. 127 | if library.Layout != config.Structured && titleMeta.Series != "" && (!hasSeries || overwrite) { 128 | gallery.Series = &titleMeta.Series 129 | } 130 | 131 | // Set as language if it's not already set and is found in the list predefined of languages. 132 | if titleMeta.Language != "" && (!hasLanguage || overwrite) { 133 | if constants.Languages[strings.ToLower(titleMeta.Language)] { 134 | gallery.Language = &titleMeta.Language 135 | } else if match, err := regexp.MatchString(`\d+`, titleMeta.Language); err == nil && match { 136 | exhGid, err := strconv.ParseInt(titleMeta.Language, 10, 32) 137 | if err == nil { 138 | exhGidInt32 := int32(exhGid) 139 | currentReference.ExhGid = &exhGidInt32 140 | } 141 | } 142 | } 143 | } 144 | 145 | // If the gallery is stored in a structured dir layout with no category assigned, assume it's a manga. 146 | if library.Layout == config.Structured && (gallery.Category == nil || *gallery.Category == "") { 147 | manga := "manga" 148 | gallery.Category = &manga 149 | } 150 | 151 | if err = db.UpdateGallery(gallery, currentTags, currentReference, true); err != nil { 152 | log.Z.Error("failed to update gallery based on its title", 153 | zap.String("gallery", gallery.UUID), 154 | zap.String("err", err.Error())) 155 | } 156 | log.Z.Info("metadata parsed based from the title", 157 | zap.String("uuid", gallery.UUID), 158 | zap.String("title", gallery.Title)) 159 | } 160 | } 161 | } 162 | 163 | // ParseTitle parses the filename or title following the standard: 164 | // (Release) [Circle (Artist)] Title (Series) [ Language] or (Release) [Artist] Title (Series) [ Language] 165 | func ParseTitle(title string) *TitleMeta { 166 | match := nameRegex.FindStringSubmatch(title) 167 | var artists []string 168 | if match[3] != "" { 169 | if strings.Contains(match[3], ", ") { 170 | artists = strings.Split(strings.TrimSpace(match[3]), ", ") 171 | } else if strings.Contains(match[3], "、") { 172 | artists = strings.Split(strings.TrimSpace(match[3]), "、") 173 | } else { 174 | artists = append(artists, strings.TrimSpace(match[3])) 175 | } 176 | } 177 | 178 | titleMeta := TitleMeta{ 179 | Released: strings.TrimSpace(match[1]), 180 | Circle: strings.TrimSpace(match[2]), 181 | Artists: strings.Split(strings.TrimSpace(match[3]), ", "), 182 | Title: strings.TrimSpace(match[4]), 183 | Series: strings.TrimSpace(match[5]), 184 | Language: strings.TrimSpace(match[6]), 185 | } 186 | 187 | if titleMeta.Released == "" && titleMeta.Circle == "" && len(titleMeta.Artists) == 0 && titleMeta.Title == "" && titleMeta.Series == "" && titleMeta.Language == "" { 188 | return nil 189 | } 190 | 191 | return &titleMeta 192 | } 193 | 194 | func containsTag(tags []model.Tag, namespace string, name *string) bool { 195 | for _, tag := range tags { 196 | if tag.Namespace == namespace && (name == nil || tag.Name == *name) { 197 | return true 198 | } 199 | } 200 | return false 201 | } 202 | -------------------------------------------------------------------------------- /pkg/metadata/x.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/Mangatsu/server/internal/config" 6 | "github.com/Mangatsu/server/pkg/constants" 7 | "github.com/Mangatsu/server/pkg/db" 8 | "github.com/Mangatsu/server/pkg/log" 9 | "github.com/Mangatsu/server/pkg/types/sqlite/model" 10 | "github.com/Mangatsu/server/pkg/utils" 11 | "go.uber.org/zap" 12 | "os" 13 | "path" 14 | "path/filepath" 15 | "regexp" 16 | "strings" 17 | ) 18 | 19 | type Tags map[string][]string 20 | 21 | type XMetadata struct { 22 | GalleryInfo struct { 23 | Title *string `json:"title"` 24 | TitleOriginal *string `json:"title_original"` 25 | Link *string `json:"link"` 26 | Category *string `json:"category"` 27 | Tags Tags `json:"tags"` 28 | Language *string `json:"language"` 29 | Translated *bool `json:"translated"` 30 | UploadDate *[]int `json:"upload_date"` 31 | Source *struct { 32 | Site *string `json:"site"` 33 | Gid *int32 `json:"gid"` 34 | Token *string `json:"token"` 35 | } `json:"source"` 36 | } `json:"gallery_info"` 37 | GalleryInfoFull *struct { 38 | Gallery struct { 39 | Gid *int32 `json:"gid"` 40 | Token *string `json:"token"` 41 | } `json:"gallery"` 42 | Title *string `json:"title"` 43 | TitleOriginal *string `json:"title_original"` 44 | DateUploaded *int64 `json:"date_uploaded"` 45 | Category *string `json:"category"` 46 | Uploader *string `json:"uploader"` 47 | ImageCount *int32 `json:"image_count"` 48 | ImagesResized *bool `json:"images_resized"` 49 | TotalFileSizeApprox *int32 `json:"total_file_size_approx"` 50 | Language *string `json:"language"` 51 | Translated *bool `json:"translated"` 52 | Tags Tags `json:"tags"` 53 | TagsHaveNamespace *bool `json:"tags_have_namespace"` 54 | Source *string `json:"source"` 55 | SourceSite *string `json:"source_site"` 56 | } `json:"gallery_info_full"` 57 | } 58 | 59 | var metaExtensions = regexp.MustCompile(`\.json$`) 60 | 61 | // unmarshalExhJSON parses ExH JSON bytes into XMetadata. 62 | func unmarshalExhJSON(byteValue []byte) (XMetadata, error) { 63 | var gallery XMetadata 64 | err := json.Unmarshal(byteValue, &gallery) 65 | if err != nil { 66 | log.Z.Error("failed to unmarshal x metadata", zap.String("err", err.Error())) 67 | return XMetadata{}, err 68 | } 69 | 70 | return gallery, err 71 | } 72 | 73 | // convertExh converts ExH model to gallery, tags and other models. 74 | func convertExh( 75 | exhGallery XMetadata, 76 | archivePath string, 77 | metaPath string, 78 | internal bool, 79 | ) (model.Gallery, []model.Tag, model.Reference) { 80 | title := archivePath 81 | if exhGallery.GalleryInfo.Title == nil { 82 | title = path.Base(archivePath) 83 | n := strings.LastIndex(title, path.Ext(title)) 84 | title = title[:n] 85 | } else { 86 | title = *exhGallery.GalleryInfo.Title 87 | } 88 | 89 | newGallery := model.Gallery{ 90 | Title: title, 91 | TitleNative: exhGallery.GalleryInfo.TitleOriginal, 92 | Category: exhGallery.GalleryInfo.Category, 93 | Language: exhGallery.GalleryInfo.Language, 94 | Translated: exhGallery.GalleryInfo.Translated, 95 | ImageCount: exhGallery.GalleryInfoFull.ImageCount, 96 | ArchiveSize: exhGallery.GalleryInfoFull.TotalFileSizeApprox, 97 | ArchivePath: archivePath, 98 | Nsfw: *exhGallery.GalleryInfo.Category != "non-h", 99 | } 100 | 101 | var tags []model.Tag 102 | for namespace, names := range exhGallery.GalleryInfo.Tags { 103 | for _, name := range names { 104 | tags = append(tags, model.Tag{Namespace: namespace, Name: name}) 105 | } 106 | } 107 | 108 | exh := model.Reference{ 109 | MetaPath: &metaPath, 110 | MetaInternal: internal, 111 | ExhGid: exhGallery.GalleryInfo.Source.Gid, 112 | ExhToken: exhGallery.GalleryInfo.Source.Token, 113 | Urls: nil, 114 | } 115 | 116 | return newGallery, tags, exh 117 | } 118 | 119 | type FuzzyResult struct { 120 | MetaTitleMatch bool 121 | Similarity float64 122 | MatchedArchivePath string 123 | RelativeMetaPath string 124 | } 125 | 126 | // fuzzyMatchExternalMeta tries to find the metadata file besides it (fuzzy match). 127 | func fuzzyMatchExternalMeta(archivePath string, libraryPath string, f os.DirEntry) (FuzzyResult, XMetadata) { 128 | fuzzyResult := FuzzyResult{ 129 | MetaTitleMatch: false, 130 | Similarity: 0.0, 131 | MatchedArchivePath: "", 132 | } 133 | 134 | if f.IsDir() || !metaExtensions.MatchString(f.Name()) { 135 | return fuzzyResult, XMetadata{} 136 | } 137 | 138 | archivePath = filepath.ToSlash(archivePath) 139 | onlyDir := filepath.Dir(archivePath) 140 | 141 | metaData, err := utils.ReadJSON(config.BuildPath(onlyDir, f.Name())) 142 | if err != nil { 143 | log.Z.Debug("failed to unmarshal x metadata", zap.String("err", err.Error())) 144 | return fuzzyResult, XMetadata{} 145 | } 146 | 147 | exhGallery, err := unmarshalExhJSON(metaData) 148 | if err != nil { 149 | log.Z.Debug("could not unmarshal exhGallery", zap.String("err", err.Error())) 150 | return fuzzyResult, XMetadata{} 151 | } 152 | 153 | relativeMetaPath := config.RelativePath(libraryPath, onlyDir+"/"+f.Name()) 154 | // Skip if the JSON metadata has already been used by another archive. 155 | if db.MetaPathFound(relativeMetaPath, libraryPath) { 156 | return FuzzyResult{}, XMetadata{} 157 | } 158 | 159 | fuzzyResult.RelativeMetaPath = relativeMetaPath 160 | relativeArchivePath := config.RelativePath(libraryPath, archivePath) 161 | metaNoExt := metaExtensions.ReplaceAllString(f.Name(), "") 162 | archiveNoExt := constants.ArchiveExtensions.ReplaceAllString(path.Base(relativeArchivePath), "") 163 | 164 | archiveSimilarity := utils.Similarity(archiveNoExt, metaNoExt) 165 | titleSimilarity := utils.Similarity(archiveNoExt, *exhGallery.GalleryInfo.Title) 166 | titleNativeSimilarity := utils.Similarity(archiveNoExt, *exhGallery.GalleryInfo.TitleOriginal) 167 | fuzzyResult.MetaTitleMatch = *exhGallery.GalleryInfo.Title == archiveNoExt || *exhGallery.GalleryInfo.TitleOriginal == archiveNoExt 168 | 169 | if fuzzyResult.MetaTitleMatch { 170 | fuzzyResult.MatchedArchivePath = relativeArchivePath 171 | return fuzzyResult, exhGallery 172 | } 173 | 174 | if archiveSimilarity > fuzzyResult.Similarity { 175 | fuzzyResult.MatchedArchivePath = relativeArchivePath 176 | fuzzyResult.Similarity = archiveSimilarity 177 | } 178 | 179 | if titleSimilarity > fuzzyResult.Similarity { 180 | fuzzyResult.MatchedArchivePath = relativeArchivePath 181 | fuzzyResult.Similarity = titleSimilarity 182 | } 183 | 184 | if titleNativeSimilarity > fuzzyResult.Similarity { 185 | fuzzyResult.MatchedArchivePath = relativeArchivePath 186 | fuzzyResult.Similarity = titleNativeSimilarity 187 | } 188 | 189 | return fuzzyResult, exhGallery 190 | } 191 | 192 | // ParseX parses x JSON file (x: https://github.com/dnsev-h/x). 193 | func ParseX(metaData []byte, metaPath string, archivePath string, internal bool) (model.Gallery, []model.Tag, model.Reference, error) { 194 | exhGallery, err := unmarshalExhJSON(metaData) 195 | if err != nil { 196 | return model.Gallery{}, nil, model.Reference{}, err 197 | } 198 | 199 | gallery, tags, reference := convertExh(exhGallery, archivePath, metaPath, internal) 200 | 201 | return gallery, tags, reference, nil 202 | } 203 | -------------------------------------------------------------------------------- /pkg/api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "github.com/Mangatsu/server/internal/config" 9 | "github.com/Mangatsu/server/pkg/db" 10 | "github.com/Mangatsu/server/pkg/log" 11 | "github.com/go-jet/jet/v2/qrm" 12 | "github.com/gorilla/mux" 13 | "github.com/rs/cors" 14 | "go.uber.org/zap" 15 | "net/http" 16 | "reflect" 17 | "time" 18 | ) 19 | 20 | // handleResult handles the result and returns if it was successful or not. 21 | // InternalServerError will be set if any error found. NotFound is set if the result is nil or empty. 22 | func handleResult(w http.ResponseWriter, result interface{}, err error, many bool, endpoint string) bool { 23 | resultType := reflect.TypeOf(result) 24 | if err != nil { 25 | if errors.Is(err, sql.ErrNoRows) || errors.Is(err, qrm.ErrNoRows) { 26 | errorHandler(w, http.StatusNotFound, "", endpoint) 27 | return true 28 | } else { 29 | errorHandler(w, http.StatusInternalServerError, err.Error(), endpoint) 30 | return true 31 | } 32 | } 33 | if !many { 34 | if result == nil { 35 | errorHandler(w, http.StatusNotFound, "", endpoint) 36 | return true 37 | } 38 | if resultType.Kind() == reflect.Slice { 39 | if resultType.Kind() == reflect.Ptr { 40 | result = reflect.ValueOf(result).Elem().Interface() 41 | } 42 | list := reflect.ValueOf(result) 43 | if list.Len() == 0 { 44 | errorHandler(w, http.StatusNotFound, "", endpoint) 45 | return true 46 | } 47 | } 48 | } 49 | return false 50 | } 51 | 52 | func resultToJSON(w http.ResponseWriter, result interface{}, endpoint string) { 53 | w.Header().Set("Content-Type", "application/json;charset=UTF-8") 54 | if err := json.NewEncoder(w).Encode(result); err != nil { 55 | errorHandler(w, http.StatusInternalServerError, err.Error(), endpoint) 56 | } 57 | } 58 | 59 | func returnInfo(w http.ResponseWriter, r *http.Request) { 60 | resultToJSON(w, struct { 61 | APIVersion int 62 | ServerVersion string 63 | Visibility config.Visibility 64 | Registrations bool 65 | }{ 66 | APIVersion: 1, 67 | ServerVersion: "0.8.1", 68 | Visibility: config.Options.Visibility, 69 | Registrations: config.Options.Registrations, 70 | }, r.URL.Path) 71 | } 72 | 73 | // Returns statistics as JSON. 74 | func returnStatistics(w http.ResponseWriter, r *http.Request) { 75 | if access, _ := hasAccess(w, r, db.NoRole); !access { 76 | return 77 | } 78 | 79 | fmt.Fprintf(w, `{ "Message": "statistics not implemented" }`) 80 | } 81 | 82 | // Returns the root path as JSON. 83 | func returnRoot(w http.ResponseWriter, _ *http.Request) { 84 | fmt.Fprintf(w, `{ "Message": "Mangatsu API available at /api" }`) 85 | } 86 | 87 | // Handles errors. Argument msg is only used with 400 and 500. 88 | func errorHandler(w http.ResponseWriter, status int, msg string, endpoint string) { 89 | switch status { 90 | case http.StatusNotFound: 91 | w.WriteHeader(status) 92 | fmt.Fprintf(w, `{ "Code": %d, "Message": "not found" }`, status) 93 | case http.StatusBadRequest: 94 | w.WriteHeader(status) 95 | fmt.Fprintf(w, `{ "Code": %d, "Message": "%s" }`, status, msg) 96 | case http.StatusForbidden: 97 | w.WriteHeader(status) 98 | fmt.Fprintf(w, `{ "Code": %d, "Message": "forbidden" }`, status) 99 | case http.StatusUnauthorized: 100 | w.WriteHeader(status) 101 | fmt.Fprintf(w, `{ "Code": %d, "Message": "unauthorized" }`, status) 102 | case http.StatusGone: 103 | w.WriteHeader(status) 104 | fmt.Fprintf(w, `{ "Code": %d, "Message": "gone" }`, status) 105 | case http.StatusConflict: 106 | w.WriteHeader(status) 107 | fmt.Fprintf(w, `{ "Code": %d, "Message": "%s" }`, status, msg) 108 | default: 109 | w.WriteHeader(http.StatusInternalServerError) 110 | fmt.Fprintf(w, `{ "status": %d, "Message": "internal server error" }`, http.StatusInternalServerError) 111 | log.Z.Error(msg, zap.Int("status", status)) 112 | return 113 | } 114 | 115 | log.Z.Debug( 116 | "api request", 117 | zap.String("endpoint", endpoint), 118 | zap.Int("status", status), 119 | zap.String("msg", msg), 120 | ) 121 | } 122 | 123 | // Handles HTTP(S) requests. 124 | func handleRequests() { 125 | baseURL := "/api/v1" 126 | uuidRegex := "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}" 127 | r := mux.NewRouter().StrictSlash(true) 128 | 129 | r.HandleFunc("/", returnRoot).Methods("GET") 130 | r.HandleFunc("/api", returnInfo).Methods("GET") 131 | r.HandleFunc(baseURL+"/statistics", returnStatistics).Methods("GET") 132 | 133 | r.HandleFunc(baseURL+"/register", register).Methods("POST") 134 | r.HandleFunc(baseURL+"/login", login).Methods("POST") 135 | r.HandleFunc(baseURL+"/logout", logout).Methods("POST") 136 | 137 | r.HandleFunc(baseURL+"/users", returnUsers).Methods("GET") 138 | r.HandleFunc(baseURL+"/users/{uuid:"+uuidRegex+"}", updateUser).Methods("PUT") 139 | r.HandleFunc(baseURL+"/users/{uuid:"+uuidRegex+"}", deleteUser).Methods("DELETE") 140 | r.HandleFunc(baseURL+"/users/me/favorites", returnFavoriteGroups).Methods("GET") 141 | r.HandleFunc(baseURL+"/users/me/sessions", returnSessions).Methods("GET") 142 | r.HandleFunc(baseURL+"/users/me/sessions", deleteSession).Methods("DELETE") 143 | 144 | r.HandleFunc(baseURL+"/status", returnProcessingStatus).Methods("GET") 145 | r.HandleFunc(baseURL+"/scan", scanLibraries).Methods("GET") 146 | r.HandleFunc(baseURL+"/thumbnails", generateThumbnails).Methods("GET") 147 | r.HandleFunc(baseURL+"/meta", findMetadata).Methods("GET") 148 | 149 | r.HandleFunc(baseURL+"/categories", returnCategories).Methods("GET") 150 | r.HandleFunc(baseURL+"/series", returnSeries).Methods("GET") 151 | r.HandleFunc(baseURL+"/tags", returnTags).Methods("GET") 152 | 153 | r.HandleFunc(baseURL+"/galleries", returnGalleries).Methods("GET") 154 | r.HandleFunc(baseURL+"/galleries/count", returnGalleryCount).Methods("GET") 155 | r.HandleFunc(baseURL+"/galleries/random", returnRandomGallery).Methods("GET") 156 | r.HandleFunc(baseURL+"/galleries/{uuid:"+uuidRegex+"}", updateGallery).Methods("PUT") 157 | r.HandleFunc(baseURL+"/galleries/{uuid:"+uuidRegex+"}", returnGallery).Methods("GET") 158 | r.HandleFunc(baseURL+"/galleries/{uuid:"+uuidRegex+"}/progress/{progress:[0-9]+}", updateProgress).Methods("PATCH") 159 | r.HandleFunc(baseURL+"/galleries/{uuid:"+uuidRegex+"}/favorite/{name}", setFavorite).Methods("PATCH") 160 | r.HandleFunc(baseURL+"/galleries/{uuid:"+uuidRegex+"}/favorite", setFavorite).Methods("PATCH") 161 | 162 | if config.Options.Cache.WebServer { 163 | r.PathPrefix("/cache/").Handler(http.StripPrefix("/cache/", http.FileServer(http.Dir(config.BuildCachePath())))) 164 | } 165 | 166 | // General 404 167 | r.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 168 | errorHandler(w, http.StatusNotFound, "", r.RequestURI) 169 | }) 170 | 171 | handler := cors.New(cors.Options{ 172 | AllowOriginFunc: func(origin string) bool { return originAllowed(origin) }, 173 | AllowedMethods: []string{ 174 | http.MethodOptions, http.MethodGet, http.MethodPost, http.MethodDelete, http.MethodPut, http.MethodPatch, 175 | }, 176 | AllowedHeaders: []string{ 177 | "Accept", "Content-Type", "Content-Length", "Accept-Encoding", "X-CSRF-Token", "Authorization", 178 | "Access-Control-Allow-Headers", "Origin", "X-Requested-With", "Access-Control-Request-Method", 179 | "Access-Control-Request-Headers", 180 | }, 181 | AllowCredentials: true, 182 | AllowPrivateNetwork: true, 183 | }).Handler(r) 184 | 185 | fullAddress := config.Options.Hostname + ":" + config.Options.Port 186 | srv := &http.Server{ 187 | Handler: handler, 188 | Addr: fullAddress, 189 | WriteTimeout: 15 * time.Second, 190 | ReadTimeout: 15 * time.Second, 191 | } 192 | 193 | log.Z.Info("starting API on: " + fullAddress) 194 | log.Z.Warn(srv.ListenAndServe().Error()) 195 | } 196 | 197 | // LaunchAPI starts handling API requests. 198 | func LaunchAPI() { 199 | handleRequests() 200 | } 201 | -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | "strconv" 8 | "time" 9 | 10 | "github.com/Mangatsu/server/pkg/log" 11 | "github.com/joho/godotenv" 12 | "go.uber.org/zap" 13 | "go.uber.org/zap/zapcore" 14 | ) 15 | 16 | type Layout string 17 | 18 | const ( 19 | Freeform Layout = "freeform" 20 | Structured = "structured" 21 | ) 22 | 23 | type Visibility string 24 | 25 | const ( 26 | Private Visibility = "private" 27 | Restricted = "restricted" 28 | Public = "public" 29 | ) 30 | 31 | type ImageFormat string 32 | 33 | const ( 34 | WebP ImageFormat = "webp" 35 | AVIF = "avif" 36 | ) 37 | 38 | type CacheOptions struct { 39 | WebServer bool 40 | TTL time.Duration 41 | Size uint64 42 | } 43 | 44 | type GalleryOptions struct { 45 | ThumbnailFormat ImageFormat 46 | FuzzySearchSimilarity float64 47 | LTR bool 48 | } 49 | 50 | type OptionsModel struct { 51 | Environment log.Environment 52 | Domain string 53 | Hostname string 54 | Port string 55 | Secure bool 56 | SameSiteMode http.SameSite 57 | StrictACAO bool 58 | Registrations bool 59 | Visibility Visibility 60 | DB DBOptions 61 | Cache CacheOptions 62 | GalleryOptions GalleryOptions 63 | } 64 | 65 | type CredentialsModel struct { 66 | JWTSecret string 67 | Passphrase string 68 | } 69 | 70 | var AppEnvironment log.Environment 71 | var LogLevel zapcore.Level 72 | 73 | // Options stores the global configuration for the server 74 | var Options *OptionsModel 75 | 76 | // Credentials stores the JWT secret, and optionally a passphrase and credentials for the db 77 | var Credentials *CredentialsModel // TODO: Encrypt in memory? 78 | 79 | // LoadEnv loads the environment variables. 80 | func LoadEnv() { 81 | var err = godotenv.Load() 82 | if err != nil { 83 | fmt.Println("No .env file or environmentals found.") 84 | } 85 | 86 | loadEnvironment() 87 | loadLogLevel() 88 | } 89 | 90 | // SetEnv sets the environment variables into Options and Credentials 91 | func SetEnv() { 92 | Options = &OptionsModel{ 93 | Domain: domain(), 94 | Hostname: hostname(), 95 | Port: port(), 96 | Secure: secure(), 97 | SameSiteMode: sameSiteMode(), 98 | StrictACAO: acao(), 99 | Registrations: registrationsEnabled(), 100 | Visibility: currentVisibility(), 101 | DB: DBOptions{ 102 | Name: dbName(), 103 | Migrations: dbMigrationsEnabled(), 104 | }, 105 | Cache: CacheOptions{ 106 | WebServer: cacheServerEnabled(), 107 | TTL: cacheTTL(), 108 | Size: cacheSize(), 109 | }, 110 | GalleryOptions: GalleryOptions{ 111 | ThumbnailFormat: thumbnailFormat(), 112 | FuzzySearchSimilarity: fuzzySearchSimilarity(), 113 | LTR: defaultLTR(), 114 | }, 115 | } 116 | 117 | Credentials = &CredentialsModel{ 118 | JWTSecret: jwtSecret(), 119 | Passphrase: restrictedPassphrase(), 120 | } 121 | } 122 | 123 | func GetInitialAdmin() (string, string) { 124 | username := os.Getenv("MTSU_INITIAL_ADMIN_NAME") 125 | password := os.Getenv("MTSU_INITIAL_ADMIN_PW") 126 | if username == "" { 127 | username = "admin" 128 | } 129 | if password == "" { 130 | password = "admin321" 131 | } 132 | return username, password 133 | } 134 | 135 | func loadEnvironment() { 136 | value := os.Getenv("MTSU_ENV") 137 | if value == "production" { 138 | AppEnvironment = log.Production 139 | return 140 | } 141 | AppEnvironment = log.Development 142 | } 143 | 144 | func loadLogLevel() { 145 | value := os.Getenv("MTSU_LOG_LEVEL") 146 | switch value { 147 | case "debug": 148 | LogLevel = zap.DebugLevel 149 | case "warn": 150 | LogLevel = zap.WarnLevel 151 | case "error": 152 | LogLevel = zap.ErrorLevel 153 | default: 154 | LogLevel = zap.InfoLevel 155 | } 156 | } 157 | 158 | func domain() string { 159 | return os.Getenv("MTSU_DOMAIN") 160 | } 161 | 162 | func hostname() string { 163 | value := os.Getenv("MTSU_HOSTNAME") 164 | if value == "" { 165 | return "localhost" 166 | } 167 | return value 168 | } 169 | 170 | func port() string { 171 | value := os.Getenv("MTSU_PORT") 172 | if value == "" { 173 | return "5050" 174 | } 175 | return value 176 | } 177 | 178 | func secure() bool { 179 | return os.Getenv("MTSU_SECURE") == "true" 180 | } 181 | 182 | func sameSiteMode() http.SameSite { 183 | if secure() { 184 | return http.SameSiteNoneMode 185 | } 186 | return http.SameSiteLaxMode 187 | } 188 | 189 | func acao() bool { 190 | return os.Getenv("MTSU_STRICT_ACAO") == "true" 191 | } 192 | 193 | func cacheServerEnabled() bool { 194 | value := os.Getenv("MTSU_DISABLE_CACHE_SERVER") 195 | if value == "true" { 196 | return false 197 | } 198 | return true 199 | } 200 | 201 | func registrationsEnabled() bool { 202 | value := os.Getenv("MTSU_REGISTRATIONS") 203 | if value == "true" { 204 | return true 205 | } 206 | return false 207 | } 208 | 209 | func currentVisibility() Visibility { 210 | value := os.Getenv("MTSU_VISIBILITY") 211 | switch value { 212 | case "public": 213 | return Public 214 | case "restricted": 215 | return Restricted 216 | default: 217 | return Private 218 | } 219 | } 220 | 221 | func restrictedPassphrase() string { 222 | value := os.Getenv("MTSU_RESTRICTED_PASSPHRASE") 223 | if value == "" { 224 | if currentVisibility() == Restricted { 225 | log.Z.Warn("MTSU_RESTRICTED_PASSPHRASE is not set. Defaulting to 's3cr3t'.") 226 | } 227 | return "s3cr3t" 228 | } 229 | return value 230 | } 231 | 232 | func jwtSecret() string { 233 | value := os.Getenv("MTSU_JWT_SECRET") 234 | if value == "" { 235 | log.Z.Error("MTSU_JWT_SECRET is not set. An unsecure secret will be used instead. DO NOT USE IN PRODUCTION.") 236 | return "9Wag7sMvKl3aF6K5lwIg6TI42ia2f6BstZAVrdJIq8Mp38lnl7UzQMC1qjKyZCBzHFGbbqsA0gKcHqDuyXQAhWoJ0lcx4K5q" 237 | } 238 | return value 239 | } 240 | 241 | func cacheTTL() time.Duration { 242 | defaultDuration := time.Hour * 336 243 | minDuration := time.Minute * 15 244 | 245 | value := os.Getenv("MTSU_CACHE_TTL") 246 | if value == "" { 247 | return defaultDuration 248 | } 249 | 250 | duration, err := time.ParseDuration(value) 251 | if err != nil { 252 | log.Z.Error(value + " is not a valid TTL for MTSU_CACHE_TTL. Defaulting to 336h (14 days).") 253 | return defaultDuration 254 | } 255 | 256 | if duration < minDuration { 257 | log.Z.Info("Minimum TTL is 15 minutes. Defaulting to 15 minutes.") 258 | return minDuration 259 | } 260 | 261 | return duration 262 | } 263 | 264 | func cacheSize() uint64 { 265 | defaultSize := uint64(20000) 266 | minSize := uint64(100) 267 | 268 | value := os.Getenv("MTSU_CACHE_SIZE") 269 | if value == "" { 270 | return defaultSize 271 | } 272 | 273 | size, err := strconv.ParseUint(value, 10, 64) 274 | if err != nil { 275 | log.Z.Error(value + " is not a valid TTL for MTSU_CACHE_SIZE. Defaulting to 20 000 MiB.") 276 | return defaultSize 277 | } 278 | 279 | if size < minSize { 280 | log.Z.Warn("Minimum TTL is 100 MiB. Defaulting to 100 MiB.") 281 | return minSize 282 | } 283 | 284 | return size 285 | } 286 | 287 | func thumbnailFormat() ImageFormat { 288 | // TODO: Add support for AVIF 289 | //value := os.Getenv("MTSU_THUMBNAIL_FORMAT") 290 | //if value == "avif" { 291 | // return AVIF 292 | //} 293 | return WebP 294 | } 295 | 296 | func fuzzySearchSimilarity() float64 { 297 | value := os.Getenv("MTSU_FUZZY_SEARCH_SIMILARITY") 298 | if value == "" { 299 | return 0.7 300 | } 301 | 302 | similarity, err := strconv.ParseFloat(value, 64) 303 | if err != nil { 304 | log.Z.Error(value + " is not a valid similarity value for MTSU_FUZZY_SEARCH_SIMILARITY. Defaulting to 0.7.") 305 | return 0.7 306 | } 307 | if similarity < 0.1 { 308 | return 0.1 309 | } 310 | if similarity > 1 { 311 | return 1 312 | } 313 | return similarity 314 | } 315 | 316 | func defaultLTR() bool { 317 | value := os.Getenv("MTSU_LTR") 318 | if value == "false" { 319 | return false 320 | } 321 | return true 322 | } 323 | -------------------------------------------------------------------------------- /pkg/metadata/scan.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | import ( 4 | "errors" 5 | "io/fs" 6 | "math" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/Mangatsu/server/internal/config" 11 | "github.com/Mangatsu/server/pkg/cache" 12 | "github.com/Mangatsu/server/pkg/constants" 13 | "github.com/Mangatsu/server/pkg/db" 14 | "github.com/Mangatsu/server/pkg/library" 15 | "github.com/Mangatsu/server/pkg/log" 16 | "github.com/Mangatsu/server/pkg/types/sqlite/model" 17 | "github.com/Mangatsu/server/pkg/utils" 18 | "github.com/mholt/archiver/v4" 19 | "go.uber.org/zap" 20 | ) 21 | 22 | type MetaType string 23 | 24 | const ( 25 | XMeta MetaType = "xmeta" 26 | HathMeta = "hathmeta" 27 | EHDLMeta = "ehdlmeta" 28 | FuzzyMatch = "fuzzy" 29 | ) 30 | 31 | type NoMatchPaths struct { 32 | libraryPath string 33 | fullPath string 34 | } 35 | 36 | // matchInternalMeta reads the internal metadata (info.json, info.txt or galleryinfo.txt) from the given archive. 37 | func matchInternalMeta(metaTypes map[MetaType]bool, fullArchivePath string) ([]byte, string, MetaType) { 38 | filesystem, err := archiver.FileSystem(nil, fullArchivePath) 39 | if err != nil { 40 | log.Z.Error("could not open archive", 41 | zap.String("path", fullArchivePath), 42 | zap.String("err", err.Error())) 43 | return nil, "", "" 44 | } 45 | 46 | var content []byte 47 | var filename string 48 | var metaType MetaType = "" 49 | 50 | err = fs.WalkDir(filesystem, ".", func(s string, d fs.DirEntry, err error) error { 51 | if err != nil { 52 | return err 53 | } 54 | 55 | if d.Name() == "info.json" && metaTypes[XMeta] { 56 | metaType = XMeta 57 | } else if d.Name() == "info.txt" && metaTypes[EHDLMeta] { 58 | metaType = EHDLMeta 59 | } else if d.Name() == "galleryinfo.txt" && metaTypes[HathMeta] { 60 | metaType = HathMeta 61 | } 62 | 63 | if metaType != "" { 64 | filename = s 65 | content, err = library.ReadAll(filesystem, s) 66 | if err != nil { 67 | return err 68 | } 69 | return errors.New("terminate walk") 70 | } 71 | return nil 72 | }) 73 | 74 | return content, filename, metaType 75 | } 76 | 77 | // matchExternalMeta tries to find the metadata file besides it (exact match). 78 | func matchExternalMeta(metaTypes map[MetaType]bool, fullArchivePath string, libraryPath string) ([]byte, string) { 79 | if !metaTypes[XMeta] { 80 | return nil, "" 81 | } 82 | 83 | externalJSON := constants.ArchiveExtensions.ReplaceAllString(fullArchivePath, ".json") 84 | 85 | if !utils.PathExists(externalJSON) { 86 | return nil, "" 87 | } 88 | 89 | metaData, err := utils.ReadJSON(externalJSON) 90 | if err != nil { 91 | log.Z.Debug("could not read external metadata", 92 | zap.String("path", externalJSON), 93 | zap.String("err", err.Error())) 94 | return nil, "" 95 | } 96 | 97 | return metaData, config.RelativePath(libraryPath, externalJSON) 98 | } 99 | 100 | // ParseMetadata scans all libraries for metadata files (json, txt). 101 | func ParseMetadata(metaTypes map[MetaType]bool) { 102 | libraries, err := db.GetLibraries() 103 | if err != nil { 104 | log.Z.Error("libraries could not be retrieved to parse meta files: ", zap.String("err", err.Error())) 105 | return 106 | } 107 | 108 | var archivesWithNoMatch []NoMatchPaths 109 | 110 | for _, galleryLibrary := range libraries { 111 | for _, gallery := range galleryLibrary.Galleries { 112 | fullPath := config.BuildLibraryPath(galleryLibrary.Path, gallery.ArchivePath) 113 | 114 | var metaData []byte 115 | var metaPath string 116 | internalDataFound := false 117 | 118 | // X, Hath, EHDL 119 | metaData, metaPath, metaType := matchInternalMeta(metaTypes, fullPath) 120 | if metaData != nil { 121 | internalDataFound = true 122 | } 123 | 124 | // X 125 | if !internalDataFound { 126 | metaData, metaPath = matchExternalMeta(metaTypes, fullPath, galleryLibrary.Path) 127 | metaType = XMeta 128 | } 129 | 130 | if metaData == nil { 131 | if metaTypes[FuzzyMatch] { 132 | archivesWithNoMatch = append(archivesWithNoMatch, NoMatchPaths{libraryPath: galleryLibrary.Path, fullPath: fullPath}) 133 | } 134 | continue 135 | } 136 | 137 | var newGallery model.Gallery 138 | var tags []model.Tag 139 | var reference model.Reference 140 | 141 | switch metaType { 142 | case XMeta: 143 | if newGallery, tags, reference, err = ParseX(metaData, metaPath, gallery.ArchivePath, internalDataFound); err != nil { 144 | log.Z.Debug("could not parse X meta", 145 | zap.String("path", metaPath), 146 | zap.String("err", err.Error())) 147 | 148 | cache.ProcessingStatusCache.AddMetadataError(gallery.UUID, err.Error(), map[string]string{ 149 | "metaType": string(metaType), 150 | "metaPath": metaPath, 151 | }) 152 | continue 153 | } 154 | case EHDLMeta: 155 | if newGallery, tags, reference, err = ParseEHDL(metaPath, metaData, internalDataFound); err != nil { 156 | log.Z.Debug("could not parse EHDL meta", 157 | zap.String("path", metaPath), 158 | zap.String("err", err.Error())) 159 | 160 | cache.ProcessingStatusCache.AddMetadataError(gallery.UUID, err.Error(), map[string]string{ 161 | "metaType": string(metaType), 162 | "metaPath": metaPath, 163 | }) 164 | continue 165 | } 166 | case HathMeta: 167 | if newGallery, tags, reference, err = ParseHath(metaPath, metaData, internalDataFound); err != nil { 168 | log.Z.Debug("could not parse Hath meta", 169 | zap.String("path", metaPath), 170 | zap.String("err", err.Error())) 171 | 172 | cache.ProcessingStatusCache.AddMetadataError(gallery.UUID, err.Error(), map[string]string{ 173 | "metaType": string(metaType), 174 | "metaPath": metaPath, 175 | }) 176 | continue 177 | } 178 | } 179 | 180 | // Adds the UUID and archive path to the new gallery. 181 | newGallery.UUID = gallery.UUID 182 | newGallery.ArchivePath = gallery.ArchivePath 183 | 184 | err = db.UpdateGallery(newGallery, tags, reference, true) 185 | if err != nil { 186 | log.Z.Debug("could not tag gallery", 187 | zap.String("path", gallery.ArchivePath), 188 | zap.String("err", err.Error())) 189 | 190 | cache.ProcessingStatusCache.AddMetadataError(newGallery.UUID, err.Error(), map[string]string{ 191 | "path": gallery.ArchivePath, 192 | }) 193 | continue 194 | } 195 | 196 | log.Z.Info("metadata parsed", 197 | zap.String("metaType", string(metaType)), 198 | zap.String("uuid", gallery.UUID), 199 | zap.String("title", gallery.Title), 200 | zap.String("path", gallery.ArchivePath), 201 | zap.String("metaPath", metaPath), 202 | ) 203 | } 204 | } 205 | 206 | // Fuzzy parsing for all archives that didn't have an exact match. 207 | for _, noMatch := range archivesWithNoMatch { 208 | onlyDir := filepath.Dir(noMatch.fullPath) 209 | files, err := os.ReadDir(onlyDir) 210 | if err != nil { 211 | log.Z.Debug("could not gallery read dir while fuzzy matching", 212 | zap.String("path", onlyDir), 213 | zap.String("err", err.Error())) 214 | } 215 | 216 | for _, f := range files { 217 | r, exhGallery := fuzzyMatchExternalMeta(noMatch.fullPath, noMatch.libraryPath, f) 218 | 219 | if r.MatchedArchivePath != "" && r.MetaTitleMatch || r.Similarity > config.Options.GalleryOptions.FuzzySearchSimilarity { 220 | gallery, tags, reference := convertExh(exhGallery, r.MatchedArchivePath, r.RelativeMetaPath, false) 221 | 222 | if !r.MetaTitleMatch { 223 | permil := int32(math.Round(r.Similarity * 1000)) 224 | reference.MetaMatch = &permil 225 | } 226 | 227 | err = db.UpdateGallery(gallery, tags, reference, true) 228 | if err != nil { 229 | log.Z.Debug("could not tag gallery", 230 | zap.String("path", gallery.ArchivePath), 231 | zap.String("err", err.Error())) 232 | continue 233 | } 234 | 235 | if r.MetaTitleMatch { 236 | log.Z.Info("exact match based on meta titles", zap.String("path", r.MatchedArchivePath)) 237 | } else { 238 | log.Z.Info("fuzzy match", 239 | zap.Float64("similarity", r.Similarity), 240 | zap.String("path", r.MatchedArchivePath)) 241 | } 242 | } 243 | } 244 | } 245 | } 246 | -------------------------------------------------------------------------------- /pkg/db/validations.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "github.com/Mangatsu/server/pkg/log" 5 | "go.uber.org/zap" 6 | "reflect" 7 | "strings" 8 | "time" 9 | 10 | "github.com/Mangatsu/server/pkg/types/sqlite/model" 11 | . "github.com/Mangatsu/server/pkg/types/sqlite/table" 12 | . "github.com/go-jet/jet/v2/sqlite" 13 | ) 14 | 15 | // SanitizeString returns the pointer of the string with all leading and trailing white space removed. 16 | // If the string is empty, it returns nil. 17 | func SanitizeString(content *string) *string { 18 | if content == nil { 19 | return nil 20 | } 21 | trimmed := strings.TrimSpace(*content) 22 | if trimmed == "" { 23 | return nil 24 | } 25 | return &trimmed 26 | } 27 | 28 | // ValidateGallery returns updated and sanitized gallery model. For external API use. 29 | func ValidateGallery(gallery model.Gallery, newGallery model.Gallery, now time.Time) (model.Gallery, ColumnList) { 30 | // Title cannot be empty or nil 31 | title := strings.TrimSpace(newGallery.Title) 32 | if title == "" { 33 | title = gallery.Title 34 | } 35 | gallery.Title = title 36 | 37 | // Boolean fields 38 | gallery.Nsfw = newGallery.Nsfw 39 | gallery.Hidden = newGallery.Hidden 40 | gallery.Translated = newGallery.Translated 41 | 42 | // Nullable string fields 43 | gallery.TitleNative = SanitizeString(newGallery.TitleNative) 44 | gallery.TitleTranslated = SanitizeString(newGallery.TitleTranslated) 45 | gallery.Category = SanitizeString(newGallery.Category) 46 | gallery.Released = SanitizeString(newGallery.Released) 47 | gallery.Series = SanitizeString(newGallery.Series) 48 | gallery.Language = SanitizeString(newGallery.Language) 49 | 50 | gallery.UpdatedAt = now 51 | 52 | return gallery, ColumnList{ 53 | Gallery.Title, 54 | Gallery.TitleNative, 55 | Gallery.TitleTranslated, 56 | Gallery.Category, 57 | Gallery.Released, 58 | Gallery.Series, 59 | Gallery.Language, 60 | Gallery.Nsfw, 61 | Gallery.Hidden, 62 | Gallery.Translated, 63 | Gallery.UpdatedAt, 64 | } 65 | } 66 | 67 | // ValidateReference returns updated and sanitized gallery reference. For external API use. 68 | func ValidateReference(reference model.Reference) model.Reference { 69 | reference.Urls = SanitizeString(reference.Urls) 70 | reference.ExhToken = SanitizeString(reference.ExhToken) 71 | 72 | return reference 73 | } 74 | 75 | // ValidateGalleryInternal returns updated and sanitized gallery model and column list. 76 | // Non-empty and non-negative values are preferred. 77 | // For internal scanner use. 78 | func ValidateGalleryInternal(newGallery model.Gallery, now time.Time) (model.Gallery, ColumnList) { 79 | galleryModel := model.Gallery{} 80 | galleryUpdateColumnList := ColumnList{} 81 | 82 | // Title cannot be empty or nil 83 | if value := strings.TrimSpace(newGallery.Title); value != "" { 84 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.Title) 85 | galleryModel.Title = value 86 | } 87 | 88 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.Nsfw) 89 | galleryModel.Nsfw = newGallery.Nsfw 90 | 91 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.Hidden) 92 | galleryModel.Hidden = newGallery.Hidden 93 | 94 | if value := SanitizeString(newGallery.TitleNative); value != nil { 95 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.TitleNative) 96 | galleryModel.TitleNative = value 97 | } 98 | if value := SanitizeString(newGallery.TitleTranslated); value != nil { 99 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.TitleTranslated) 100 | galleryModel.TitleTranslated = value 101 | } 102 | if value := SanitizeString(newGallery.Category); value != nil { 103 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.Category) 104 | galleryModel.Category = value 105 | } 106 | if value := SanitizeString(newGallery.Released); value != nil { 107 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.Released) 108 | galleryModel.Released = value 109 | } 110 | if value := SanitizeString(newGallery.Series); value != nil { 111 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.Series) 112 | galleryModel.Series = value 113 | } 114 | if value := SanitizeString(newGallery.Language); value != nil { 115 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.Language) 116 | galleryModel.Language = value 117 | } 118 | if newGallery.Translated != nil { 119 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.Translated) 120 | galleryModel.Translated = newGallery.Translated 121 | } 122 | 123 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.Nsfw) 124 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.Hidden) 125 | 126 | if newGallery.ImageCount != nil && *newGallery.ImageCount >= 0 { 127 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.ImageCount) 128 | galleryModel.ImageCount = newGallery.ImageCount 129 | } 130 | if newGallery.ArchiveSize != nil && *newGallery.ArchiveSize >= 0 { 131 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.ArchiveSize) 132 | galleryModel.ArchiveSize = newGallery.ArchiveSize 133 | } 134 | if value := SanitizeString(newGallery.ArchiveHash); value != nil { 135 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.ArchiveHash) 136 | galleryModel.ArchiveHash = value 137 | } 138 | 139 | galleryUpdateColumnList = append(galleryUpdateColumnList, Gallery.UpdatedAt) 140 | galleryModel.UpdatedAt = now 141 | 142 | return galleryModel, galleryUpdateColumnList 143 | } 144 | 145 | // ValidateReferenceInternal returns updated and sanitized gallery reference. 146 | // Non-empty values are preferred. 147 | // For internal scanner use. 148 | func ValidateReferenceInternal(reference model.Reference) model.Reference { 149 | referenceModel := model.Reference{} 150 | 151 | referenceModel.MetaInternal = reference.MetaInternal 152 | 153 | if reference.MetaTitleHash != nil && *reference.MetaTitleHash != "" { 154 | referenceModel.MetaTitleHash = reference.MetaTitleHash 155 | } 156 | 157 | if value := SanitizeString(reference.MetaPath); value != nil { 158 | referenceModel.MetaPath = value 159 | } 160 | 161 | if reference.MetaMatch != nil { 162 | referenceModel.MetaPath = reference.MetaPath 163 | } 164 | 165 | if value := SanitizeString(reference.Urls); value != nil { 166 | referenceModel.Urls = value 167 | } 168 | 169 | if reference.ExhGid != nil { 170 | referenceModel.ExhGid = reference.ExhGid 171 | } 172 | 173 | if value := SanitizeString(reference.ExhToken); value != nil { 174 | referenceModel.ExhToken = value 175 | } 176 | 177 | if reference.AnilistID != nil && *reference.AnilistID > 0 { 178 | referenceModel.AnilistID = reference.AnilistID 179 | } 180 | 181 | return referenceModel 182 | } 183 | 184 | // ConstructExpressions constructs expressions from a reference model. 185 | func ConstructExpressions(model interface{}) ([]Expression, ColumnList, error) { 186 | v := reflect.ValueOf(model) 187 | typeOfS := v.Type() 188 | values := make([]Expression, 0) 189 | 190 | referenceColumnList := ColumnList{} 191 | 192 | for i := 0; i < v.NumField(); i++ { 193 | var value Expression 194 | 195 | field := v.Field(i) 196 | kind := field.Kind() 197 | if kind == reflect.Ptr || kind == reflect.Interface { 198 | if field.IsNil() { 199 | continue 200 | } 201 | field = field.Elem() 202 | kind = field.Kind() 203 | } 204 | 205 | switch kind { 206 | case reflect.String: 207 | if v.Field(i).IsZero() { 208 | continue 209 | } 210 | value = String(field.String()) 211 | case reflect.Bool: 212 | value = Bool(field.Bool()) 213 | case reflect.Int: 214 | value = Int(field.Int()) 215 | case reflect.Int32: 216 | value = Int32(int32(field.Int())) 217 | case reflect.Int64: 218 | value = Int64(field.Int()) 219 | default: 220 | log.Z.Error("unsupported reflect type", 221 | zap.String("type", v.Field(i).Kind().String()), 222 | zap.String("field", v.Type().Field(i).Name), 223 | ) 224 | continue 225 | } 226 | 227 | columName := ConstructColumn(typeOfS.Field(i).Name) 228 | if columName != nil { 229 | //log.Z.Debug("constructed column", 230 | // zap.String("column", columName.Name()), 231 | // zap.Any("value", value), 232 | //) 233 | 234 | referenceColumnList = append(referenceColumnList, columName) 235 | values = append(values, value) 236 | } 237 | } 238 | 239 | return values, referenceColumnList, nil 240 | } 241 | 242 | func ConstructColumn(columnName string) Column { 243 | switch columnName { 244 | case "GalleryUUID": 245 | return Reference.GalleryUUID 246 | case "MetaInternal": 247 | return Reference.MetaInternal 248 | case "MetaPath": 249 | return Reference.MetaPath 250 | case "MetaMatch": 251 | return Reference.MetaMatch 252 | case "Urls": 253 | return Reference.Urls 254 | case "ExhGid": 255 | return Reference.ExhGid 256 | case "ExhToken": 257 | return Reference.ExhToken 258 | case "AnilistID": 259 | return Reference.AnilistID 260 | case "MetaTitleHash": 261 | return Reference.MetaTitleHash 262 | default: 263 | log.Z.Error("unsupported column name", zap.String("column", columnName)) 264 | return nil 265 | } 266 | } 267 | -------------------------------------------------------------------------------- /pkg/api/gallery.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/Mangatsu/server/pkg/utils" 7 | "net/http" 8 | 9 | "github.com/Mangatsu/server/internal/config" 10 | "github.com/Mangatsu/server/pkg/cache" 11 | "github.com/Mangatsu/server/pkg/db" 12 | "github.com/Mangatsu/server/pkg/log" 13 | "github.com/Mangatsu/server/pkg/types/sqlite/model" 14 | "github.com/gorilla/mux" 15 | "go.uber.org/zap" 16 | ) 17 | 18 | type MetadataResult struct { 19 | Hidden bool `json:",omitempty"` 20 | ArchivePath string `json:",omitempty"` 21 | LibraryID string `json:",omitempty"` 22 | SubGalleryCount *uint64 `json:",omitempty"` 23 | model.Gallery 24 | 25 | Tags map[string][]string 26 | 27 | Reference struct { 28 | ExhToken *string 29 | ExhGid *int32 30 | Urls *string 31 | } `alias:"reference.*"` 32 | 33 | GalleryPref *struct { 34 | FavoriteGroup *string 35 | Progress int32 36 | UpdatedAt string 37 | } `alias:"gallery_pref.*"` 38 | 39 | Library model.Library `json:"-"` 40 | } 41 | 42 | type GalleryResult struct { 43 | Meta MetadataResult 44 | Files []string 45 | Count int 46 | } 47 | 48 | type GenericStringResult struct { 49 | Data []string 50 | Count int 51 | } 52 | 53 | type UpdateGalleryForm struct { 54 | Title string 55 | TitleNative string 56 | TitleTranslated string 57 | Released string 58 | Series string 59 | Category string 60 | Language string 61 | Translated bool 62 | Nsfw bool 63 | Hidden bool 64 | ExhToken string 65 | ExhGid int32 66 | AnilistID int32 67 | Urls string 68 | Tags map[string][]string 69 | } 70 | 71 | // returnGalleries returns galleries as JSON. 72 | func returnGalleries(w http.ResponseWriter, r *http.Request) { 73 | access, userUUID := hasAccess(w, r, db.NoRole) 74 | if !access { 75 | return 76 | } 77 | 78 | queryParams := parseQueryParams(r) 79 | galleries, totalCount, err := db.GetGalleries(queryParams, true, userUUID) 80 | if handleResult(w, galleries, err, true, r.RequestURI) { 81 | return 82 | } 83 | 84 | var galleriesResult []MetadataResult 85 | count := len(galleries) 86 | if count > 0 { 87 | for _, gallery := range galleries { 88 | galleriesResult = append(galleriesResult, convertMetadata(gallery)) 89 | } 90 | } 91 | 92 | grouped := queryParams.Grouped == "true" 93 | groupedResult := utils.NewOrderedMap() 94 | if grouped { 95 | for _, gallery := range galleriesResult { 96 | if gallery.Series != nil && *gallery.Series != "" { 97 | subGalleriesCount, err := db.GetGalleryCount(db.Filters{Series: *gallery.Series}, true, userUUID) 98 | if err != nil { 99 | log.Z.Debug("failed getting sub gallery count", 100 | zap.Stringp("name", gallery.Series), 101 | zap.String("err", err.Error())) 102 | continue 103 | } 104 | 105 | gallery.SubGalleryCount = &subGalleriesCount 106 | groupedResult.Set(*gallery.Series, gallery) 107 | } else { 108 | subGalleriesCount := uint64(1) 109 | gallery.SubGalleryCount = &subGalleriesCount 110 | groupedResult.Set(gallery.UUID, gallery) 111 | } 112 | } 113 | } 114 | 115 | if grouped { 116 | resultToJSON(w, struct { 117 | Data *utils.OrderedMap 118 | Count int 119 | TotalCount uint64 120 | }{ 121 | Data: groupedResult, 122 | Count: count, 123 | TotalCount: totalCount, 124 | }, r.RequestURI) 125 | return 126 | } 127 | 128 | resultToJSON(w, struct { 129 | Data []MetadataResult 130 | Count int 131 | TotalCount uint64 132 | }{ 133 | Data: galleriesResult, 134 | Count: count, 135 | TotalCount: totalCount, 136 | }, r.RequestURI) 137 | } 138 | 139 | // returnGalleryCount returns the amount of galleries. 140 | func returnGalleryCount(w http.ResponseWriter, r *http.Request) { 141 | access, userUUID := hasAccess(w, r, db.NoRole) 142 | if !access { 143 | return 144 | } 145 | 146 | queryParams := parseQueryParams(r) 147 | count, err := db.GetGalleryCount(queryParams, true, userUUID) 148 | if handleResult(w, count, err, false, r.URL.Path) { 149 | return 150 | } 151 | 152 | resultToJSON(w, struct{ Count uint64 }{Count: count}, r.URL.Path) 153 | } 154 | 155 | // returnGallery returns one gallery as JSON. 156 | func returnGallery(w http.ResponseWriter, r *http.Request) { 157 | access, userUUID := hasAccess(w, r, db.NoRole) 158 | if !access { 159 | return 160 | } 161 | 162 | params := mux.Vars(r) 163 | galleryUUID := params["uuid"] 164 | 165 | gallery, err := db.GetGallery(&galleryUUID, userUUID, nil) 166 | if handleResult(w, gallery, err, false, r.RequestURI) { 167 | return 168 | } 169 | 170 | galleryWithMeta := convertMetadata(gallery) 171 | if r.URL.Query().Get("meta") == "true" { 172 | resultToJSON(w, GalleryResult{ 173 | Meta: galleryWithMeta, 174 | Files: nil, 175 | Count: 0, 176 | }, r.RequestURI) 177 | return 178 | } 179 | 180 | galleryPath := config.BuildLibraryPath(galleryWithMeta.Library.Path, galleryWithMeta.ArchivePath) 181 | files, count := cache.Read(galleryPath, galleryWithMeta.UUID) 182 | resultToJSON(w, GalleryResult{ 183 | Meta: galleryWithMeta, 184 | Files: files, 185 | Count: count, 186 | }, r.RequestURI) 187 | } 188 | 189 | // returnRandomGallery returns one random gallery as JSON in the same way as returnGallery. 190 | func returnRandomGallery(w http.ResponseWriter, r *http.Request) { 191 | access, userUUID := hasAccess(w, r, db.NoRole) 192 | if !access { 193 | return 194 | } 195 | 196 | gallery, err := db.GetGallery(nil, userUUID, nil) 197 | if handleResult(w, gallery, err, false, r.URL.Path) { 198 | return 199 | } 200 | 201 | galleryWithMeta := convertMetadata(gallery) 202 | galleryPath := config.BuildLibraryPath(galleryWithMeta.Library.Path, galleryWithMeta.ArchivePath) 203 | files, count := cache.Read(galleryPath, galleryWithMeta.UUID) 204 | 205 | resultToJSON(w, GalleryResult{ 206 | Meta: galleryWithMeta, 207 | Files: files, 208 | Count: count, 209 | }, r.URL.Path) 210 | } 211 | 212 | // returnTags returns all tags as JSON. 213 | func returnTags(w http.ResponseWriter, r *http.Request) { 214 | if access, _ := hasAccess(w, r, db.NoRole); !access { 215 | return 216 | } 217 | 218 | tags, _, err := db.GetTags("", true) 219 | if handleResult(w, tags, err, true, r.RequestURI) { 220 | return 221 | } 222 | 223 | resultToJSON(w, tags, r.RequestURI) 224 | } 225 | 226 | // returnCategories returns all public categories as JSON. 227 | func returnCategories(w http.ResponseWriter, r *http.Request) { 228 | if access, _ := hasAccess(w, r, db.NoRole); !access { 229 | return 230 | } 231 | 232 | categories, err := db.GetCategories() 233 | if handleResult(w, categories, err, true, r.RequestURI) { 234 | return 235 | } 236 | 237 | resultToJSON(w, GenericStringResult{ 238 | Data: categories, 239 | Count: len(categories), 240 | }, r.RequestURI) 241 | } 242 | 243 | // returnSeries returns all series as JSON. 244 | func returnSeries(w http.ResponseWriter, r *http.Request) { 245 | if access, _ := hasAccess(w, r, db.NoRole); !access { 246 | return 247 | } 248 | 249 | series, err := db.GetSeries() 250 | if handleResult(w, series, err, true, r.RequestURI) { 251 | return 252 | } 253 | 254 | resultToJSON(w, GenericStringResult{ 255 | Data: series, 256 | Count: len(series), 257 | }, r.RequestURI) 258 | } 259 | 260 | // updateGallery updates a gallery and its reference and tags. 261 | // If tags field is specified and empty, all references to this gallery's tags will be removed. 262 | // If tags is not specified, no changes to tags will be made. 263 | func updateGallery(w http.ResponseWriter, r *http.Request) { 264 | access, _ := hasAccess(w, r, db.Admin) 265 | if !access { 266 | return 267 | } 268 | 269 | params := mux.Vars(r) 270 | galleryUUID := params["uuid"] 271 | if galleryUUID == "" { 272 | errorHandler(w, http.StatusBadRequest, "gallery uuid is required", r.URL.Path) 273 | return 274 | } 275 | 276 | formData := &UpdateGalleryForm{} 277 | if err := json.NewDecoder(r.Body).Decode(formData); err != nil { 278 | errorHandler(w, http.StatusBadRequest, err.Error(), r.URL.Path) 279 | return 280 | } 281 | 282 | newGallery := model.Gallery{ 283 | UUID: galleryUUID, 284 | Title: formData.Title, 285 | TitleNative: &formData.TitleNative, 286 | TitleTranslated: &formData.TitleTranslated, 287 | Released: &formData.Released, 288 | Series: &formData.Series, 289 | Category: &formData.Category, 290 | Language: &formData.Language, 291 | Translated: &formData.Translated, 292 | Nsfw: formData.Nsfw, 293 | Hidden: formData.Hidden, 294 | } 295 | 296 | newReference := model.Reference{ 297 | GalleryUUID: galleryUUID, 298 | Urls: &formData.Urls, 299 | ExhToken: &formData.ExhToken, 300 | ExhGid: &formData.ExhGid, 301 | AnilistID: &formData.AnilistID, 302 | } 303 | 304 | var tags []model.Tag 305 | if formData.Tags != nil { 306 | tags = []model.Tag{} 307 | for namespace, names := range formData.Tags { 308 | for _, name := range names { 309 | tag := model.Tag{ 310 | Namespace: namespace, 311 | Name: name, 312 | } 313 | tags = append(tags, tag) 314 | } 315 | } 316 | } 317 | 318 | if err := db.UpdateGallery(newGallery, tags, newReference, false); err != nil { 319 | errorHandler(w, http.StatusInternalServerError, err.Error(), r.URL.Path) 320 | return 321 | } 322 | fmt.Fprintf(w, `{ "Message": "gallery updated" }`) 323 | } 324 | -------------------------------------------------------------------------------- /pkg/db/user.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "crypto/rand" 5 | "database/sql" 6 | "encoding/base64" 7 | "github.com/Mangatsu/server/pkg/log" 8 | "github.com/Mangatsu/server/pkg/types/sqlite/model" 9 | . "github.com/Mangatsu/server/pkg/types/sqlite/table" 10 | "github.com/Mangatsu/server/pkg/utils" 11 | . "github.com/go-jet/jet/v2/sqlite" 12 | "github.com/google/uuid" 13 | "go.uber.org/zap" 14 | "io" 15 | "strconv" 16 | "time" 17 | ) 18 | 19 | type UserForm struct { 20 | Password *string `json:"password"` 21 | Username *string `json:"username"` 22 | Role *string `json:"role"` 23 | } 24 | 25 | type FavoriteGroups struct { 26 | Data []string `json:"Data"` 27 | } 28 | 29 | type Role uint8 30 | 31 | const ( 32 | SuperAdmin Role = 110 33 | Admin Role = 100 34 | Member Role = 20 35 | Viewer Role = 10 36 | NoRole Role = 0 37 | ) 38 | 39 | // GetUser returns a user from the database. 40 | func GetUser(name string) ([]model.User, error) { 41 | stmt := SELECT( 42 | User.AllColumns, 43 | ).FROM( 44 | User.Table, 45 | ).WHERE( 46 | User.Username.EQ(String(name)), 47 | ) 48 | 49 | var user []model.User 50 | err := stmt.Query(db(), &user) 51 | return user, err 52 | } 53 | 54 | // GetUsers returns users from the database. 55 | func GetUsers() ([]model.User, error) { 56 | stmt := SELECT( 57 | User.UUID, 58 | User.Username, 59 | User.Role, 60 | User.CreatedAt, 61 | User.UpdatedAt, 62 | ).FROM( 63 | User.Table, 64 | ) 65 | 66 | var users []model.User 67 | err := stmt.Query(db(), &users) 68 | return users, err 69 | } 70 | 71 | // GetFavoriteGroups returns user's favorite groups. 72 | func GetFavoriteGroups(userUUID string) ([]string, error) { 73 | stmt := SELECT(GalleryPref.FavoriteGroup).DISTINCT(). 74 | FROM(GalleryPref.Table). 75 | WHERE(GalleryPref.UserUUID.EQ(String(userUUID)). 76 | AND(GalleryPref.FavoriteGroup.IS_NOT_NULL()). 77 | AND(GalleryPref.FavoriteGroup.NOT_EQ(String("")))) 78 | 79 | var favoriteGroups []string 80 | err := stmt.Query(db(), &favoriteGroups) 81 | return favoriteGroups, err 82 | } 83 | 84 | // Register registers a new user. 85 | func Register(username string, password string, role Role) error { 86 | now := time.Now() 87 | 88 | hashSalt, err := utils.DefaultArgon2idHash().GenerateHash([]byte(password), nil) 89 | if err != nil { 90 | log.Z.Error("failed to hash password", zap.Error(err), zap.String("username", username)) 91 | return err 92 | } 93 | 94 | userUUID, err := uuid.NewRandom() 95 | if err != nil { 96 | log.Z.Error("failed to generate UUID", zap.Error(err), zap.String("username", username)) 97 | return err 98 | } 99 | 100 | insertUser := User. 101 | INSERT(User.UUID, User.Username, User.Password, User.Salt, User.Role, User.CreatedAt, User.UpdatedAt). 102 | VALUES(userUUID.String(), username, hashSalt.Hash, hashSalt.Salt, role, now, now) 103 | 104 | _, err = insertUser.Exec(db()) 105 | return err 106 | } 107 | 108 | // Login logs the user in and returns the UUID of the user. 109 | func Login(username string, password string, role Role) (*string, *int32, error) { 110 | result, err := GetUser(username) 111 | if err != nil { 112 | return nil, nil, err 113 | } 114 | 115 | // No user found 116 | if len(result) == 0 { 117 | return nil, nil, sql.ErrNoRows 118 | } 119 | 120 | if Role(result[0].Role) < role { 121 | return nil, nil, nil 122 | } 123 | 124 | ok := utils.DefaultArgon2idHash().Compare(result[0].Password, result[0].Salt, []byte(password)) 125 | if !ok { 126 | return nil, nil, nil 127 | } 128 | 129 | return &result[0].UUID, &result[0].Role, nil 130 | } 131 | 132 | // Logout logs out a user by removing a session. 133 | func Logout(sessionUUID string, userUUID string) error { 134 | return DeleteSession(sessionUUID, userUUID) 135 | } 136 | 137 | // NewSession creates a new session for a user. 138 | func NewSession(userUUID string, expiresIn *int64, sessionName *string) (string, error) { 139 | x := make([]byte, 32) 140 | if _, err := io.ReadFull(rand.Reader, x); err != nil { 141 | return "", err 142 | } 143 | sessionID := base64.URLEncoding.EncodeToString(x) 144 | 145 | if expiresIn != nil { 146 | expiresAt := time.Now().Add(time.Duration(*expiresIn) * time.Second).Unix() 147 | expiresIn = &expiresAt 148 | } 149 | 150 | stmt := Session. 151 | INSERT(Session.ID, Session.UserUUID, Session.Name, Session.ExpiresAt). 152 | VALUES(sessionID, userUUID, sessionName, expiresIn) 153 | 154 | _, err := stmt.Exec(db()) 155 | return sessionID, err 156 | } 157 | 158 | // VerifySession verifies a session by checking if it exists based on the session ID and user UUID. 159 | func VerifySession(id string, userUUID string) bool { 160 | stmt := SELECT(Session.ID). 161 | FROM(Session.Table). 162 | WHERE(Session.ID.EQ(String(id)).AND(Session.UserUUID.EQ(String(userUUID)))). 163 | LIMIT(1) 164 | 165 | var sessions []model.Session 166 | err := stmt.Query(db(), &sessions) 167 | if err != nil { 168 | return false 169 | } 170 | 171 | return len(sessions) > 0 172 | } 173 | 174 | // UpdateUser can be used to update role, password or username of users. 175 | func UpdateUser(userUUID string, userForm *UserForm) error { 176 | now := time.Now() 177 | 178 | tx, err := db().Begin() 179 | if err != nil { 180 | return err 181 | } 182 | defer rollbackTx(tx) 183 | 184 | if userForm.Role != nil { 185 | role, err := strconv.ParseInt(*userForm.Role, 10, 8) 186 | if err != nil { 187 | return err 188 | } 189 | role = utils.Clamp(role, int64(NoRole), int64(Admin)) 190 | 191 | updateUserStmt := User. 192 | UPDATE(User.Role, User.UpdatedAt). 193 | SET(role, now). 194 | WHERE(User.UUID.EQ(String(userUUID))) 195 | if _, err = updateUserStmt.Exec(tx); err != nil { 196 | return err 197 | } 198 | 199 | // If role is changed, delete all sessions of the user. 200 | deleteSessionsStmt := Session.DELETE().WHERE(Session.UserUUID.EQ(String(userUUID))) 201 | if _, err = deleteSessionsStmt.Exec(tx); err != nil { 202 | return err 203 | } 204 | } 205 | 206 | if userForm.Username != nil && *userForm.Username != "" { 207 | updateUserStmt := User. 208 | UPDATE(User.Username, User.UpdatedAt). 209 | SET(userForm.Username, now). 210 | WHERE(User.UUID.EQ(String(userUUID))) 211 | if _, err = updateUserStmt.Exec(tx); err != nil { 212 | return err 213 | } 214 | } 215 | 216 | if userForm.Password != nil && *userForm.Password != "" { 217 | hashSalt, err := utils.DefaultArgon2idHash().GenerateHash([]byte(*userForm.Password), nil) 218 | updateUserStmt := User. 219 | UPDATE(User.Password, User.Salt, User.UpdatedAt). 220 | SET(hashSalt.Hash, hashSalt.Salt, now). 221 | WHERE(User.UUID.EQ(String(userUUID))) 222 | if _, err = updateUserStmt.Exec(tx); err != nil { 223 | return err 224 | } 225 | } 226 | 227 | // Commit transaction. Rollback on error. 228 | err = tx.Commit() 229 | return err 230 | } 231 | 232 | // DeleteUser removes user. Super admin users cannot be deleted. 233 | func DeleteUser(userUUID string) error { 234 | stmt := User.DELETE().WHERE(User.UUID.EQ(String(userUUID)).AND(User.Role.LT_EQ(Int8(int8(Admin))))) 235 | _, err := stmt.Exec(db()) 236 | return err 237 | } 238 | 239 | // GetSessions returns all sessions of a user. 240 | func GetSessions(userUUID string) ([]model.Session, error) { 241 | stmt := Session.SELECT(Session.AllColumns).WHERE(Session.UserUUID.EQ(String(userUUID))) 242 | var sessions []model.Session 243 | err := stmt.Query(db(), &sessions) 244 | return sessions, err 245 | } 246 | 247 | // DeleteSession removes a session based on the session ID and user UUID. 248 | func DeleteSession(id string, userUUID string) error { 249 | stmt := Session.DELETE().WHERE(Session.ID.EQ(String(id)).AND(Session.UserUUID.EQ(String(userUUID)))) 250 | _, err := stmt.Exec(db()) 251 | return err 252 | } 253 | 254 | // PruneExpiredSessions removes all expired sessions. 255 | func PruneExpiredSessions() { 256 | // unixepoch() returns the current unix time in seconds 257 | stmt := Session.DELETE().WHERE(BoolExp(Raw("unixepoch() > session.expires_at"))) 258 | if _, err := stmt.Exec(db()); err != nil { 259 | log.Z.Error("failed to prune expired sessions", zap.String("err", err.Error())) 260 | } 261 | } 262 | 263 | // MigratePassword migrates password from bcrypt to argon2id with salt. 264 | func MigratePassword(username string, password string) error { 265 | selectStmt := SELECT( 266 | User.Password, 267 | User.BcryptPw, 268 | ).FROM( 269 | User.Table, 270 | ).WHERE( 271 | User.Username.EQ(String(username)), 272 | ) 273 | 274 | var user []model.User 275 | err := selectStmt.Query(db(), &user) 276 | if err != nil { 277 | log.Z.Debug("failed to query user while migrating", zap.Error(err), zap.String("username", username)) 278 | return sql.ErrNoRows 279 | } 280 | 281 | if len(user) == 0 { 282 | log.Z.Debug("no user found while migrating", zap.String("username", username)) 283 | return sql.ErrNoRows 284 | } 285 | 286 | // Returns if the password is already migrated 287 | if len(user[0].Password) > 0 { 288 | return nil 289 | } 290 | 291 | argon2idHashSalt, err := utils.DefaultArgon2idHash().GenerateHash([]byte(password), nil) 292 | 293 | if err != nil { 294 | log.Z.Error("failed to hash password while migrating", zap.Error(err), zap.String("username", username)) 295 | return err 296 | } 297 | 298 | updateStmt := User. 299 | UPDATE(User.Password, User.Salt, User.BcryptPw, User.UpdatedAt). 300 | SET(argon2idHashSalt.Hash, argon2idHashSalt.Salt, NULL, time.Now()). 301 | WHERE(User.Username.EQ(String(username))) 302 | 303 | _, err = updateStmt.Exec(db()) 304 | if err != nil { 305 | log.Z.Error("failed to update password while migrating", zap.Error(err), zap.String("username", username)) 306 | } 307 | 308 | return err 309 | } 310 | --------------------------------------------------------------------------------