├── .editorconfig ├── .github ├── ISSUE_TEMPLATE │ ├── 1-bug-report.md │ └── 2-feature-request.md └── workflows │ ├── build.yml │ └── release.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── .pre-commit-config.yaml ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── aliyun.go ├── flags │ ├── config.go │ ├── config_test.go │ └── driver.go ├── hsu.go ├── k12.go ├── root.go ├── sobooks.go ├── talebook.go ├── telegram.go └── version.go ├── go.mod ├── go.sum ├── internal ├── client │ └── client.go ├── driver │ ├── aliyun.go │ ├── aliyun │ │ ├── auth.go │ │ ├── common.go │ │ ├── response.go │ │ └── share.go │ ├── common.go │ ├── lanzou.go │ ├── lanzou │ │ ├── common.go │ │ ├── response.go │ │ ├── share.go │ │ └── share_test.go │ ├── telecom.go │ └── telecom │ │ ├── common.go │ │ ├── crypto.go │ │ ├── login.go │ │ ├── response.go │ │ └── share.go ├── fetcher │ ├── common.go │ ├── fetcher.go │ ├── hsu.go │ ├── k12.go │ ├── service.go │ ├── sobooks.go │ ├── talebook.go │ └── telegram.go ├── file │ ├── decompress.go │ ├── formats.go │ ├── replacer.go │ ├── replacer_windows.go │ └── writer.go ├── log │ ├── console.go │ ├── printer.go │ └── progress.go ├── progress │ ├── progress.go │ └── progress_test.go ├── sobooks │ ├── metadata.go │ └── metadata_test.go ├── talebook │ └── response.go └── telegram │ ├── auth.go │ ├── channel.go │ ├── common.go │ ├── download.go │ └── proxy.go ├── main.go └── scripts └── goimports.sh /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | insert_final_newline = true 7 | trim_trailing_whitespace = true 8 | 9 | [{*.go,Makefile,.gitmodules,go.mod,go.sum}] 10 | indent_style = tab 11 | 12 | [*.md] 13 | indent_style = tab 14 | trim_trailing_whitespace = false 15 | 16 | [*.{yml,yaml,json}] 17 | indent_style = space 18 | indent_size = 2 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/1-bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F41B Bug report" 3 | about: Create a report to help us improve 4 | 5 | --- 6 | 7 | 18 | 19 | * **Version**: 20 | * **Platform**: 21 | 22 | 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/2-feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F680 Feature request" 3 | about: Suggest an idea for this project 4 | 5 | --- 6 | 7 | 12 | 13 | **Is your feature request related to a problem? Please describe.** 14 | Please describe the problem you are trying to solve. 15 | 16 | **Describe the solution you'd like** 17 | Please describe the desired behavior. 18 | 19 | **Describe alternatives you've considered** 20 | Please describe alternative solutions or features you have considered. 21 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | 9 | permissions: 10 | contents: read 11 | pull-requests: read 12 | 13 | jobs: 14 | golangci: 15 | name: lint 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Set up Go 20 | uses: actions/setup-go@v5 21 | with: 22 | go-version: '1.23' 23 | check-latest: true 24 | - name: golangci-lint 25 | uses: golangci/golangci-lint-action@v3 26 | with: 27 | version: latest 28 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | release: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v4 14 | with: 15 | fetch-depth: 0 16 | - name: Set up Go 17 | uses: actions/setup-go@v5 18 | with: 19 | go-version: '1.23' 20 | check-latest: true 21 | - name: Run GoReleaser 22 | uses: goreleaser/goreleaser-action@v6 23 | with: 24 | distribution: goreleaser 25 | version: latest 26 | args: release --clean 27 | env: 28 | GITHUB_TOKEN: ${{ secrets.GH_PAT }} 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IDE releated configure files 2 | .idea/* 3 | test_data 4 | bin/ 5 | .vscode/ 6 | .DS_Store 7 | *.log 8 | .history 9 | *.bat 10 | 11 | # Temp file during testing 12 | *.test 13 | .vscode/ 14 | cpu_profile 15 | *gomock* 16 | *.db 17 | 18 | ## Go 19 | 20 | # Binaries for programs and plugins 21 | *.exe 22 | *.exe~ 23 | *.dll 24 | *.so 25 | *.dylib 26 | bookhunter 27 | bookhunter.* 28 | 29 | # Go releaser 30 | dist/ 31 | 32 | # Test binary, built with `go test -c` 33 | *.test 34 | 35 | # Output of the go coverage tool, specifically when used with LiteIDE 36 | *.out 37 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | dupl: 3 | threshold: 100 4 | funlen: 5 | lines: 100 6 | statements: 50 7 | goconst: 8 | min-len: 2 9 | min-occurrences: 3 10 | gocritic: 11 | enabled-tags: 12 | - diagnostic 13 | - experimental 14 | - opinionated 15 | - performance 16 | - style 17 | disabled-checks: 18 | - dupImport 19 | - ifElseChain 20 | - octalLiteral 21 | - whyNoLint 22 | - wrapperFunc 23 | - exitAfterDefer 24 | - paramTypeCombine 25 | gocyclo: 26 | min-complexity: 15 27 | goimports: 28 | local-prefixes: github.com/bookstairs/bookhunter 29 | lll: 30 | line-length: 120 31 | misspell: 32 | locale: US 33 | nolintlint: 34 | allow-unused: false # report any unused nolint directives 35 | require-explanation: false # don't require an explanation for nolint directives 36 | require-specific: false # don't require nolint directives to be specific about which linter is being skipped 37 | gosec: 38 | excludes: 39 | - G107 40 | - G401 41 | - G501 42 | - G306 43 | - G115 44 | depguard: 45 | rules: 46 | main: 47 | # Packages that are not allowed where the value is a suggestion. 48 | deny: 49 | - pkg: "github.com/sirupsen/logrus" 50 | desc: not allowed 51 | - pkg: "github.com/pkg/errors" 52 | desc: Should be replaced by standard lib errors package 53 | 54 | 55 | linters: 56 | disable-all: true 57 | 58 | ## https://github.com/golangci/golangci-lint/issues/2649 59 | enable: 60 | - bodyclose 61 | - depguard 62 | - dogsled 63 | - dupl 64 | - errcheck 65 | - funlen 66 | - goconst 67 | - gocritic 68 | - gocyclo 69 | - gofmt 70 | - goimports 71 | - goprintffuncname 72 | - gosec 73 | - gosimple 74 | - govet 75 | - ineffassign 76 | - lll 77 | - misspell 78 | - nakedret 79 | - noctx 80 | - nolintlint 81 | - staticcheck 82 | - stylecheck 83 | - typecheck 84 | - unconvert 85 | - unparam 86 | - unused 87 | - whitespace 88 | 89 | run: 90 | timeout: 5m 91 | 92 | issues: 93 | exclude-dirs: 94 | - .github 95 | - docker 96 | - scripts 97 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: bookhunter 2 | version: 2 3 | 4 | before: 5 | hooks: 6 | - go mod tidy 7 | 8 | builds: 9 | - id: bookhunter 10 | binary: bookhunter 11 | gcflags: 12 | - all=-l -B 13 | ldflags: 14 | - -s -w 15 | - -X github.com/bookstairs/bookhunter/cmd.gitVersion={{ .Version }} 16 | - -X github.com/bookstairs/bookhunter/cmd.gitCommit={{ .Commit }} 17 | - -X github.com/bookstairs/bookhunter/cmd.buildDate={{ .Date }} 18 | env: 19 | - CGO_ENABLED=0 20 | goos: 21 | - linux 22 | - windows 23 | - darwin 24 | goarch: 25 | - amd64 26 | - arm64 27 | - loong64 28 | ignore: 29 | - goos: windows 30 | goarch: loong64 31 | - goos: darwin 32 | goarch: loong64 33 | 34 | scoops: 35 | - repository: 36 | owner: bookstairs 37 | name: scoop-bucket 38 | branch: master 39 | commit_author: 40 | name: syhily 41 | email: syhily@gmail.com 42 | homepage: "https://github.com/bookstairs" 43 | description: "Software to download chinese ebooks from Internet." 44 | license: MIT 45 | 46 | brews: 47 | - name: bookhunter 48 | repository: 49 | owner: bookstairs 50 | name: homebrew-tap 51 | branch: master 52 | commit_author: 53 | name: syhily 54 | email: syhily@gmail.com 55 | directory: Formula 56 | homepage: "https://github.com/bookstairs" 57 | description: "Software to download chinese ebooks from Internet." 58 | license: "MIT" 59 | install: | 60 | bin.install "bookhunter" 61 | 62 | # Install shell completions 63 | output = Utils.safe_popen_read("#{bin}/bookhunter", "completion", "bash") 64 | (bash_completion/"bookhunter").write output 65 | 66 | output = Utils.safe_popen_read("#{bin}/bookhunter", "completion", "zsh") 67 | (zsh_completion/"_bookhunter").write output 68 | 69 | output = Utils.safe_popen_read("#{bin}/bookhunter", "completion", "fish") 70 | (fish_completion/"bookhunter.fish").write output 71 | test: | 72 | system "#{bin}/bookhunter version" 73 | 74 | checksum: 75 | name_template: 'checksums.txt' 76 | 77 | changelog: 78 | sort: asc 79 | filters: 80 | exclude: 81 | - '^docs:' 82 | - '^test:' 83 | - '^web:' 84 | - '^build:' 85 | 86 | archives: 87 | - id: bookhunter 88 | builds: 89 | - bookhunter 90 | format: tar.gz 91 | wrap_in_directory: "true" 92 | format_overrides: 93 | - goos: windows 94 | format: zip 95 | 96 | release: 97 | draft: true 98 | 99 | snapshot: 100 | version_template: "{{ incminor .Version }}-next" 101 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.6.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-added-large-files 8 | - repo: https://github.com/tekwizely/pre-commit-golang 9 | rev: v1.0.0-rc.1 10 | hooks: 11 | - id: go-mod-tidy-repo 12 | - id: golangci-lint-mod 13 | - id: my-cmd 14 | name: goimports 15 | alias: goimports 16 | args: [ scripts/goimports.sh, github.com/bookstairs/bookhunter ] 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 bookstairs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: help build test deps clean 2 | 3 | help: ## Display this help 4 | @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n\nTargets:\n"} \ 5 | /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-10s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) 6 | 7 | build: ## Build executable files 8 | @goreleaser release --clean --snapshot 9 | 10 | test: ## Run tests 11 | go install "github.com/rakyll/gotest@latest" 12 | GIN_MODE=release 13 | LOG_LEVEL=fatal ## disable log for test 14 | gotest -v -coverprofile=coverage.out -covermode=atomic ./... 15 | 16 | deps: ## Update vendor. 17 | go mod verify 18 | go mod tidy -v 19 | go get -u ./... 20 | 21 | clean: ## Clean up build files. 22 | rm -rf dist/ 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ⏬ bookhunter 2 | 3 | [![LICENSE](https://img.shields.io/github/license/bookstairs/bookhunter)](https://github.com/bookstairs/bookhunter/blob/main/LICENSE) 4 | [![Language](https://img.shields.io/badge/Language-Go-blue.svg)](https://golang.org/) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/bookstairs/bookhunter)](https://goreportcard.com/report/github.com/bookstairs/bookhunter) 6 | ![](https://img.shields.io/github/stars/bookstairs/bookhunter.svg) 7 | ![](https://img.shields.io/github/forks/bookstairs/bookhunter.svg) 8 | ![Release](https://github.com/bookstairs/bookhunter/workflows/release/badge.svg) 9 | 10 | Downloading books from [talebook](https://github.com/talebook/talebook), 11 | [SoBooks](https://sobooks.cc) 12 | [中小学教材](https://basic.smartedu.cn/tchMaterial) 13 | and Telegram Channels. This is a totally 14 | rewritten fork compared to its [original version](https://github.com/hellojukay/dl-talebook). 15 | 16 | ## 🚧 Development 17 | 18 | 1. [Go Releaser](https://github.com/goreleaser/goreleaser) is used for releasing and local building 19 | 2. [golangci-lint](https://github.com/golangci/golangci-lint) is used for code style. 20 | 3. [pre-commit](https://pre-commit.com/) is used for checking code before committing. 21 | 22 | ## 💾 Install 23 | 24 | ### 🍎 Homebrew (for macOS, Linux) 25 | 26 | ```shell 27 | brew tap bookstairs/tap 28 | brew install bookhunter 29 | ``` 30 | 31 | ### 💻 Scope (for Windows) 32 | 33 | ```shell 34 | scoop bucket add bookstairs https://github.com/bookstairs/scoop-bucket.git 35 | scoop install bookstairs/bookhunter 36 | ``` 37 | 38 | ### 🛠 Manually 39 | 40 | Download the latest release in [release page](https://github.com/bookstairs/bookhunter/releases). Choose related tarball 41 | by your running environment. 42 | 43 | ## 📚 Usage 44 | 45 | | Website | Address | Direct Download | [Aliyun](https://www.aliyundrive.com/) | [Lanzou](https://www.lanzou.com/) | [Telecom](https://cloud.189.cn/) | 46 | |--------------------------------------------------|----------------------------------------|-----------------|----------------------------------------|-----------------------------------|----------------------------------| 47 | | [智慧教育平台](#download-textbooks-for-kids) | | ✅ | ❌ | ❌ | ❌ | 48 | | [Talebook](#download-books-from-talebook) | | ✅ | ❌ | ❌ | ❌ | 49 | | [SoBooks](#download-books-from-sobooks) | | ✅ | ❌ | ✅ | ❌ | 50 | | [Telegram](#download-books-from-telegram-groups) | | ✅ | ❌ | ❌ | ❌ | 51 | | [Hsu Life](#download-books-from-hsu-life) | | ✅ | ❌ | ❌ | ❌ | 52 | 53 | ### Login Aliyundrive to get the `refreshToken` 54 | 55 | We would show a QR code at the first time. And cache the `refreshToken` after successfully login. 56 | 57 | ```shell 58 | bookhunter aliyun 59 | ``` 60 | 61 | ### Download textbooks for Kids 62 | 63 | ```text 64 | Usage: 65 | bookhunter k12 [flags] 66 | 67 | Flags: 68 | -d, --download string The book directory you want to use (default ".") 69 | -h, --help help for k12 70 | --ratelimit int The allowed requests per minutes for every thread (default 30) 71 | -t, --thread int The number of download thead (default 1) 72 | 73 | Global Flags: 74 | -c, --config string The config path for bookhunter 75 | -k, --keyword strings The keywords for books 76 | --proxy string The request proxy 77 | --retry int The retry times for a failed download (default 3) 78 | -s, --skip-error Continue to download the next book if the current book download failed (default true) 79 | --verbose Print all the logs for debugging 80 | ``` 81 | 82 | ### Register account in Talebook 83 | 84 | ```text 85 | Usage: 86 | bookhunter talebook register [flags] 87 | 88 | Flags: 89 | -e, --email string The talebook email 90 | -h, --help help for register 91 | -p, --password string The talebook password 92 | -u, --username string The talebook username 93 | -w, --website string The talebook link 94 | 95 | Global Flags: 96 | -c, --config string The config path for bookhunter 97 | -k, --keyword strings The keywords for books 98 | --proxy string The request proxy 99 | --retry int The retry times for a failed download (default 3) 100 | -s, --skip-error Continue to download the next book if the current book download failed (default true) 101 | --verbose Print all the logs for debugging 102 | ``` 103 | 104 | ### Download books from Talebook 105 | 106 | ```text 107 | Usage: 108 | bookhunter talebook download [flags] 109 | 110 | Flags: 111 | -d, --download string The book directory you want to use (default ".") 112 | -f, --format strings The file formats you want to download (default [epub,azw3,mobi,pdf,zip]) 113 | -h, --help help for download 114 | -i, --initial int The book id you want to start download (default 1) 115 | -p, --password string The talebook password 116 | --ratelimit int The allowed requests per minutes for every thread (default 30) 117 | -r, --rename Rename the book file by book id 118 | -t, --thread int The number of download thead (default 1) 119 | -u, --username string The talebook username 120 | -w, --website string The talebook link 121 | 122 | Global Flags: 123 | -c, --config string The config path for bookhunter 124 | -k, --keyword strings The keywords for books 125 | --proxy string The request proxy 126 | --retry int The retry times for a failed download (default 3) 127 | -s, --skip-error Continue to download the next book if the current book download failed (default true) 128 | --verbose Print all the logs for debugging 129 | ``` 130 | 131 | ### Download books from SoBooks 132 | 133 | ```text 134 | Usage: 135 | bookhunter sobooks [flags] 136 | 137 | Flags: 138 | --code string The secret code for SoBooks (default "244152") 139 | -d, --download string The book directory you want to use (default ".") 140 | -e, --extract Extract the archive file for filtering 141 | -f, --format strings The file formats you want to download (default [epub,azw3,mobi,pdf,zip]) 142 | -h, --help help for sobooks 143 | -i, --initial int The book id you want to start download (default 1) 144 | --ratelimit int The allowed requests per minutes for every thread (default 30) 145 | -r, --rename Rename the book file by book id 146 | -t, --thread int The number of download thead (default 1) 147 | 148 | Global Flags: 149 | -c, --config string The config path for bookhunter 150 | -k, --keyword strings The keywords for books 151 | --proxy string The request proxy 152 | --retry int The retry times for a failed download (default 3) 153 | -s, --skip-error Continue to download the next book if the current book download failed (default true) 154 | --verbose Print all the logs for debugging 155 | ``` 156 | 157 | ### Download books from Telegram groups 158 | 159 | Example command: `bookhunter telegram --appID ****** --appHash ****** --thread 4 --proxy http://127.0.0.1:7890 --channelID https://t.me/sharebooks4you` 160 | 161 | Please refer [Creating your Telegram Application](https://core.telegram.org/api/obtaining_api_id) to obtain your `appID` 162 | and `appHash`. 163 | 164 | ```text 165 | Usage: 166 | bookhunter telegram [flags] 167 | 168 | Flags: 169 | --appHash string The app hash for telegram 170 | --appID int The app id for telegram 171 | --channelID string The channel id for telegram 172 | -d, --download string The book directory you want to use (default ".") 173 | -e, --extract Extract the archive file for filtering 174 | -f, --format strings The file formats you want to download (default [epub,azw3,mobi,pdf,zip]) 175 | -h, --help help for telegram 176 | -i, --initial int The book id you want to start download (default 1) 177 | --mobile string The mobile number, we will add +86 as default zone code 178 | --ratelimit int The allowed requests per minutes for every thread (default 30) 179 | --refresh Refresh the login session 180 | -r, --rename Rename the book file by book id 181 | -t, --thread int The number of download thead (default 1) 182 | 183 | Global Flags: 184 | -c, --config string The config path for bookhunter 185 | -k, --keyword strings The keywords for books 186 | --proxy string The request proxy 187 | --retry int The retry times for a failed download (default 3) 188 | -s, --skip-error Continue to download the next book if the current book download failed (default true) 189 | --verbose Print all the logs for debugging 190 | ``` 191 | 192 | ### Download books from Hsu Life 193 | 194 | Example command: `bookhunter hsu --username ****** --password ******` 195 | 196 | ```text 197 | Usage: 198 | bookhunter hsu [flags] 199 | 200 | Flags: 201 | -d, --download string The book directory you want to use (default ".") 202 | -f, --format strings The file formats you want to download (default [epub,azw3,mobi,pdf,zip]) 203 | -h, --help help for hsu 204 | -i, --initial int The book id you want to start download (default 1) 205 | -p, --password string The hsu.life password 206 | --ratelimit int The allowed requests per minutes for every thread (default 30) 207 | -r, --rename Rename the book file by book id 208 | -t, --thread int The number of download thead (default 1) 209 | -u, --username string The hsu.life username 210 | 211 | Global Flags: 212 | -c, --config string The config path for bookhunter 213 | -k, --keyword strings The keywords for books 214 | --proxy string The request proxy 215 | --retry int The retry times for a failed download (default 3) 216 | -s, --skip-error Continue to download the next book if the current book download failed (default true) 217 | --verbose Print all the logs for debugging 218 | ``` 219 | -------------------------------------------------------------------------------- /cmd/aliyun.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | 6 | "github.com/bookstairs/bookhunter/cmd/flags" 7 | "github.com/bookstairs/bookhunter/internal/driver/aliyun" 8 | "github.com/bookstairs/bookhunter/internal/log" 9 | ) 10 | 11 | var aliyunCmd = &cobra.Command{ 12 | Use: "aliyun", 13 | Short: "A command line tool for acquiring the refresh token from aliyundrive with QR code login", 14 | Run: func(cmd *cobra.Command, args []string) { 15 | // Create the client config. 16 | flags.Website = "https://api.aliyundrive.com" 17 | c, err := flags.NewClientConfig() 18 | if err != nil { 19 | log.Exit(err) 20 | } 21 | 22 | // Perform login. 23 | aliyun, err := aliyun.New(c, "") 24 | if err != nil { 25 | log.Exit(err) 26 | } 27 | log.Info("RefreshToken: ", aliyun.RefreshToken()) 28 | }, 29 | } 30 | -------------------------------------------------------------------------------- /cmd/flags/config.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | import ( 4 | "os" 5 | "runtime" 6 | "strings" 7 | 8 | "github.com/bookstairs/bookhunter/internal/client" 9 | "github.com/bookstairs/bookhunter/internal/fetcher" 10 | "github.com/bookstairs/bookhunter/internal/file" 11 | ) 12 | 13 | var ( 14 | // Flags for talebook registering. 15 | 16 | Username = "" 17 | Password = "" 18 | Email = "" 19 | 20 | // Common flags. 21 | 22 | Website = "" 23 | Proxy = "" 24 | ConfigRoot = "" 25 | Keywords []string 26 | Retry = 3 27 | SkipError = true 28 | 29 | // Common download flags. 30 | 31 | Formats = []string{ 32 | string(file.EPUB), 33 | string(file.AZW3), 34 | string(file.MOBI), 35 | string(file.PDF), 36 | string(file.ZIP), 37 | } 38 | Extract = false 39 | DownloadPath, _ = os.Getwd() 40 | InitialBookID = int64(1) 41 | Rename = false 42 | Thread = runtime.NumCPU() 43 | RateLimit = 30 44 | 45 | // Telegram configurations. 46 | 47 | ChannelID = "" 48 | Mobile = "" 49 | ReLogin = false 50 | AppID = int64(0) 51 | AppHash = "" 52 | 53 | // SoBooks configurations. 54 | 55 | SoBooksCode = "881120" 56 | ) 57 | 58 | func NewClientConfig() (*client.Config, error) { 59 | return client.NewConfig(Website, Proxy, ConfigRoot) 60 | } 61 | 62 | // NewFetcher will create the fetcher by the command line arguments. 63 | func NewFetcher(category fetcher.Category, properties map[string]string) (fetcher.Fetcher, error) { 64 | cc, err := NewClientConfig() 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | fs, err := fetcher.ParseFormats(Formats) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | return fetcher.New(&fetcher.Config{ 75 | Config: cc, 76 | Category: category, 77 | Formats: fs, 78 | Keywords: Keywords, 79 | Extract: Extract, 80 | DownloadPath: DownloadPath, 81 | InitialBookID: InitialBookID, 82 | Rename: Rename, 83 | Thread: Thread, 84 | RateLimit: RateLimit, 85 | Properties: properties, 86 | Retry: Retry, 87 | SkipError: SkipError, 88 | }) 89 | } 90 | 91 | // HideSensitive will replace the sensitive content with star but keep the original length. 92 | func HideSensitive(content string) string { 93 | if content == "" { 94 | return "" 95 | } 96 | 97 | // Preserve only the prefix and suffix, replace others with * 98 | s := []rune(content) 99 | c := len(s) 100 | 101 | // Determine the visible length of the prefix and suffix. 102 | l := 1 103 | if c >= 9 { 104 | l = 3 105 | } else if c >= 6 { 106 | l = 2 107 | } 108 | 109 | return string(s[0:l]) + strings.Repeat("*", c-l*2) + string(s[c-l:c]) 110 | } 111 | -------------------------------------------------------------------------------- /cmd/flags/config_test.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | import "testing" 4 | 5 | func TestHideSensitive(t *testing.T) { 6 | type args struct { 7 | content string 8 | } 9 | tests := []struct { 10 | name string 11 | args args 12 | want string 13 | }{ 14 | {name: "Content which length is [0, 6)", args: args{content: "12345"}, want: "1***5"}, 15 | {name: "Content which length is [6, 9)", args: args{content: "12345678"}, want: "12****78"}, 16 | {name: "Content which length is [9, +∞)", args: args{content: "1234567890"}, want: "123****890"}, 17 | } 18 | for _, tt := range tests { 19 | t.Run(tt.name, func(t *testing.T) { 20 | if got := HideSensitive(tt.args.content); got != tt.want { 21 | t.Errorf("HideSensitive() = %v, want %v", got, tt.want) 22 | } 23 | }) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /cmd/flags/driver.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | var ( 4 | Driver = "telecom" 5 | 6 | // Aliyun Drive 7 | 8 | RefreshToken = "" 9 | 10 | // Telecom Cloud 11 | 12 | TelecomUsername = "" 13 | TelecomPassword = "" 14 | ) 15 | 16 | // NewDriverProperties which would be used in driver.New 17 | func NewDriverProperties() map[string]string { 18 | return map[string]string{ 19 | "driver": Driver, 20 | "refreshToken": RefreshToken, 21 | "telecomUsername": TelecomUsername, 22 | "telecomPassword": TelecomPassword, 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /cmd/hsu.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | 6 | "github.com/bookstairs/bookhunter/cmd/flags" 7 | "github.com/bookstairs/bookhunter/internal/fetcher" 8 | "github.com/bookstairs/bookhunter/internal/log" 9 | ) 10 | 11 | const hsuWebsite = "https://book.hsu.life" 12 | 13 | var hsuCmd = &cobra.Command{ 14 | Use: "hsu", 15 | Short: "A tool for downloading book from hsu.life", 16 | Run: func(cmd *cobra.Command, args []string) { 17 | log.NewPrinter(). 18 | Title("hsu.life Download Information"). 19 | Head(log.DefaultHead...). 20 | Row("Username", flags.Username). 21 | Row("Password", flags.HideSensitive(flags.Password)). 22 | Row("Config Path", flags.ConfigRoot). 23 | Row("Proxy", flags.Proxy). 24 | Row("Formats", flags.Formats). 25 | Row("Download Path", flags.DownloadPath). 26 | Row("Initial ID", flags.InitialBookID). 27 | Row("Rename File", flags.Rename). 28 | Row("Thread", flags.Thread). 29 | Row("Keywords", flags.Keywords). 30 | Row("Thread Limit (req/min)", flags.RateLimit). 31 | Print() 32 | 33 | flags.Website = hsuWebsite 34 | 35 | // Create the fetcher. 36 | f, err := flags.NewFetcher(fetcher.Hsu, map[string]string{ 37 | "username": flags.Username, 38 | "password": flags.Password, 39 | }) 40 | log.Exit(err) 41 | 42 | // Start downloading the books. 43 | err = f.Download() 44 | log.Exit(err) 45 | 46 | // Finished all the tasks. 47 | log.Info("Successfully download all the books.") 48 | }, 49 | } 50 | 51 | func init() { 52 | // Add flags for use info. 53 | f := hsuCmd.Flags() 54 | 55 | // Talebook related flags. 56 | f.StringVarP(&flags.Username, "username", "u", flags.Username, "The hsu.life username") 57 | f.StringVarP(&flags.Password, "password", "p", flags.Password, "The hsu.life password") 58 | 59 | // Common download flags. 60 | f.StringSliceVarP(&flags.Formats, "format", "f", flags.Formats, "The file formats you want to download") 61 | f.StringVarP(&flags.DownloadPath, "download", "d", flags.DownloadPath, "The book directory you want to use") 62 | f.Int64VarP(&flags.InitialBookID, "initial", "i", flags.InitialBookID, "The book id you want to start download") 63 | f.BoolVarP(&flags.Rename, "rename", "r", flags.Rename, "Rename the book file by book id") 64 | f.IntVarP(&flags.Thread, "thread", "t", flags.Thread, "The number of download thead") 65 | f.IntVar(&flags.RateLimit, "ratelimit", flags.RateLimit, "The allowed requests per minutes for every thread") 66 | 67 | // Mark some flags as required. 68 | _ = hsuCmd.MarkFlagRequired("username") 69 | _ = hsuCmd.MarkFlagRequired("password") 70 | } 71 | -------------------------------------------------------------------------------- /cmd/k12.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | 6 | "github.com/bookstairs/bookhunter/cmd/flags" 7 | "github.com/bookstairs/bookhunter/internal/fetcher" 8 | "github.com/bookstairs/bookhunter/internal/log" 9 | ) 10 | 11 | const k12Website = "https://www.zxx.edu.cn" 12 | 13 | var k12Cmd = &cobra.Command{ 14 | Use: "k12", 15 | Short: "A tool for downloading textbooks from www.zxx.edu.cn", 16 | Run: func(cmd *cobra.Command, args []string) { 17 | // Print download configuration. 18 | log.NewPrinter(). 19 | Title("Textbook Download Information"). 20 | Head(log.DefaultHead...). 21 | Row("Config Path", flags.ConfigRoot). 22 | Row("Proxy", flags.Proxy). 23 | Row("Download Path", flags.DownloadPath). 24 | Row("Thread", flags.Thread). 25 | Row("Thread Limit (req/min)", flags.RateLimit). 26 | Row("Keywords", flags.Keywords). 27 | Print() 28 | 29 | flags.Website = k12Website 30 | f, err := flags.NewFetcher(fetcher.K12, map[string]string{}) 31 | log.Exit(err) 32 | 33 | err = f.Download() 34 | log.Exit(err) 35 | 36 | // Finished all the tasks. 37 | log.Info("Successfully download all the textbooks.") 38 | }, 39 | } 40 | 41 | func init() { 42 | f := k12Cmd.Flags() 43 | 44 | f.StringVarP(&flags.DownloadPath, "download", "d", flags.DownloadPath, "The book directory you want to use") 45 | f.IntVarP(&flags.Thread, "thread", "t", flags.Thread, "The number of download thead") 46 | f.IntVar(&flags.RateLimit, "ratelimit", flags.RateLimit, "The allowed requests per minutes for every thread") 47 | } 48 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/spf13/cobra" 7 | 8 | "github.com/bookstairs/bookhunter/cmd/flags" 9 | "github.com/bookstairs/bookhunter/internal/log" 10 | ) 11 | 12 | // rootCmd represents the base command when called without any subcommands 13 | var rootCmd = &cobra.Command{ 14 | Use: "bookhunter", 15 | Short: "A downloader for downloading books from internet", 16 | Long: `You can use this command to download book from these websites 17 | 18 | 1. Self-hosted talebook websites 19 | 2. https://www.sanqiu.mobi 20 | 3. Telegram channel`, 21 | } 22 | 23 | // Execute adds all child commands to the root command and sets flags appropriately. 24 | // This is called by main.main(). It only needs to happen once to the rootCmd. 25 | func Execute() { 26 | err := rootCmd.Execute() 27 | if err != nil { 28 | os.Exit(1) 29 | } 30 | } 31 | 32 | func init() { 33 | // Download commands. 34 | rootCmd.AddCommand(talebookCmd) 35 | rootCmd.AddCommand(telegramCmd) 36 | rootCmd.AddCommand(sobooksCmd) 37 | rootCmd.AddCommand(k12Cmd) 38 | rootCmd.AddCommand(hsuCmd) 39 | 40 | // Tool commands. 41 | rootCmd.AddCommand(aliyunCmd) 42 | rootCmd.AddCommand(versionCmd) 43 | 44 | persistentFlags := rootCmd.PersistentFlags() 45 | 46 | // Common flags. 47 | persistentFlags.StringVarP(&flags.ConfigRoot, "config", "c", flags.ConfigRoot, "The config path for bookhunter") 48 | persistentFlags.StringVar(&flags.Proxy, "proxy", flags.Proxy, "The request proxy") 49 | persistentFlags.IntVarP(&flags.Retry, "retry", "", flags.Retry, "The retry times for a failed download") 50 | persistentFlags.BoolVarP(&flags.SkipError, "skip-error", "s", flags.SkipError, 51 | "Continue to download the next book if the current book download failed") 52 | persistentFlags.StringSliceVarP(&flags.Keywords, "keyword", "k", flags.Keywords, "The keywords for books") 53 | persistentFlags.BoolVar(&log.EnableDebug, "verbose", false, "Print all the logs for debugging") 54 | } 55 | -------------------------------------------------------------------------------- /cmd/sobooks.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | 6 | "github.com/bookstairs/bookhunter/cmd/flags" 7 | "github.com/bookstairs/bookhunter/internal/driver" 8 | "github.com/bookstairs/bookhunter/internal/fetcher" 9 | "github.com/bookstairs/bookhunter/internal/log" 10 | ) 11 | 12 | const ( 13 | lowestSobooksBookID = 18000 14 | sobooksWebsite = "https://sobooks.cc" 15 | ) 16 | 17 | // sobooksCmd used for download books from sobooks.cc 18 | var sobooksCmd = &cobra.Command{ 19 | Use: "sobooks", 20 | Short: "A tool for downloading books from sobooks.cc", 21 | Run: func(cmd *cobra.Command, args []string) { 22 | // Set the default start index. 23 | if flags.InitialBookID < lowestSobooksBookID { 24 | flags.InitialBookID = lowestSobooksBookID 25 | } 26 | 27 | // Print download configuration. 28 | log.NewPrinter(). 29 | Title("SoBooks Download Information"). 30 | Head(log.DefaultHead...). 31 | Row("SoBooks Code", flags.SoBooksCode). 32 | Row("Config Path", flags.ConfigRoot). 33 | Row("Proxy", flags.Proxy). 34 | Row("Formats", flags.Formats). 35 | Row("Extract Archive", flags.Extract). 36 | Row("Download Path", flags.DownloadPath). 37 | Row("Initial ID", flags.InitialBookID). 38 | Row("Rename File", flags.Rename). 39 | Row("Thread", flags.Thread). 40 | Row("Keywords", flags.Keywords). 41 | Row("Thread Limit (req/min)", flags.RateLimit). 42 | Print() 43 | 44 | // Set the domain for using in the client.Client. 45 | flags.Website = sobooksWebsite 46 | flags.Driver = string(driver.LANZOU) 47 | 48 | // Create the fetcher. 49 | properties := flags.NewDriverProperties() 50 | properties["code"] = flags.SoBooksCode 51 | f, err := flags.NewFetcher(fetcher.SoBooks, properties) 52 | log.Exit(err) 53 | 54 | // Wait all the threads have finished. 55 | err = f.Download() 56 | log.Exit(err) 57 | 58 | // Finished all the tasks. 59 | log.Info("Successfully download all the books.") 60 | }, 61 | } 62 | 63 | func init() { 64 | f := sobooksCmd.Flags() 65 | 66 | // Common download flags. 67 | f.StringSliceVarP(&flags.Formats, "format", "f", flags.Formats, "The file formats you want to download") 68 | f.BoolVarP(&flags.Extract, "extract", "e", flags.Extract, "Extract the archive file for filtering") 69 | f.StringVarP(&flags.DownloadPath, "download", "d", flags.DownloadPath, "The book directory you want to use") 70 | f.Int64VarP(&flags.InitialBookID, "initial", "i", flags.InitialBookID, "The book id you want to start download") 71 | f.BoolVarP(&flags.Rename, "rename", "r", flags.Rename, "Rename the book file by book id") 72 | f.IntVarP(&flags.Thread, "thread", "t", flags.Thread, "The number of download thead") 73 | f.IntVar(&flags.RateLimit, "ratelimit", flags.RateLimit, "The allowed requests per minutes for every thread") 74 | 75 | // SoBooks books flags. 76 | f.StringVar(&flags.SoBooksCode, "code", flags.SoBooksCode, "The secret code for SoBooks") 77 | 78 | _ = sobooksCmd.MarkFlagRequired("code") 79 | } 80 | -------------------------------------------------------------------------------- /cmd/talebook.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | 8 | "github.com/bookstairs/bookhunter/cmd/flags" 9 | "github.com/bookstairs/bookhunter/internal/client" 10 | "github.com/bookstairs/bookhunter/internal/fetcher" 11 | "github.com/bookstairs/bookhunter/internal/log" 12 | "github.com/bookstairs/bookhunter/internal/talebook" 13 | ) 14 | 15 | // talebookCmd used to download books from talebook 16 | var talebookCmd = &cobra.Command{ 17 | Use: "talebook", 18 | Short: "A tool for downloading books from talebook server", 19 | } 20 | 21 | // talebookDownloadCmd represents the download command 22 | var talebookDownloadCmd = &cobra.Command{ 23 | Use: "download", 24 | Short: "Download the books from talebook", 25 | Run: func(cmd *cobra.Command, args []string) { 26 | // Print download configuration. 27 | log.NewPrinter(). 28 | Title("Talebook Download Information"). 29 | Head(log.DefaultHead...). 30 | Row("Website", flags.Website). 31 | Row("Username", flags.HideSensitive(flags.Username)). 32 | Row("Password", flags.HideSensitive(flags.Password)). 33 | Row("Config Path", flags.ConfigRoot). 34 | Row("Proxy", flags.Proxy). 35 | Row("Formats", flags.Formats). 36 | Row("Download Path", flags.DownloadPath). 37 | Row("Initial ID", flags.InitialBookID). 38 | Row("Rename File", flags.Rename). 39 | Row("Thread", flags.Thread). 40 | Row("Keywords", flags.Keywords). 41 | Row("Thread Limit (req/min)", flags.RateLimit). 42 | Print() 43 | 44 | // Create the fetcher. 45 | f, err := flags.NewFetcher(fetcher.Talebook, map[string]string{ 46 | "username": flags.Username, 47 | "password": flags.Password, 48 | }) 49 | log.Exit(err) 50 | 51 | // Start downloading the books. 52 | err = f.Download() 53 | log.Exit(err) 54 | 55 | // Finished all the tasks. 56 | log.Info("Successfully download all the books.") 57 | }, 58 | } 59 | 60 | // talebookRegisterCmd represents the register command. 61 | var talebookRegisterCmd = &cobra.Command{ 62 | Use: "register", 63 | Short: "Register account on talebook", 64 | Long: `Some talebook website need a user account for downloading books 65 | You can use this register command for creating account`, 66 | Run: func(cmd *cobra.Command, args []string) { 67 | // Print register configuration. 68 | log.NewPrinter(). 69 | Title("Talebook Register Information"). 70 | Head(log.DefaultHead...). 71 | Row("Website", flags.Website). 72 | Row("Username", flags.Username). 73 | Row("Password", flags.Password). 74 | Row("Email", flags.Email). 75 | Row("Config Path", flags.ConfigRoot). 76 | Row("Proxy", flags.Proxy). 77 | Print() 78 | 79 | // Create client config. 80 | config, err := client.NewConfig(flags.Website, flags.Proxy, flags.ConfigRoot) 81 | log.Exit(err) 82 | 83 | // Create http client. 84 | c, err := client.New(config) 85 | log.Exit(err) 86 | 87 | // Execute the register request. 88 | resp, err := c.R(). 89 | SetFormData(map[string]string{ 90 | "username": flags.Username, 91 | "password": flags.Password, 92 | "nickname": flags.Username, 93 | "email": flags.Email, 94 | }). 95 | SetResult(&talebook.CommonResp{}). 96 | ForceContentType("application/json"). 97 | Post("/api/user/sign_up") 98 | log.Exit(err) 99 | 100 | result := resp.Result().(*talebook.CommonResp) 101 | if result.Err == talebook.SuccessStatus { 102 | log.Info("Register success.") 103 | } else { 104 | log.Exit(fmt.Errorf("register failed, reason: %s", result.Err)) 105 | } 106 | }, 107 | } 108 | 109 | func init() { 110 | /// Add the download command. 111 | 112 | // Add flags for use info. 113 | f := talebookDownloadCmd.Flags() 114 | 115 | // Talebook related flags. 116 | f.StringVarP(&flags.Username, "username", "u", flags.Username, "The talebook username") 117 | f.StringVarP(&flags.Password, "password", "p", flags.Password, "The talebook password") 118 | f.StringVarP(&flags.Website, "website", "w", flags.Website, "The talebook link") 119 | 120 | // Common download flags. 121 | f.StringSliceVarP(&flags.Formats, "format", "f", flags.Formats, "The file formats you want to download") 122 | f.StringVarP(&flags.DownloadPath, "download", "d", flags.DownloadPath, "The book directory you want to use") 123 | f.Int64VarP(&flags.InitialBookID, "initial", "i", flags.InitialBookID, "The book id you want to start download") 124 | f.BoolVarP(&flags.Rename, "rename", "r", flags.Rename, "Rename the book file by book id") 125 | f.IntVarP(&flags.Thread, "thread", "t", flags.Thread, "The number of download thead") 126 | f.IntVar(&flags.RateLimit, "ratelimit", flags.RateLimit, "The allowed requests per minutes for every thread") 127 | 128 | // Mark some flags as required. 129 | _ = talebookDownloadCmd.MarkFlagRequired("website") 130 | 131 | talebookCmd.AddCommand(talebookDownloadCmd) 132 | 133 | /// Add the register command. 134 | 135 | f = talebookRegisterCmd.Flags() 136 | 137 | // Add flags for registering. 138 | f.StringVarP(&flags.Username, "username", "u", flags.Username, "The talebook username") 139 | f.StringVarP(&flags.Password, "password", "p", flags.Password, "The talebook password") 140 | f.StringVarP(&flags.Email, "email", "e", flags.Email, "The talebook email") 141 | f.StringVarP(&flags.Website, "website", "w", flags.Website, "The talebook link") 142 | 143 | // Mark some flags as required. 144 | _ = talebookRegisterCmd.MarkFlagRequired("website") 145 | _ = talebookRegisterCmd.MarkFlagRequired("username") 146 | _ = talebookRegisterCmd.MarkFlagRequired("password") 147 | _ = talebookRegisterCmd.MarkFlagRequired("email") 148 | 149 | talebookCmd.AddCommand(talebookRegisterCmd) 150 | } 151 | -------------------------------------------------------------------------------- /cmd/telegram.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "strconv" 5 | "strings" 6 | 7 | "github.com/spf13/cobra" 8 | 9 | "github.com/bookstairs/bookhunter/cmd/flags" 10 | "github.com/bookstairs/bookhunter/internal/fetcher" 11 | "github.com/bookstairs/bookhunter/internal/log" 12 | ) 13 | 14 | // telegramCmd used for download books from the telegram channel 15 | var telegramCmd = &cobra.Command{ 16 | Use: "telegram", 17 | Short: "A tool for downloading books from telegram channel", 18 | Run: func(cmd *cobra.Command, args []string) { 19 | // Remove prefix for telegram. 20 | flags.Website = flags.ChannelID 21 | flags.ChannelID = strings.TrimPrefix(flags.ChannelID, "https://t.me/") 22 | 23 | // Print download configuration. 24 | log.NewPrinter(). 25 | Title("Telegram Download Information"). 26 | Head(log.DefaultHead...). 27 | Row("Config Path", flags.ConfigRoot). 28 | Row("Proxy", flags.Proxy). 29 | Row("Channel ID", flags.ChannelID). 30 | Row("Mobile", flags.HideSensitive(flags.Mobile)). 31 | Row("AppID", flags.HideSensitive(strconv.FormatInt(flags.AppID, 10))). 32 | Row("AppHash", flags.HideSensitive(flags.AppHash)). 33 | Row("Formats", flags.Formats). 34 | Row("Extract Archive", flags.Extract). 35 | Row("Download Path", flags.DownloadPath). 36 | Row("Initial ID", flags.InitialBookID). 37 | Row("Rename File", flags.Rename). 38 | Row("Thread", flags.Thread). 39 | Row("Keywords", flags.Keywords). 40 | Row("Thread Limit (req/min)", flags.RateLimit). 41 | Print() 42 | 43 | // Create the fetcher. 44 | f, err := flags.NewFetcher(fetcher.Telegram, map[string]string{ 45 | "channelID": flags.ChannelID, 46 | "mobile": flags.Mobile, 47 | "reLogin": strconv.FormatBool(flags.ReLogin), 48 | "appID": strconv.FormatInt(flags.AppID, 10), 49 | "appHash": flags.AppHash, 50 | }) 51 | log.Exit(err) 52 | 53 | // Wait all the threads have finished. 54 | err = f.Download() 55 | log.Exit(err) 56 | 57 | // Finished all the tasks. 58 | log.Info("Successfully download all the telegram books.") 59 | }, 60 | } 61 | 62 | func init() { 63 | f := telegramCmd.Flags() 64 | 65 | // Telegram download arguments. 66 | f.StringVarP(&flags.ChannelID, "channelID", "", flags.ChannelID, "The channel id for telegram") 67 | f.StringVarP(&flags.Mobile, "mobile", "", flags.Mobile, "The mobile number, we will add +86 as default zone code") 68 | f.BoolVar(&flags.ReLogin, "refresh", flags.ReLogin, "Refresh the login session") 69 | f.Int64Var(&flags.AppID, "appID", flags.AppID, "The app id for telegram") 70 | f.StringVar(&flags.AppHash, "appHash", flags.AppHash, "The app hash for telegram") 71 | 72 | // Common download flags. 73 | f.StringSliceVarP(&flags.Formats, "format", "f", flags.Formats, "The file formats you want to download") 74 | f.BoolVarP(&flags.Extract, "extract", "e", flags.Extract, "Extract the archive file for filtering") 75 | f.StringVarP(&flags.DownloadPath, "download", "d", flags.DownloadPath, "The book directory you want to use") 76 | f.Int64VarP(&flags.InitialBookID, "initial", "i", flags.InitialBookID, "The book id you want to start download") 77 | f.BoolVarP(&flags.Rename, "rename", "r", flags.Rename, "Rename the book file by book id") 78 | f.IntVarP(&flags.Thread, "thread", "t", flags.Thread, "The number of download thead") 79 | f.IntVar(&flags.RateLimit, "ratelimit", flags.RateLimit, "The allowed requests per minutes for every thread") 80 | 81 | // Bind the required arguments 82 | _ = telegramCmd.MarkFlagRequired("channelID") 83 | _ = telegramCmd.MarkFlagRequired("appID") 84 | _ = telegramCmd.MarkFlagRequired("appHash") 85 | } 86 | -------------------------------------------------------------------------------- /cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | 7 | "github.com/spf13/cobra" 8 | 9 | "github.com/bookstairs/bookhunter/internal/log" 10 | ) 11 | 12 | var ( 13 | gitVersion = "" 14 | gitCommit = "" // sha1 from git, output of $(git rev-parse HEAD) 15 | buildDate = "" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') 16 | goVersion = runtime.Version() 17 | platform = fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH) 18 | ) 19 | 20 | // versionCmd represents the version command. 21 | var versionCmd = &cobra.Command{ 22 | Use: "version", 23 | Short: "Return the bookhunter version info", 24 | Run: func(cmd *cobra.Command, args []string) { 25 | log.NewPrinter(). 26 | Title("bookhunter version info"). 27 | Row("Version", gitVersion). 28 | Row("Commit", gitCommit). 29 | Row("Build Date", buildDate). 30 | Row("Go Version", goVersion). 31 | Row("Platform", platform). 32 | Print() 33 | }, 34 | } 35 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/bookstairs/bookhunter 2 | 3 | go 1.23.3 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/PuerkitoBio/goquery v1.10.1 9 | github.com/bits-and-blooms/bitset v1.20.0 10 | github.com/corpix/uarand v0.2.0 11 | github.com/go-resty/resty/v2 v2.16.2 12 | github.com/gotd/contrib v0.21.0 13 | github.com/gotd/td v0.117.0 14 | github.com/jedib0t/go-pretty/v6 v6.6.5 15 | github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 16 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db 17 | github.com/schollz/progressbar/v3 v3.17.1 18 | github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e 19 | github.com/spf13/cobra v1.8.1 20 | github.com/stretchr/testify v1.10.0 21 | go.uber.org/ratelimit v0.3.1 22 | golang.org/x/net v0.33.0 23 | golang.org/x/term v0.28.0 24 | golang.org/x/text v0.21.0 25 | ) 26 | 27 | require ( 28 | github.com/andybalholm/cascadia v1.3.3 // indirect 29 | github.com/benbjohnson/clock v1.3.5 // indirect 30 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 31 | github.com/coder/websocket v1.8.12 // indirect 32 | github.com/davecgh/go-spew v1.1.1 // indirect 33 | github.com/dlclark/regexp2 v1.11.4 // indirect 34 | github.com/fatih/color v1.18.0 // indirect 35 | github.com/ghodss/yaml v1.0.0 // indirect 36 | github.com/go-faster/errors v0.7.1 // indirect 37 | github.com/go-faster/jx v1.1.0 // indirect 38 | github.com/go-faster/xor v1.0.0 // indirect 39 | github.com/go-faster/yaml v0.4.6 // indirect 40 | github.com/google/uuid v1.6.0 // indirect 41 | github.com/gotd/ige v0.2.2 // indirect 42 | github.com/gotd/neo v0.1.5 // indirect 43 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 44 | github.com/klauspost/compress v1.17.11 // indirect 45 | github.com/mattn/go-colorable v0.1.13 // indirect 46 | github.com/mattn/go-isatty v0.0.20 // indirect 47 | github.com/mattn/go-runewidth v0.0.16 // indirect 48 | github.com/ogen-go/ogen v1.8.1 // indirect 49 | github.com/pmezard/go-difflib v1.0.0 // indirect 50 | github.com/rivo/uniseg v0.4.7 // indirect 51 | github.com/segmentio/asm v1.2.0 // indirect 52 | github.com/spf13/pflag v1.0.5 // indirect 53 | go.opentelemetry.io/otel v1.33.0 // indirect 54 | go.opentelemetry.io/otel/metric v1.33.0 // indirect 55 | go.opentelemetry.io/otel/trace v1.33.0 // indirect 56 | go.uber.org/atomic v1.11.0 // indirect 57 | go.uber.org/multierr v1.11.0 // indirect 58 | go.uber.org/zap v1.27.0 // indirect 59 | golang.org/x/crypto v0.31.0 // indirect 60 | golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 // indirect 61 | golang.org/x/mod v0.22.0 // indirect 62 | golang.org/x/sync v0.10.0 // indirect 63 | golang.org/x/sys v0.29.0 // indirect 64 | golang.org/x/tools v0.28.0 // indirect 65 | gopkg.in/yaml.v2 v2.4.0 // indirect 66 | gopkg.in/yaml.v3 v3.0.1 // indirect 67 | rsc.io/qr v0.2.0 // indirect 68 | ) 69 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/PuerkitoBio/goquery v1.10.1 h1:Y8JGYUkXWTGRB6Ars3+j3kN0xg1YqqlwvdTV8WTFQcU= 2 | github.com/PuerkitoBio/goquery v1.10.1/go.mod h1:IYiHrOMps66ag56LEH7QYDDupKXyo5A8qrjIx3ZtujY= 3 | github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= 4 | github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= 5 | github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= 6 | github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= 7 | github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= 8 | github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= 9 | github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= 10 | github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= 11 | github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= 12 | github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= 13 | github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= 14 | github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= 15 | github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= 16 | github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM= 17 | github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 18 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 19 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 20 | github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= 21 | github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= 22 | github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= 23 | github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= 24 | github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= 25 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 26 | github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= 27 | github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= 28 | github.com/go-faster/jx v1.1.0 h1:ZsW3wD+snOdmTDy9eIVgQdjUpXRRV4rqW8NS3t+20bg= 29 | github.com/go-faster/jx v1.1.0/go.mod h1:vKDNikrKoyUmpzaJ0OkIkRQClNHFX/nF3dnTJZb3skg= 30 | github.com/go-faster/xor v0.3.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ= 31 | github.com/go-faster/xor v1.0.0 h1:2o8vTOgErSGHP3/7XwA5ib1FTtUsNtwCoLLBjl31X38= 32 | github.com/go-faster/xor v1.0.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ= 33 | github.com/go-faster/yaml v0.4.6 h1:lOK/EhI04gCpPgPhgt0bChS6bvw7G3WwI8xxVe0sw9I= 34 | github.com/go-faster/yaml v0.4.6/go.mod h1:390dRIvV4zbnO7qC9FGo6YYutc+wyyUSHBgbXL52eXk= 35 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 36 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 37 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 38 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 39 | github.com/go-resty/resty/v2 v2.16.2 h1:CpRqTjIzq/rweXUt9+GxzzQdlkqMdt8Lm/fuK/CAbAg= 40 | github.com/go-resty/resty/v2 v2.16.2/go.mod h1:0fHAoK7JoBy/Ch36N8VFeMsK7xQOHhvWaC3iOktwmIU= 41 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 42 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 43 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 44 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 45 | github.com/gotd/contrib v0.21.0 h1:4Fj05jnyBE84toXZl7mVTvt7f732n5uglvztyG6nTr4= 46 | github.com/gotd/contrib v0.21.0/go.mod h1:ENoUh75IhHGxfz/puVJg8BU4ZF89yrL6Q47TyoNqFYo= 47 | github.com/gotd/ige v0.2.2 h1:XQ9dJZwBfDnOGSTxKXBGP4gMud3Qku2ekScRjDWWfEk= 48 | github.com/gotd/ige v0.2.2/go.mod h1:tuCRb+Y5Y3eNTo3ypIfNpQ4MFjrnONiL2jN2AKZXmb0= 49 | github.com/gotd/neo v0.1.5 h1:oj0iQfMbGClP8xI59x7fE/uHoTJD7NZH9oV1WNuPukQ= 50 | github.com/gotd/neo v0.1.5/go.mod h1:9A2a4bn9zL6FADufBdt7tZt+WMhvZoc5gWXihOPoiBQ= 51 | github.com/gotd/td v0.117.0 h1:Z6vU5thb5DW/I1s0sLSeSfA/QWvwszx6SxHhEEYJiU8= 52 | github.com/gotd/td v0.117.0/go.mod h1:jf1Zf1ViTN+H1x8dhDTCBHOYY/2E/40HsyOsohxqXYA= 53 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 54 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 55 | github.com/jedib0t/go-pretty/v6 v6.6.5 h1:9PgMJOVBedpgYLI56jQRJYqngxYAAzfEUua+3NgSqAo= 56 | github.com/jedib0t/go-pretty/v6 v6.6.5/go.mod h1:Uq/HrbhuFty5WSVNfjpQQe47x16RwVGXIveNGEyGtHs= 57 | github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= 58 | github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= 59 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= 60 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 61 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 62 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 63 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 64 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 65 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 66 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 67 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 68 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 69 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 70 | github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= 71 | github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 72 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= 73 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= 74 | github.com/ogen-go/ogen v1.8.1 h1:7TZ+oIeLkcBiyl0qu0fHPrFUrGWDj3Fi/zKSWg2i2Tg= 75 | github.com/ogen-go/ogen v1.8.1/go.mod h1:2ShRm6u/nXUHuwdVKv2SeaG8enBKPKAE3kSbHwwFh6o= 76 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 77 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 78 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 79 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 80 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 81 | github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= 82 | github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= 83 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 84 | github.com/schollz/progressbar/v3 v3.17.1 h1:bI1MTaoQO+v5kzklBjYNRQLoVpe0zbyRZNK6DFkVC5U= 85 | github.com/schollz/progressbar/v3 v3.17.1/go.mod h1:RzqpnsPQNjUyIgdglUjRLgD7sVnxN1wpmBMV+UiEbL4= 86 | github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= 87 | github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= 88 | github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= 89 | github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= 90 | github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= 91 | github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= 92 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 93 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 94 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 95 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 96 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 97 | go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= 98 | go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= 99 | go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= 100 | go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= 101 | go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= 102 | go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= 103 | go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= 104 | go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= 105 | go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= 106 | go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= 107 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 108 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 109 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 110 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 111 | go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= 112 | go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= 113 | go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= 114 | go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= 115 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 116 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 117 | golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= 118 | golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= 119 | golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= 120 | golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= 121 | golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= 122 | golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 h1:9kj3STMvgqy3YA4VQXBrN7925ICMxD5wzMRcgA30588= 123 | golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= 124 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 125 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 126 | golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 127 | golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= 128 | golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= 129 | golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= 130 | golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= 131 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 132 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 133 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 134 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 135 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 136 | golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= 137 | golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= 138 | golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= 139 | golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= 140 | golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= 141 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 142 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 143 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 144 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 145 | golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= 146 | golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 147 | golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 148 | golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= 149 | golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 150 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 151 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 152 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 153 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 154 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 155 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 156 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 157 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 158 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 159 | golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 160 | golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 161 | golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 162 | golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 163 | golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= 164 | golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 165 | golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= 166 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 167 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 168 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 169 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= 170 | golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= 171 | golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= 172 | golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= 173 | golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= 174 | golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= 175 | golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= 176 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 177 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 178 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 179 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 180 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 181 | golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= 182 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 183 | golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 184 | golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= 185 | golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= 186 | golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= 187 | golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= 188 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 189 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 190 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 191 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 192 | golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= 193 | golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= 194 | golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= 195 | golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= 196 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 197 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 198 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 199 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 200 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 201 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 202 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 203 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 204 | nhooyr.io/websocket v1.8.17 h1:KEVeLJkUywCKVsnLIDlD/5gtayKp8VoCkksHCGGfT9Y= 205 | nhooyr.io/websocket v1.8.17/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= 206 | rsc.io/qr v0.2.0 h1:6vBLea5/NRMVTz8V66gipeLycZMl/+UlFmk8DvqQ6WY= 207 | rsc.io/qr v0.2.0/go.mod h1:IF+uZjkb9fqyeF/4tlBoynqmQxUoPfWEKh921coOuXs= 208 | -------------------------------------------------------------------------------- /internal/client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http/cookiejar" 7 | "net/url" 8 | "os" 9 | "path/filepath" 10 | "time" 11 | 12 | "github.com/corpix/uarand" 13 | "github.com/go-resty/resty/v2" 14 | 15 | "github.com/bookstairs/bookhunter/internal/log" 16 | ) 17 | 18 | var ( 19 | ErrInvalidRequestURL = errors.New("invalid request url, we only support https:// or http://") 20 | ) 21 | 22 | // Client is the wrapper for resty.Client we may provide extra method on this wrapper. 23 | type Client struct { 24 | *resty.Client 25 | *Config 26 | } 27 | 28 | // Config is the basic configuration for creating the client. 29 | type Config struct { 30 | HTTPS bool // If the request was under the https of http. 31 | Host string // The request host name. 32 | Proxy string // The proxy address, such as the http://127.0.0.1:7890, socks://127.0.0.1:7890 33 | ConfigRoot string // The root config path for the whole bookhunter download service. 34 | 35 | // The custom redirect function. 36 | Redirect resty.RedirectPolicy `json:"-"` 37 | } 38 | 39 | // ConfigPath will return a unique path for this download service. 40 | func (c *Config) ConfigPath() (string, error) { 41 | if c.ConfigRoot == "" { 42 | var err error 43 | c.ConfigRoot, err = DefaultConfigRoot() 44 | if err != nil { 45 | return "", err 46 | } 47 | } 48 | 49 | return mkdir(filepath.Join(c.ConfigRoot, c.Host)) 50 | } 51 | 52 | func (c *Config) redirectPolicy() []any { 53 | policies := []any{ 54 | resty.FlexibleRedirectPolicy(5), 55 | } 56 | if c.Redirect != nil { 57 | policies = append(policies, c.Redirect) 58 | } 59 | 60 | return policies 61 | } 62 | 63 | func (c *Config) baseURL() string { 64 | if c.HTTPS { 65 | return "https://" + c.Host 66 | } 67 | 68 | return "http://" + c.Host 69 | } 70 | 71 | func (c *Client) SetDefaultHostname(host string) { 72 | c.Host = host 73 | c.Client.SetBaseURL(c.baseURL()) 74 | } 75 | 76 | func (c *Client) CleanCookies() { 77 | jar, _ := cookiejar.New(nil) 78 | c.SetCookieJar(jar) 79 | } 80 | 81 | // DefaultConfigRoot will generate the default config path based on the user and his running environment. 82 | func DefaultConfigRoot() (string, error) { 83 | home, err := os.UserHomeDir() 84 | if err != nil { 85 | return "", err 86 | } 87 | 88 | return mkdir(filepath.Join(home, ".config", "bookhunter")) 89 | } 90 | 91 | func mkdir(path string) (string, error) { 92 | if err := os.MkdirAll(path, 0o755); err != nil { 93 | return "", err 94 | } 95 | 96 | return path, nil 97 | } 98 | 99 | // NewConfig will create a config instance by using the request url. 100 | func NewConfig(rawURL, proxy, configRoot string) (*Config, error) { 101 | u, err := url.Parse(rawURL) 102 | if err != nil { 103 | return nil, fmt.Errorf(rawURL, err) 104 | } 105 | 106 | if u.Scheme != "http" && u.Scheme != "https" { 107 | return nil, ErrInvalidRequestURL 108 | } 109 | 110 | if configRoot == "" { 111 | configRoot, err = DefaultConfigRoot() 112 | if err != nil { 113 | return nil, err 114 | } 115 | } else { 116 | if err := os.MkdirAll(configRoot, 0o755); err != nil { 117 | return nil, err 118 | } 119 | } 120 | 121 | return &Config{ 122 | HTTPS: u.Scheme == "https", 123 | Host: u.Host, 124 | Proxy: proxy, 125 | ConfigRoot: configRoot, 126 | }, nil 127 | } 128 | 129 | // New will create a resty client with a lot of predefined settings. 130 | func New(c *Config) (*Client, error) { 131 | client := resty.New(). 132 | SetRetryCount(3). 133 | SetRetryWaitTime(3*time.Second). 134 | SetRetryMaxWaitTime(10*time.Second). 135 | SetAllowGetMethodPayload(true). 136 | SetTimeout(5*time.Minute). 137 | SetContentLength(true). 138 | SetHeader("User-Agent", uarand.GetRandom()) 139 | 140 | if !log.EnableDebug { 141 | client.DisableTrace().SetDebug(false).SetDisableWarn(true) 142 | } else { 143 | client.SetDebug(true).SetDisableWarn(false) 144 | } 145 | 146 | if c.Host != "" { 147 | client.SetBaseURL(c.baseURL()) 148 | } 149 | 150 | if len(c.redirectPolicy()) > 0 { 151 | client.SetRedirectPolicy(c.redirectPolicy()...) 152 | } 153 | 154 | // Setting the proxy for the resty client. 155 | // The proxy environment is also supported. 156 | if c.Proxy != "" { 157 | client.SetProxy(c.Proxy) 158 | } 159 | 160 | return &Client{Client: client, Config: c}, nil 161 | } 162 | -------------------------------------------------------------------------------- /internal/driver/aliyun.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "io" 5 | "strings" 6 | 7 | "github.com/bookstairs/bookhunter/internal/client" 8 | "github.com/bookstairs/bookhunter/internal/driver/aliyun" 9 | ) 10 | 11 | // newAliyunDriver will create the aliyun driver. 12 | func newAliyunDriver(c *client.Config, properties map[string]string) (Driver, error) { 13 | // Create the aliyun client. 14 | a, err := aliyun.New(c, properties["refreshToken"]) 15 | if err != nil { 16 | return nil, err 17 | } 18 | 19 | return &aliyunDriver{client: a}, nil 20 | } 21 | 22 | type aliyunDriver struct { 23 | client *aliyun.Aliyun 24 | } 25 | 26 | func (a *aliyunDriver) Source() Source { 27 | return ALIYUN 28 | } 29 | 30 | func (a *aliyunDriver) Resolve(link, passcode string) ([]Share, error) { 31 | shareID := strings.TrimPrefix(link, "https://www.aliyundrive.com/s/") 32 | sharePwd := strings.TrimSpace(passcode) 33 | 34 | token, err := a.client.ShareToken(shareID, sharePwd) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | files, err := a.client.Share(shareID, token.ShareToken) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | var shares []Share 45 | for index := range files { 46 | item := files[index] 47 | share := Share{ 48 | FileName: item.Name, 49 | Size: int64(item.Size), 50 | URL: item.FileID, 51 | Properties: map[string]any{ 52 | "shareToken": token.ShareToken, 53 | "shareID": shareID, 54 | "fileID": item.FileID, 55 | }, 56 | } 57 | 58 | shares = append(shares, share) 59 | } 60 | 61 | return shares, nil 62 | } 63 | 64 | func (a *aliyunDriver) Download(share Share) (io.ReadCloser, int64, error) { 65 | shareToken := share.Properties["shareToken"].(string) 66 | shareID := share.Properties["shareID"].(string) 67 | fileID := share.Properties["fileID"].(string) 68 | 69 | url, err := a.client.DownloadURL(shareToken, shareID, fileID) 70 | if err != nil { 71 | return nil, 0, err 72 | } 73 | 74 | file, err := a.client.DownloadFile(url) 75 | 76 | return file, 0, err 77 | } 78 | -------------------------------------------------------------------------------- /internal/driver/aliyun/auth.go: -------------------------------------------------------------------------------- 1 | package aliyun 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "net/http" 9 | "net/url" 10 | "os" 11 | "path/filepath" 12 | "strconv" 13 | "time" 14 | 15 | "github.com/go-resty/resty/v2" 16 | "github.com/skip2/go-qrcode" 17 | 18 | "github.com/bookstairs/bookhunter/internal/client" 19 | "github.com/bookstairs/bookhunter/internal/log" 20 | ) 21 | 22 | var ( 23 | // Token will be refreshed before this time. 24 | acceleratedExpirationDuration = 10 * time.Minute 25 | headerAuthorization = http.CanonicalHeaderKey("Authorization") 26 | ) 27 | 28 | // RefreshToken can only be called after the New method. 29 | func (ali *Aliyun) RefreshToken() string { 30 | return ali.authentication.refreshToken 31 | } 32 | 33 | func newAuthentication(c *client.Config, refreshToken string) (*authentication, error) { 34 | // Create session file. 35 | path, err := c.ConfigPath() 36 | if err != nil { 37 | return nil, err 38 | } 39 | sessionFile := filepath.Join(path, "session.db") 40 | open, err := os.OpenFile(sessionFile, os.O_RDONLY|os.O_CREATE, 0666) 41 | if err != nil { 42 | log.Exit(err) 43 | } 44 | _ = open.Close() 45 | 46 | cl, err := client.New(c) 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | return &authentication{ 52 | Client: cl, 53 | sessionFile: sessionFile, 54 | refreshToken: refreshToken, 55 | }, nil 56 | } 57 | 58 | type authentication struct { 59 | *client.Client 60 | sessionFile string 61 | refreshToken string 62 | tokenCache *TokenResp 63 | } 64 | 65 | func (auth *authentication) authenticationHook() resty.PreRequestHook { 66 | return func(_ *resty.Client, req *http.Request) error { 67 | if req.Header.Get("x-empty-content-type") != "" { 68 | req.Header.Del("x-empty-content-type") 69 | req.Header.Set("content-type", "") 70 | } 71 | 72 | // Refresh the token and set it on the header. 73 | token, err := auth.accessToken() 74 | if err != nil { 75 | return err 76 | } 77 | req.Header.Set(headerAuthorization, "Bearer "+token) 78 | 79 | return nil 80 | } 81 | } 82 | 83 | func (auth *authentication) Auth() error { 84 | // Check the given token if it's valid. 85 | if auth.validateRefreshToken() { 86 | return nil 87 | } 88 | 89 | // The given token isn't valid. Try to load the token from file. 90 | b, err := os.ReadFile(auth.sessionFile) 91 | if err != nil { 92 | return err 93 | } 94 | auth.refreshToken = string(b) 95 | if auth.validateRefreshToken() { 96 | return nil 97 | } 98 | 99 | // If the token doesn't exist. We will get the refreshToken by QR code. 100 | if err := auth.login(); err != nil { 101 | return err 102 | } 103 | _ = auth.validateRefreshToken() 104 | 105 | return nil 106 | } 107 | 108 | func (auth *authentication) login() error { 109 | // Set the OAuth2 request into cookies. 110 | _, err := auth.R(). 111 | SetQueryParams(map[string]string{ 112 | "client_id": "25dzX3vbYqktVxyX", 113 | "redirect_uri": "https://www.aliyundrive.com/sign/callback", 114 | "response_type": "code", 115 | "login_type": "custom", 116 | "state": `{"origin":"https://www.aliyundrive.com"}`, 117 | }). 118 | Get("https://auth.aliyundrive.com/v2/oauth/authorize") 119 | if err != nil { 120 | return err 121 | } 122 | 123 | // Get the QR code. 124 | resp, err := auth.R(). 125 | SetQueryParams(map[string]string{ 126 | "appName": "aliyun_drive", 127 | "fromSite": "52", 128 | "appEntrance": "web", 129 | "isMobile": "false", 130 | "lang": "zh_CN", 131 | "_bx-v": "2.0.31", 132 | "returnUrl": "", 133 | "bizParams": "", 134 | }). 135 | SetResult(&QRCodeResp{}). 136 | Get("https://passport.aliyundrive.com/newlogin/qrcode/generate.do") 137 | if err != nil { 138 | return err 139 | } 140 | qr := resp.Result().(*QRCodeResp).Content.Data 141 | if msg := qr.TitleMsg; msg != "" { 142 | return errors.New(msg) 143 | } 144 | 145 | // Print the QR code into console. 146 | code, _ := qrcode.New(qr.CodeContent, qrcode.Low) 147 | fmt.Println() 148 | fmt.Println(code.ToSmallString(false)) 149 | log.Info("Use Aliyun Drive App to scan this QR code.") 150 | 151 | // Wait for the scan result. 152 | scanned := false 153 | for { 154 | // NEW / SCANED / EXPIRED / CANCELED / CONFIRMED 155 | resp, err := auth.queryQRCode(qr.T, qr.Ck) 156 | if err != nil { 157 | return err 158 | } 159 | res := resp.Content.Data 160 | 161 | switch res.QrCodeStatus { 162 | case "NEW": 163 | case "SCANED": 164 | if !scanned { 165 | log.Info("Scan success. Confirm login on your mobile.") 166 | } 167 | scanned = true 168 | case "EXPIRED": 169 | return fmt.Errorf("the QR code expired") 170 | case "CANCELED": 171 | return fmt.Errorf("user canceled login") 172 | case "CONFIRMED": 173 | biz := res.BizAction.PdsLoginResult 174 | if err := auth.confirmLogin(biz.AccessToken); err != nil { 175 | return err 176 | } 177 | auth.refreshToken = biz.RefreshToken 178 | return nil 179 | default: 180 | return fmt.Errorf("%v", res) 181 | } 182 | 183 | // Sleep one second. 184 | time.Sleep(time.Second) 185 | } 186 | } 187 | 188 | func (auth *authentication) queryQRCode(t int64, ck string) (*QueryQRCodeResp, error) { 189 | resp, err := auth.R(). 190 | SetQueryParams(map[string]string{ 191 | "appName": "aliyun_drive", 192 | "fromSite": "52", 193 | "_bx-v": "2.0.31", 194 | }). 195 | SetFormData(map[string]string{ 196 | "t": strconv.FormatInt(t, 10), 197 | "ck": ck, 198 | "appName": "aliyun_drive", 199 | "appEntrance": "web", 200 | "isMobile": "false", 201 | "lang": "zh_CN", 202 | "returnUrl": "", 203 | "fromSite": "52", 204 | "bizParams": "", 205 | "navlanguage": "zh-CN", 206 | "navPlatform": "MacIntel", 207 | }). 208 | SetResult(&QueryQRCodeResp{}). 209 | Post("https://passport.aliyundrive.com/newlogin/qrcode/query.do") 210 | if err != nil { 211 | return nil, err 212 | } 213 | 214 | res := resp.Result().(*QueryQRCodeResp) 215 | if res.Content.Data.BizExt != "" { 216 | bs, _ := base64.StdEncoding.DecodeString(res.Content.Data.BizExt) 217 | _ = json.Unmarshal(bs, &res.Content.Data.BizAction) 218 | } 219 | 220 | return res, nil 221 | } 222 | 223 | func (auth *authentication) confirmLogin(accessToken string) error { 224 | resp, err := auth.R(). 225 | SetBody(map[string]string{ 226 | "token": accessToken, 227 | }). 228 | SetResult(&ConfirmLoginResp{}). 229 | Post("https://auth.aliyundrive.com/v2/oauth/token_login") 230 | if err != nil { 231 | return err 232 | } 233 | jump := resp.Result().(*ConfirmLoginResp).Goto 234 | 235 | _, err = auth.R().Get(jump) 236 | if err != nil { 237 | return err 238 | } 239 | 240 | gotoURL, _ := url.Parse(jump) 241 | if gotoURL != nil && gotoURL.Query().Has("code") { 242 | code := gotoURL.Query().Get("code") 243 | _, err := auth.getToken(code) 244 | if err != nil { 245 | return err 246 | } else { 247 | log.Info("Successfully login.") 248 | } 249 | } 250 | 251 | return nil 252 | } 253 | 254 | func (auth *authentication) getToken(code string) (*TokenResp, error) { 255 | resp, err := auth.R(). 256 | SetBody(map[string]string{ 257 | "code": code, 258 | "loginType": "normal", 259 | "deviceId": "aliyundrive", 260 | }). 261 | SetResult(&TokenResp{}). 262 | Get("https://api.aliyundrive.com/token/get") 263 | if err != nil { 264 | return nil, err 265 | } 266 | 267 | return resp.Result().(*TokenResp), nil 268 | } 269 | 270 | func (auth *authentication) validateRefreshToken() bool { 271 | if auth.refreshToken != "" { 272 | _, err := auth.accessToken() 273 | if err != nil { 274 | log.Fatal(err) 275 | return false 276 | } 277 | return true 278 | } 279 | return false 280 | } 281 | 282 | // accessToken will return the token by the given refreshToken. 283 | // You can call this method for automatically refreshing the access token. 284 | func (auth *authentication) accessToken() (string, error) { 285 | // Get the token from the cache or refreshToken. 286 | if auth.tokenCache == nil || time.Now().Add(acceleratedExpirationDuration).After(auth.tokenCache.ExpireTime) { 287 | // Refresh the access token by the refresh token. 288 | resp, err := auth.R(). 289 | SetBody(&TokenReq{GrantType: "refresh_token", RefreshToken: auth.refreshToken}). 290 | SetResult(&TokenResp{}). 291 | SetError(&ErrorResp{}). 292 | Post("https://auth.aliyundrive.com/v2/account/token") 293 | if err != nil { 294 | return "", err 295 | } 296 | if e, ok := resp.Error().(*ErrorResp); ok { 297 | return "", fmt.Errorf("token: %s, %s", e.Code, e.Message) 298 | } 299 | 300 | // Cache the token. 301 | auth.tokenCache = resp.Result().(*TokenResp) 302 | auth.refreshToken = auth.tokenCache.RefreshToken 303 | 304 | // Persist the refresh token if it's valid. 305 | if err := os.WriteFile(auth.sessionFile, []byte(auth.refreshToken), 0o644); err != nil { 306 | return "", err 307 | } 308 | } 309 | 310 | return auth.tokenCache.AccessToken, nil 311 | } 312 | -------------------------------------------------------------------------------- /internal/driver/aliyun/common.go: -------------------------------------------------------------------------------- 1 | package aliyun 2 | 3 | import ( 4 | "github.com/bookstairs/bookhunter/internal/client" 5 | ) 6 | 7 | type Aliyun struct { 8 | *client.Client 9 | authentication *authentication 10 | } 11 | 12 | // New will create an aliyun download service. 13 | func New(c *client.Config, refreshToken string) (*Aliyun, error) { 14 | c = &client.Config{ 15 | HTTPS: true, 16 | Host: "api.aliyundrive.com", 17 | Proxy: c.Proxy, 18 | ConfigRoot: c.ConfigRoot, 19 | } 20 | 21 | authentication, err := newAuthentication(c, refreshToken) 22 | if err != nil { 23 | return nil, err 24 | } 25 | if err := authentication.Auth(); err != nil { 26 | return nil, err 27 | } 28 | 29 | cl, err := client.New(c) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | // Set extra middleware for cleaning up the header and authentication. 35 | cl.SetPreRequestHook(authentication.authenticationHook()) 36 | 37 | return &Aliyun{Client: cl, authentication: authentication}, nil 38 | } 39 | -------------------------------------------------------------------------------- /internal/driver/aliyun/response.go: -------------------------------------------------------------------------------- 1 | package aliyun 2 | 3 | import "time" 4 | 5 | type ErrorResp struct { 6 | Code string `json:"code"` 7 | Message string `json:"message"` 8 | } 9 | 10 | type QRCodeResp struct { 11 | Content struct { 12 | Data struct { 13 | TitleMsg string `json:"title_msg"` 14 | T int64 `json:"t"` 15 | CodeContent string `json:"codeContent"` 16 | Ck string `json:"ck"` 17 | ResultCode int `json:"resultCode"` 18 | } `json:"data"` 19 | } `json:"content"` 20 | } 21 | 22 | type QueryQRCodeResp struct { 23 | Content struct { 24 | Data struct { 25 | QrCodeStatus string `json:"qrCodeStatus"` 26 | ResultCode int `json:"resultCode"` 27 | LoginResult string `json:"loginResult"` 28 | LoginSucResultAction string `json:"loginSucResultAction"` 29 | BizAction struct { 30 | PdsLoginResult struct { 31 | Role string `json:"role"` 32 | IsFirstLogin bool `json:"isFirstLogin"` 33 | NeedLink bool `json:"needLink"` 34 | LoginType string `json:"loginType"` 35 | NickName string `json:"nickName"` 36 | NeedRpVerify bool `json:"needRpVerify"` 37 | Avatar string `json:"avatar"` 38 | AccessToken string `json:"accessToken"` 39 | UserName string `json:"userName"` 40 | UserID string `json:"userId"` 41 | DefaultDriveID string `json:"defaultDriveId"` 42 | ExistLink []interface{} `json:"existLink"` 43 | ExpiresIn int `json:"expiresIn"` 44 | ExpireTime time.Time `json:"expireTime"` 45 | RequestID string `json:"requestId"` 46 | DataPinSetup bool `json:"dataPinSetup"` 47 | State string `json:"state"` 48 | TokenType string `json:"tokenType"` 49 | DataPinSaved bool `json:"dataPinSaved"` 50 | RefreshToken string `json:"refreshToken"` 51 | Status string `json:"status"` 52 | } `json:"pds_login_result"` 53 | } `json:"-"` 54 | St string `json:"st"` 55 | LoginType string `json:"loginType"` 56 | BizExt string `json:"bizExt"` 57 | LoginScene string `json:"loginScene"` 58 | AppEntrance string `json:"appEntrance"` 59 | Smartlock bool `json:"smartlock"` 60 | } `json:"data"` 61 | } `json:"content"` 62 | } 63 | 64 | type ConfirmLoginResp struct { 65 | Goto string `json:"goto"` 66 | } 67 | 68 | type TokenReq struct { 69 | GrantType string `json:"grant_type"` 70 | RefreshToken string `json:"refresh_token"` 71 | } 72 | 73 | type TokenResp struct { 74 | DefaultSboxDriveID string `json:"default_sbox_drive_id"` 75 | Role string `json:"role"` 76 | DeviceID string `json:"device_id"` 77 | UserName string `json:"user_name"` 78 | NeedLink bool `json:"need_link"` 79 | ExpireTime time.Time `json:"expire_time"` 80 | PinSetup bool `json:"pin_setup"` 81 | NeedRpVerify bool `json:"need_rp_verify"` 82 | Avatar string `json:"avatar"` 83 | TokenType string `json:"token_type"` 84 | AccessToken string `json:"access_token"` 85 | DefaultDriveID string `json:"default_drive_id"` 86 | DomainID string `json:"domain_id"` 87 | RefreshToken string `json:"refresh_token"` 88 | IsFirstLogin bool `json:"is_first_login"` 89 | UserID string `json:"user_id"` 90 | NickName string `json:"nick_name"` 91 | State string `json:"state"` 92 | ExpiresIn int `json:"expires_in"` 93 | Status string `json:"status"` 94 | } 95 | 96 | type ShareInfoReq struct { 97 | ShareID string `json:"share_id"` 98 | } 99 | 100 | type ShareInfoResp struct { 101 | Avatar string `json:"avatar"` 102 | CreatorID string `json:"creator_id"` 103 | CreatorName string `json:"creator_name"` 104 | CreatorPhone string `json:"creator_phone"` 105 | Expiration string `json:"expiration"` 106 | UpdatedAt string `json:"updated_at"` 107 | ShareName string `json:"share_name"` 108 | FileCount int `json:"file_count"` 109 | FileInfos []ShareItemInfo `json:"file_infos"` 110 | Vip string `json:"vip"` 111 | DisplayName string `json:"display_name"` 112 | IsFollowingCreator bool `json:"is_following_creator"` 113 | } 114 | 115 | type ShareItemInfo struct { 116 | Category string `json:"category"` 117 | FileExtension string `json:"file_extension"` 118 | FileID string `json:"file_id"` 119 | Thumbnail string `json:"thumbnail"` 120 | FileType string `json:"type"` 121 | } 122 | 123 | type ShareLinkDownloadURLReq struct { 124 | ShareID string `json:"share_id"` 125 | FileID string `json:"file_id"` 126 | ExpireSec int `json:"expire_sec"` 127 | } 128 | 129 | type ShareLinkDownloadURLResp struct { 130 | DownloadURL string `json:"download_url"` 131 | URL string `json:"url"` 132 | Thumbnail string `json:"thumbnail"` 133 | } 134 | 135 | type ShareTokenReq struct { 136 | ShareID string `json:"share_id"` 137 | SharePwd string `json:"share_pwd"` 138 | } 139 | 140 | type ShareTokenResp struct { 141 | ShareToken string `json:"share_token"` 142 | ExpireTime string `json:"expire_time"` 143 | ExpiresIn int `json:"expires_in"` 144 | } 145 | 146 | type ShareFileListReq struct { 147 | ShareID string `json:"share_id"` 148 | Starred bool `json:"starred"` 149 | All bool `json:"all"` 150 | Category string `json:"category"` 151 | Fields string `json:"fields"` 152 | ImageThumbnailProcess string `json:"image_thumbnail_process"` 153 | Limit int `json:"limit"` 154 | Marker string `json:"marker"` 155 | OrderBy string `json:"order_by"` 156 | OrderDirection string `json:"order_direction"` 157 | ParentFileID string `json:"parent_file_id"` 158 | Status string `json:"status"` 159 | FileType string `json:"type"` 160 | URLExpireSec int `json:"url_expire_sec"` 161 | VideoThumbnailProcess string `json:"video_thumbnail_process"` 162 | } 163 | 164 | type ShareFileListResp struct { 165 | Items []*ShareFile `json:"items"` 166 | NextMarker string `json:"next_marker"` 167 | } 168 | 169 | type ShareFile struct { 170 | ShareID string `json:"share_id"` 171 | Name string `json:"name"` 172 | Size int `json:"size"` 173 | Creator string `json:"creator"` 174 | Description string `json:"description"` 175 | Category string `json:"category"` 176 | DownloadURL int `json:"download_url"` 177 | URL int `json:"url"` 178 | FileExtension string `json:"file_extension"` 179 | FileID string `json:"file_id"` 180 | Thumbnail string `json:"thumbnail"` 181 | ParentFileID string `json:"parent_file_id"` 182 | FileType string `json:"type"` 183 | UpdatedAt string `json:"updated_at"` 184 | CreatedAt string `json:"created_at"` 185 | Selected string `json:"selected"` 186 | MimeExtension string `json:"mime_extension"` 187 | MimeType string `json:"mime_type"` 188 | PunishFlag int `json:"punish_flag"` 189 | ActionList []string `json:"action_list"` 190 | DriveID string `json:"drive_id"` 191 | DomainID string `json:"domain_id"` 192 | RevisionID string `json:"revision_id"` 193 | } 194 | 195 | // listShareFilesParam is used in file list query context. 196 | type listShareFilesParam struct { 197 | shareToken string 198 | shareID string 199 | parentFileID string 200 | marker string 201 | } 202 | -------------------------------------------------------------------------------- /internal/driver/aliyun/share.go: -------------------------------------------------------------------------------- 1 | package aliyun 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/bookstairs/bookhunter/internal/log" 7 | ) 8 | 9 | // AnonymousShare will try to access the share without the user information. 10 | func (ali *Aliyun) AnonymousShare(shareID string) (*ShareInfoResp, error) { 11 | resp, err := ali.R(). 12 | SetBody(&ShareInfoReq{ShareID: shareID}). 13 | SetResult(&ShareInfoResp{}). 14 | SetError(&ErrorResp{}). 15 | Post("https://api.aliyundrive.com/adrive/v2/share_link/get_share_by_anonymous") 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | return resp.Result().(*ShareInfoResp), nil 21 | } 22 | 23 | func (ali *Aliyun) Share(shareID, shareToken string) ([]ShareFile, error) { 24 | return ali.listShareFiles(&listShareFilesParam{ 25 | shareToken: shareToken, 26 | shareID: shareID, 27 | parentFileID: "root", 28 | marker: "", 29 | }) 30 | } 31 | 32 | func (ali *Aliyun) listShareFiles(param *listShareFilesParam) ([]ShareFile, error) { 33 | resp, err := ali.R(). 34 | SetHeader("x-share-token", param.shareToken). 35 | SetBody(&ShareFileListReq{ 36 | ShareID: param.shareID, 37 | ParentFileID: param.parentFileID, 38 | URLExpireSec: 14400, 39 | OrderBy: "name", 40 | OrderDirection: "DESC", 41 | Limit: 20, 42 | Marker: param.marker, 43 | }). 44 | SetResult(&ShareFileListResp{}). 45 | SetError(&ErrorResp{}). 46 | Post("https://api.aliyundrive.com/adrive/v3/file/list") 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | res := resp.Result().(*ShareFileListResp) 52 | var files []ShareFile 53 | 54 | for _, item := range res.Items { 55 | if item.FileType == "folder" { 56 | list, err := ali.listShareFiles(&listShareFilesParam{ 57 | shareToken: param.shareToken, 58 | shareID: param.shareID, 59 | parentFileID: item.FileID, 60 | marker: "", 61 | }) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | files = append(files, list...) 67 | } else { 68 | files = append(files, *item) 69 | } 70 | } 71 | 72 | if res.NextMarker != "" { 73 | list, err := ali.listShareFiles(&listShareFilesParam{ 74 | shareToken: param.shareToken, 75 | shareID: param.shareID, 76 | parentFileID: param.parentFileID, 77 | marker: res.NextMarker, 78 | }) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | files = append(files, list...) 84 | } 85 | 86 | return files, nil 87 | } 88 | 89 | func (ali *Aliyun) ShareToken(shareID, sharePwd string) (*ShareTokenResp, error) { 90 | resp, err := ali.R(). 91 | SetBody(&ShareTokenReq{ShareID: shareID, SharePwd: sharePwd}). 92 | SetResult(&ShareTokenResp{}). 93 | SetError(&ErrorResp{}). 94 | Post("https://auth.aliyundrive.com/v2/share_link/get_share_token") 95 | if err != nil { 96 | return nil, err 97 | } 98 | 99 | return resp.Result().(*ShareTokenResp), nil 100 | } 101 | 102 | func (ali *Aliyun) DownloadURL(shareToken, shareID, fileID string) (string, error) { 103 | resp, err := ali.R(). 104 | SetHeader("x-share-token", shareToken). 105 | SetBody(&ShareLinkDownloadURLReq{ 106 | ShareID: shareID, 107 | FileID: fileID, 108 | // Only ten minutes valid 109 | ExpireSec: 600, 110 | }). 111 | SetResult(&ShareLinkDownloadURLResp{}). 112 | SetError(&ErrorResp{}). 113 | Post("https://api.aliyundrive.com/v2/file/get_share_link_download_url") 114 | if err != nil { 115 | return "", err 116 | } 117 | 118 | res := resp.Result().(*ShareLinkDownloadURLResp) 119 | return res.DownloadURL, nil 120 | } 121 | 122 | func (ali *Aliyun) DownloadFile(downloadURL string) (io.ReadCloser, error) { 123 | log.Debugf("Start to download file from aliyun drive: %s", downloadURL) 124 | 125 | resp, err := ali.R(). 126 | SetDoNotParseResponse(true). 127 | Get(downloadURL) 128 | if err != nil { 129 | return nil, err 130 | } 131 | 132 | return resp.RawBody(), err 133 | } 134 | -------------------------------------------------------------------------------- /internal/driver/common.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/bookstairs/bookhunter/internal/client" 8 | ) 9 | 10 | type ( 11 | // Source is a net drive disk provider. 12 | Source string 13 | 14 | // Share is an atomic downloadable file. 15 | Share struct { 16 | // FileName is a file name with the file extension. 17 | FileName string 18 | // SubPath The path for saving the files. 19 | SubPath string 20 | // Size is the file size in bytes. 21 | Size int64 22 | // URL is the downloadable url for this file. 23 | URL string 24 | // Properties could be some metadata, such as the token for this downloadable share. 25 | Properties map[string]any 26 | } 27 | 28 | // Driver is used to resolve the links from a Source. 29 | Driver interface { 30 | // Source will return the driver identity. 31 | Source() Source 32 | 33 | // Resolve the given link and return the file name with the download link. 34 | Resolve(link, passcode string) ([]Share, error) 35 | 36 | // Download the given link. 37 | Download(share Share) (io.ReadCloser, int64, error) 38 | } 39 | ) 40 | 41 | const ( 42 | ALIYUN Source = "aliyun" 43 | LANZOU Source = "lanzou" 44 | TELECOM Source = "telecom" 45 | BAIDU Source = "baidu" 46 | CTFILE Source = "ctfile" 47 | QUARK Source = "quark" 48 | DIRECT Source = "direct" 49 | ) 50 | 51 | // New will create the basic driver service. 52 | func New(config *client.Config, properties map[string]string) (Driver, error) { 53 | source := Source(properties["driver"]) 54 | switch source { 55 | case ALIYUN: 56 | return newAliyunDriver(config, properties) 57 | case TELECOM: 58 | return newTelecomDriver(config, properties) 59 | case LANZOU: 60 | return newLanzouDriver(config, properties) 61 | default: 62 | return nil, fmt.Errorf("invalid driver service %s", source) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /internal/driver/lanzou.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/bookstairs/bookhunter/internal/client" 7 | "github.com/bookstairs/bookhunter/internal/driver/lanzou" 8 | ) 9 | 10 | func newLanzouDriver(c *client.Config, _ map[string]string) (Driver, error) { 11 | drive, err := lanzou.New(c) 12 | if err != nil { 13 | return nil, err 14 | } 15 | 16 | return &lanzouDriver{driver: drive}, nil 17 | } 18 | 19 | type lanzouDriver struct { 20 | driver *lanzou.Lanzou 21 | } 22 | 23 | func (l *lanzouDriver) Source() Source { 24 | return LANZOU 25 | } 26 | 27 | func (l *lanzouDriver) Resolve(link, passcode string) ([]Share, error) { 28 | resp, err := l.driver.ResolveShareURL(link, passcode) 29 | if err != nil { 30 | return nil, err 31 | } 32 | shareList := make([]Share, len(resp)) 33 | for i, item := range resp { 34 | shareList[i] = Share{ 35 | FileName: item.Name, 36 | URL: item.URL, 37 | } 38 | } 39 | return shareList, err 40 | } 41 | 42 | func (l *lanzouDriver) Download(share Share) (io.ReadCloser, int64, error) { 43 | return l.driver.DownloadFile(share.URL) 44 | } 45 | -------------------------------------------------------------------------------- /internal/driver/lanzou/common.go: -------------------------------------------------------------------------------- 1 | package lanzou 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/bookstairs/bookhunter/internal/client" 8 | "github.com/bookstairs/bookhunter/internal/log" 9 | ) 10 | 11 | type Lanzou struct { 12 | *client.Client 13 | } 14 | 15 | var ( 16 | // 蓝奏云的主域名有时会挂掉, 此时尝试切换到备用域名 17 | availableHostnames = []string{ 18 | "lanzouw.com", 19 | "lanzoui.com", 20 | "lanzoux.com", 21 | "lanzouy.com", 22 | "lanzoup.com", 23 | } 24 | ) 25 | 26 | func checkOrSwitchHostname(c *client.Client) error { 27 | checkHostname := func(hostname string) bool { 28 | c.SetDefaultHostname(hostname) 29 | head, err := c.R().Head("/") 30 | return err == nil && !head.IsError() 31 | } 32 | 33 | for _, hostnames := range availableHostnames { 34 | if available := checkHostname(hostnames); available { 35 | return nil 36 | } 37 | } 38 | 39 | return fmt.Errorf("no available lanzou hostname") 40 | } 41 | 42 | func New(config *client.Config) (*Lanzou, error) { 43 | cl, err := client.New(&client.Config{ 44 | HTTPS: true, 45 | Host: availableHostnames[0], 46 | Proxy: config.Proxy, 47 | ConfigRoot: config.ConfigRoot, 48 | }) 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | cl.Client. 54 | SetHeader("Accept-Language", "zh-CN,zh;q=0.9"). 55 | SetHeader("Referer", cl.BaseURL) 56 | 57 | if err := checkOrSwitchHostname(cl); err != nil { 58 | return nil, err 59 | } 60 | 61 | return &Lanzou{Client: cl}, nil 62 | } 63 | 64 | func (l *Lanzou) DownloadFile(downloadURL string) (io.ReadCloser, int64, error) { 65 | log.Debugf("Start to download file from aliyun drive: %s", downloadURL) 66 | 67 | resp, err := l.R(). 68 | SetDoNotParseResponse(true). 69 | Get(downloadURL) 70 | if err != nil { 71 | return nil, 0, err 72 | } 73 | 74 | return resp.RawBody(), resp.RawResponse.ContentLength, nil 75 | } 76 | -------------------------------------------------------------------------------- /internal/driver/lanzou/response.go: -------------------------------------------------------------------------------- 1 | package lanzou 2 | 3 | import "regexp" 4 | 5 | var ( 6 | lxReg = regexp.MustCompile(`'lx':(\d+),`) 7 | fidReg = regexp.MustCompile(`'fid':(\d+),`) 8 | uidReg = regexp.MustCompile(`'uid':'(\d+)',`) 9 | repReg = regexp.MustCompile(`'rep':'(\d+)',`) 10 | upReg = regexp.MustCompile(`'up':(\d+),`) 11 | lsReg = regexp.MustCompile(`'ls':(\d+),`) 12 | tVar = regexp.MustCompile(`'t':(\S+),`) 13 | kVar = regexp.MustCompile(`'k':(\S+),`) 14 | 15 | dirURLRe = regexp.MustCompile(`(?m)https?://[a-zA-Z0-9-]*?\.?lanzou[a-z]\.com/(/s/)?b[a-zA-Z0-9]{7,}/?`) 16 | fileURLRe = regexp.MustCompile(`(?m)https?://[a-zA-Z0-9-]*?\.?lanzou[a-z]\.com/(/s/)?i[a-zA-Z0-9]{7,}/?`) 17 | 18 | find1Re = regexp.MustCompile(`(?m)var\s+skdklds\s+=\s+'(.*?)';`) 19 | find2Re = regexp.MustCompile(`(?m)`) 20 | find2TitleRe = regexp.MustCompile(`(?m)(.*?)\s-\s蓝奏云`) 21 | 22 | htmlNoteRe = regexp.MustCompile(`(?m)|\s+//\s*.+`) 23 | jsNoteRe = regexp.MustCompile(`(?m)(.+?[,;])\s*//.+`) 24 | ) 25 | 26 | type ( 27 | ResponseData struct { 28 | Name string `json:"name"` 29 | Size string `json:"size"` 30 | URL string `json:"url"` 31 | } 32 | 33 | Dom struct { 34 | Zt int `json:"zt"` 35 | Dom string `json:"dom"` 36 | // URL 可能为string或int 37 | URL interface{} `json:"url"` 38 | // Inf 可能为string或int 39 | Inf interface{} `json:"inf"` 40 | } 41 | 42 | FileList struct { 43 | Zt int `json:"zt"` 44 | Info string `json:"info"` 45 | // Text 可能为int或数组 46 | Text interface{} `json:"text"` 47 | } 48 | 49 | FileItem []struct { 50 | Icon string `json:"icon"` 51 | T int `json:"t"` 52 | ID string `json:"id"` 53 | NameAll string `json:"name_all"` 54 | Size string `json:"size"` 55 | Time string `json:"time"` 56 | Duan string `json:"duan"` 57 | PIco int `json:"p_ico"` 58 | } 59 | ) 60 | -------------------------------------------------------------------------------- /internal/driver/lanzou/share.go: -------------------------------------------------------------------------------- 1 | package lanzou 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | "regexp" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/go-resty/resty/v2" 13 | 14 | "github.com/bookstairs/bookhunter/internal/log" 15 | ) 16 | 17 | func (l *Lanzou) ResolveShareURL(shareURL, pwd string) ([]ResponseData, error) { 18 | shareURL = strings.TrimSpace(shareURL) 19 | // 移除url前部的主机 20 | rawURL, _ := url.Parse(shareURL) 21 | parsedURI := rawURL.RequestURI() 22 | 23 | if l.IsFileURL(shareURL) { 24 | fileShareURL, err := l.resolveFileShareURL(parsedURI, pwd) 25 | if err != nil { 26 | return nil, err 27 | } 28 | return []ResponseData{*fileShareURL}, err 29 | } else if l.IsDirURL(shareURL) { 30 | return l.resolveFileItemShareURL(parsedURI, pwd) 31 | } else { 32 | log.Warnf("Unexpected share url, try to download by using directory share API. %s", shareURL) 33 | return l.resolveFileItemShareURL(parsedURI, pwd) 34 | } 35 | } 36 | 37 | func (l *Lanzou) IsDirURL(shareURL string) bool { 38 | return dirURLRe.MatchString(shareURL) 39 | } 40 | 41 | func (l *Lanzou) IsFileURL(shareURL string) bool { 42 | return fileURLRe.MatchString(shareURL) 43 | } 44 | 45 | func (l *Lanzou) removeNotes(html string) string { 46 | html = htmlNoteRe.ReplaceAllString(html, "") 47 | html = jsNoteRe.ReplaceAllString(html, "$1") 48 | return html 49 | } 50 | 51 | func (l *Lanzou) resolveFileShareURL(parsedURI, pwd string) (*ResponseData, error) { 52 | resp, err := l.R().Get(parsedURI) 53 | if err != nil { 54 | return nil, err 55 | } 56 | firstPage := resp.String() 57 | firstPage = l.removeNotes(firstPage) 58 | 59 | // 参考https://github.com/zaxtyson/LanZouCloud-API 中对acwScV2的处理 60 | if strings.Contains(firstPage, "acw_sc__v2") { 61 | // 在页面被过多访问或其他情况下，有时候会先返回一个加密的页面，其执行计算出一个acw_sc__v2后放入页面后再重新访问页面才能获得正常页面 62 | // 若该页面进行了js加密，则进行解密，计算acw_sc__v2，并加入cookie 63 | acwScV2 := l.calcAcwScV2(firstPage) 64 | l.SetCookie(&http.Cookie{ 65 | Name: "acw_sc__v2", 66 | Value: acwScV2, 67 | }) 68 | log.Infof("Set Cookie: acw_sc__v2=%v", acwScV2) 69 | get, _ := l.R().Get(parsedURI) 70 | firstPage = get.String() 71 | } 72 | 73 | if strings.Contains(firstPage, "文件取消") || strings.Contains(firstPage, "文件不存在") { 74 | return nil, fmt.Errorf("文件不存在 %v", parsedURI) 75 | } 76 | 77 | // Share with password 78 | if strings.Contains(firstPage, "id=\"pwdload\"") || 79 | strings.Contains(firstPage, "id=\"passwddiv\"") { 80 | if pwd == "" { 81 | return nil, fmt.Errorf("缺少密码 %v", parsedURI) 82 | } 83 | return l.ParsePasswordShare(parsedURI, pwd, firstPage) 84 | } else if find2Re.MatchString(firstPage) { 85 | lanzouDom, err := l.ParseAnonymousShare(parsedURI, firstPage) 86 | return lanzouDom, err 87 | } 88 | return nil, fmt.Errorf("解析页面失败") 89 | } 90 | 91 | func (l *Lanzou) ParsePasswordShare(parsedURI string, pwd string, firstPage string) (*ResponseData, error) { 92 | sign := find1Re.FindStringSubmatch(firstPage) 93 | urlpath := "/ajaxm.php" 94 | 95 | result := &Dom{} 96 | data := make(map[string]string) 97 | data["action"] = "downprocess" 98 | data["sign"] = sign[1] 99 | data["p"] = pwd 100 | 101 | _, err := l.R(). 102 | SetHeader("referer", l.BaseURL+parsedURI). 103 | SetHeader("Content-Type", "application/x-www-form-urlencoded"). 104 | SetResult(result). 105 | SetFormData(data). 106 | Post(urlpath) 107 | if err != nil { 108 | return nil, err 109 | } 110 | return l.parseDom(result) 111 | } 112 | 113 | func (l *Lanzou) ParseAnonymousShare(parsedURI string, firstPage string) (*ResponseData, error) { 114 | allString := find2Re.FindStringSubmatch(firstPage) 115 | 116 | dom, err := l.R().Get(allString[1]) 117 | if err != nil { 118 | return nil, err 119 | } 120 | data := make(map[string]string) 121 | fnDom := l.removeNotes(dom.String()) 122 | var re = regexp.MustCompile(`(?m)var\s+(\w+)\s+=\s+'(.*)';`) 123 | for _, match := range re.FindAllStringSubmatch(fnDom, -1) { 124 | data[match[1]] = match[2] 125 | } 126 | title := l.extractRegex(find2TitleRe, firstPage) 127 | 128 | var formRe = regexp.MustCompile(`(?m)('(\w+)':([\w']+),?)`) 129 | 130 | fromData := make(map[string]string) 131 | for _, match := range formRe.FindAllStringSubmatch(fnDom, -1) { 132 | k := match[2] 133 | v := match[3] 134 | 135 | if strings.HasPrefix(v, "'") && strings.HasSuffix(v, "'") { 136 | v = strings.TrimLeft(strings.TrimRight(v, "'"), "'") 137 | } else if v != "1" { 138 | v = data[v] 139 | } 140 | fromData[k] = v 141 | } 142 | 143 | result := &Dom{} 144 | _, err = l.R(). 145 | SetHeader("origin", l.BaseURL). 146 | SetHeader("referer", l.BaseURL+parsedURI). 147 | SetHeader("Content-Type", "application/x-www-form-urlencoded"). 148 | SetResult(result). 149 | SetFormData(fromData). 150 | Post("/ajaxm.php") 151 | if err != nil { 152 | return nil, err 153 | } 154 | lanzouDom, err := l.parseDom(result) 155 | if lanzouDom != nil { 156 | lanzouDom.Name = title 157 | } 158 | return lanzouDom, err 159 | } 160 | 161 | func (l *Lanzou) parseDom(result *Dom) (*ResponseData, error) { 162 | if result.Zt != 1 { 163 | return nil, fmt.Errorf("解析直链失败") 164 | } 165 | 166 | var header = map[string]string{ 167 | "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", 168 | "Referer": "https://" + l.Host, 169 | } 170 | 171 | request := resty.New().SetRedirectPolicy(resty.NoRedirectPolicy()). 172 | R() 173 | rr, err := request.SetHeaders(header). 174 | Get(result.Dom + "/file/" + result.URL.(string)) 175 | if rr.StatusCode() != 302 && err != nil { 176 | log.Fatalf("解析链接失败 %v", err) 177 | } 178 | 179 | if strings.Contains(rr.String(), "网络异常") { 180 | log.Fatalf("访问过多，被限制，解限功能待实现 %v", err) 181 | } 182 | 183 | location := rr.Header().Get("location") 184 | 185 | title, _ := result.Inf.(string) 186 | return &ResponseData{ 187 | Name: title, 188 | URL: location, 189 | }, nil 190 | } 191 | 192 | func (l *Lanzou) calcAcwScV2(htmlText string) string { 193 | arg1Re := regexp.MustCompile(`arg1='([0-9A-Z]+)'`) 194 | arg1 := l.extractRegex(arg1Re, htmlText) 195 | acwScV2 := l.hexXor(l.unbox(arg1), "3000176000856006061501533003690027800375") 196 | return acwScV2 197 | } 198 | 199 | func (l *Lanzou) unbox(arg string) string { 200 | v1 := []int{15, 35, 29, 24, 33, 16, 1, 38, 10, 9, 19, 31, 40, 27, 22, 23, 25, 13, 6, 11, 201 | 39, 18, 20, 8, 14, 21, 32, 26, 2, 30, 7, 4, 17, 5, 3, 28, 34, 37, 12, 36} 202 | v2 := make([]string, len(v1)) 203 | for idx, v3 := range arg { 204 | for idx2, in := range v1 { 205 | if in == (idx + 1) { 206 | v2[idx2] = string(v3) 207 | } 208 | } 209 | } 210 | return strings.Join(v2, "") 211 | } 212 | 213 | func (l *Lanzou) hexXor(arg, args string) string { 214 | a := min(len(arg), len(args)) 215 | res := "" 216 | for idx := 0; idx < a; idx += 2 { 217 | v1, _ := strconv.ParseInt(arg[idx:idx+2], 16, 32) 218 | v2, _ := strconv.ParseInt(args[idx:idx+2], 16, 32) 219 | // v to lower case hex 220 | v3 := fmt.Sprintf("%02x", v1^v2) 221 | res += v3 222 | } 223 | return res 224 | } 225 | 226 | func (l *Lanzou) extractRegex(reg *regexp.Regexp, str string) string { 227 | matches := reg.FindStringSubmatch(str) 228 | if len(matches) >= 2 { 229 | return matches[1] 230 | } 231 | return "" 232 | } 233 | 234 | func (l *Lanzou) resolveFileItemShareURL(parsedURI, pwd string) ([]ResponseData, error) { 235 | resp, _ := l.R().Get(parsedURI) 236 | str := resp.String() 237 | formData := map[string]string{ 238 | "lx": l.extractRegex(lxReg, str), 239 | "fid": l.extractRegex(fidReg, str), 240 | "uid": l.extractRegex(uidReg, str), 241 | "pg": "1", 242 | "rep": l.extractRegex(repReg, str), 243 | "t": l.extractRegex(regexp.MustCompile("var "+l.extractRegex(tVar, str)+" = '(\\d+)';"), str), 244 | "k": l.extractRegex(regexp.MustCompile("var "+l.extractRegex(kVar, str)+" = '(\\S+)';"), str), 245 | "up": l.extractRegex(upReg, str), 246 | "ls": l.extractRegex(lsReg, str), 247 | "pwd": pwd, 248 | } 249 | 250 | result := &FileList{} 251 | _, err := l.R().SetFormData(formData).SetResult(result).Post("/filemoreajax.php") 252 | if err != nil { 253 | return nil, err 254 | } 255 | 256 | if result.Zt != 1 { 257 | log.Warnf("lanzou 文件列表解析失败 %v %v %v", parsedURI, pwd, result.Info) 258 | return []ResponseData{}, nil 259 | } 260 | 261 | item, ok := result.Text.([]interface{}) 262 | 263 | if !ok { 264 | log.Warnf("lanzou 文件列表解析失败 %v %v %v", parsedURI, pwd, result) 265 | return nil, errors.New("lanzou 文件列表解析失败") 266 | } 267 | 268 | data := make([]ResponseData, len(item)) 269 | for i, d := range item { 270 | file := d.(map[string]interface{}) 271 | 272 | respData, err := l.resolveFileShareURL("/"+file["id"].(string), pwd) 273 | if err != nil { 274 | return nil, err 275 | } 276 | respData.Name = file["name_all"].(string) 277 | data[i] = *respData 278 | } 279 | return data, nil 280 | } 281 | -------------------------------------------------------------------------------- /internal/driver/lanzou/share_test.go: -------------------------------------------------------------------------------- 1 | package lanzou 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/bookstairs/bookhunter/internal/client" 9 | "github.com/bookstairs/bookhunter/internal/log" 10 | ) 11 | 12 | func TestParseLanzouUrl(t *testing.T) { 13 | log.EnableDebug = true 14 | 15 | type args struct { 16 | url string 17 | pwd string 18 | } 19 | tests := []struct { 20 | name string 21 | args args 22 | }{ 23 | { 24 | name: "test lanzoue.com", 25 | args: args{ 26 | url: "https://tianlangbooks.lanzoue.com/iI0lc0fj3mpa", 27 | pwd: "tlsw", 28 | }, 29 | }, 30 | { 31 | name: "test lanzoui", 32 | args: args{ 33 | url: "https://tianlangbooks.lanzoui.com/i9CTIws9s6d", 34 | pwd: "tlsw", 35 | }, 36 | }, 37 | { 38 | name: "test lanzouf.com", 39 | args: args{ 40 | url: "https://tianlangbooks.lanzouf.com/ic7HY05ejl2h", 41 | pwd: "tlsw", 42 | }, 43 | }, 44 | { 45 | name: "test lanzoup.com", 46 | args: args{ 47 | url: "https://tianlangbooks.lanzoup.com/i4q4Chcm2cf", 48 | pwd: "tlsw", 49 | }, 50 | }, 51 | { 52 | name: "test lanzouy.com", 53 | args: args{ 54 | url: "https://fast8.lanzouy.com/ibZCg0b8tibi", 55 | pwd: "", 56 | }, 57 | }, 58 | { 59 | name: "test sobook lanzouy.com", 60 | args: args{ 61 | url: "https://sobooks.lanzoum.com/b03phl3te", 62 | pwd: "htuj", 63 | }, 64 | }, 65 | { 66 | name: "test sobook lanzouy.com 2", 67 | args: args{ 68 | url: "https://sobooks.lanzoum.com/ihOex0fiodri", 69 | pwd: "", 70 | }, 71 | }, { 72 | name: "test lanzou file list", 73 | args: args{ 74 | url: "https://wwx.lanzoui.com/b04azyong", 75 | pwd: "7drb", 76 | }, 77 | }, { 78 | name: "test lanzou file list1", 79 | args: args{ 80 | url: "https://sobooks.lanzoui.com/b03nqddti", 81 | pwd: "gw0h", 82 | }, 83 | }, 84 | } 85 | 86 | drive, err := New(&client.Config{}) 87 | assert.NoError(t, err, "Failed to create lanzou") 88 | 89 | for _, tt := range tests { 90 | t.Run(tt.name, func(t *testing.T) { 91 | response, err := drive.ResolveShareURL(tt.args.url, tt.args.pwd) 92 | assert.NoError(t, err, "Failed to resolve link") 93 | assert.NotEmpty(t, response) 94 | 95 | for _, item := range response { 96 | assert.NotEmpty(t, item.URL) 97 | assert.NotEmpty(t, item.Name) 98 | } 99 | }) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /internal/driver/telecom.go: -------------------------------------------------------------------------------- 1 | package driver 2 | 3 | import ( 4 | "io" 5 | "strconv" 6 | 7 | "github.com/bookstairs/bookhunter/internal/client" 8 | "github.com/bookstairs/bookhunter/internal/driver/telecom" 9 | ) 10 | 11 | func newTelecomDriver(config *client.Config, properties map[string]string) (Driver, error) { 12 | // Create the pan client. 13 | t, err := telecom.New(config, properties["telecomUsername"], properties["telecomPassword"]) 14 | if err != nil { 15 | return nil, err 16 | } 17 | 18 | return &telecomDriver{client: t}, nil 19 | } 20 | 21 | type telecomDriver struct { 22 | client *telecom.Telecom 23 | } 24 | 25 | func (t *telecomDriver) Source() Source { 26 | return TELECOM 27 | } 28 | 29 | func (t *telecomDriver) Resolve(link, passcode string) ([]Share, error) { 30 | code, err := t.client.ShareCode(link) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | info, files, err := t.client.ShareFiles(link, passcode) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | // Convert this into a share entity. 41 | shares := make([]Share, 0, len(files)) 42 | for _, file := range files { 43 | shares = append(shares, Share{ 44 | FileName: file.Name, 45 | Size: file.Size, 46 | Properties: map[string]any{ 47 | "shareCode": code, 48 | "shareID": strconv.FormatInt(info.ShareID, 10), 49 | "fileID": strconv.FormatInt(file.ID, 10), 50 | }, 51 | }) 52 | } 53 | 54 | return shares, nil 55 | } 56 | 57 | func (t *telecomDriver) Download(share Share) (io.ReadCloser, int64, error) { 58 | shareCode := share.Properties["shareCode"].(string) 59 | shareID := share.Properties["shareID"].(string) 60 | fileID := share.Properties["fileID"].(string) 61 | 62 | // Resolve the link. 63 | url, err := t.client.DownloadURL(shareCode, shareID, fileID) 64 | if err != nil { 65 | return nil, 0, err 66 | } 67 | 68 | // Download the file. 69 | file, err := t.client.DownloadFile(url) 70 | 71 | return file, 0, err 72 | } 73 | -------------------------------------------------------------------------------- /internal/driver/telecom/common.go: -------------------------------------------------------------------------------- 1 | package telecom 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/bookstairs/bookhunter/internal/client" 7 | ) 8 | 9 | const ( 10 | webPrefix = "https://cloud.189.cn" 11 | authPrefix = "https://open.e.189.cn/api/logbox/oauth2" 12 | apiPrefix = "https://api.cloud.189.cn" 13 | ) 14 | 15 | type Telecom struct { 16 | *client.Client 17 | appToken *AppLoginToken 18 | } 19 | 20 | func New(c *client.Config, username, password string) (*Telecom, error) { 21 | cl, err := client.New(&client.Config{ 22 | HTTPS: false, 23 | Host: "cloud.189.cn", 24 | Proxy: c.Proxy, 25 | ConfigRoot: c.ConfigRoot, 26 | }) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | cl.SetHeader("Accept", "application/json;charset=UTF-8") 32 | t := &Telecom{Client: cl} 33 | 34 | if username == "" || password == "" { 35 | return nil, errors.New("no username or password provide, we may not able to download from telecom disk") 36 | } 37 | 38 | // Start to sign in. 39 | if err := t.login(username, password); err != nil { 40 | return nil, err 41 | } 42 | 43 | return t, nil 44 | } 45 | -------------------------------------------------------------------------------- /internal/driver/telecom/crypto.go: -------------------------------------------------------------------------------- 1 | package telecom 2 | 3 | import ( 4 | "bytes" 5 | "crypto/md5" 6 | "crypto/rand" 7 | "crypto/rsa" 8 | "crypto/x509" 9 | "encoding/base64" 10 | "encoding/hex" 11 | "encoding/pem" 12 | "errors" 13 | "sort" 14 | "strings" 15 | ) 16 | 17 | const ( 18 | b64map = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" 19 | biRm = "0123456789abcdefghijklmnopqrstuvwxyz" 20 | ) 21 | 22 | // base64Encode base64 encoded. 23 | func base64Encode(raw []byte) []byte { 24 | var encoded bytes.Buffer 25 | encoder := base64.NewEncoder(base64.StdEncoding, &encoded) 26 | _, _ = encoder.Write(raw) 27 | _ = encoder.Close() 28 | return encoded.Bytes() 29 | } 30 | 31 | // rsaEncrypt RSA encrypt. 32 | func rsaEncrypt(publicKey, origData []byte) ([]byte, error) { 33 | block, _ := pem.Decode(publicKey) 34 | if block == nil { 35 | return nil, errors.New("public key error") 36 | } 37 | pubInterface, err := x509.ParsePKIXPublicKey(block.Bytes) 38 | if err != nil { 39 | return nil, err 40 | } 41 | pub := pubInterface.(*rsa.PublicKey) 42 | return rsa.EncryptPKCS1v15(rand.Reader, pub, origData) 43 | } 44 | 45 | // signatureOfMd5 MD5 Signature for telecom. 46 | func signatureOfMd5(params map[string]string) string { 47 | var keys []string 48 | for k, v := range params { 49 | keys = append(keys, k+"="+v) 50 | } 51 | 52 | // sort 53 | sort.Strings(keys) 54 | 55 | signStr := strings.Join(keys, "&") 56 | 57 | h := md5.New() 58 | h.Write([]byte(signStr)) 59 | return hex.EncodeToString(h.Sum(nil)) 60 | } 61 | 62 | // b64toHex 将base64字符串转换成HEX十六进制字符串 63 | func b64toHex(b64 []byte) string { 64 | b64str := string(b64) 65 | sb := strings.Builder{} 66 | e := 0 67 | c := 0 68 | for _, r := range b64str { 69 | if r != '=' { 70 | v := strings.Index(b64map, string(r)) 71 | if e == 0 { 72 | e = 1 73 | sb.WriteByte(int2char(v >> 2)) 74 | c = 3 & v 75 | } else if e == 1 { 76 | e = 2 77 | sb.WriteByte(int2char(c<<2 | v>>4)) 78 | c = 15 & v 79 | } else if e == 2 { 80 | e = 3 81 | sb.WriteByte(int2char(c)) 82 | sb.WriteByte(int2char(v >> 2)) 83 | c = 3 & v 84 | } else { 85 | e = 0 86 | sb.WriteByte(int2char(c<<2 | v>>4)) 87 | sb.WriteByte(int2char(15 & v)) 88 | } 89 | } 90 | } 91 | if e == 1 { 92 | sb.WriteByte(int2char(c << 2)) 93 | } 94 | return sb.String() 95 | } 96 | 97 | func int2char(i int) (r byte) { 98 | return biRm[i] 99 | } 100 | -------------------------------------------------------------------------------- /internal/driver/telecom/login.go: -------------------------------------------------------------------------------- 1 | package telecom 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | "regexp" 7 | "strconv" 8 | "time" 9 | 10 | "github.com/bookstairs/bookhunter/internal/log" 11 | ) 12 | 13 | func (t *Telecom) login(username, password string) error { 14 | // Query the rsa key. 15 | params, err := t.loginParams() 16 | if err != nil { 17 | return err 18 | } 19 | 20 | // Perform the login by the app API. 21 | app, err := t.appLogin(params, username, password) 22 | if err != nil { 23 | return err 24 | } 25 | 26 | // Access the login session. 27 | session, err := t.createSession(app.ToURL) 28 | if err != nil { 29 | return err 30 | } 31 | 32 | // Acquire the Ssk token. 33 | token, err := t.createToken(session.SessionKey) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | // Refresh the cookies. 39 | err = t.refreshCookies(session.SessionKey) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | // Save token into the driver client. 45 | t.appToken = &AppLoginToken{ 46 | RsaPublicKey: params.jRsaKey, 47 | SessionKey: session.SessionKey, 48 | SessionSecret: session.SessionSecret, 49 | FamilySessionKey: session.FamilySessionKey, 50 | FamilySessionSecret: session.FamilySessionSecret, 51 | AccessToken: session.AccessToken, 52 | RefreshToken: session.RefreshToken, 53 | SskAccessTokenExpiresIn: token.ExpiresIn, 54 | SskAccessToken: token.AccessToken, 55 | } 56 | 57 | log.Info("Login into telecom success.") 58 | 59 | return nil 60 | } 61 | 62 | func (t *Telecom) loginParams() (*AppLoginParams, error) { 63 | // Do a fresh login. 64 | t.CleanCookies() 65 | 66 | resp, err := t.R(). 67 | SetQueryParam("appId", "8025431004"). 68 | SetQueryParam("clientType", "10020"). 69 | SetQueryParam("returnURL", "https://m.cloud.189.cn/zhuanti/2020/loginErrorPc/index.html"). 70 | SetQueryParam("timeStamp", timeStamp()). 71 | SetHeader("Content-Type", "application/x-www-form-urlencoded"). 72 | Get(webPrefix + "/unifyLoginForPC.action") 73 | 74 | if err != nil { 75 | log.Debugf("login redirectURL occurs error: %s", err) 76 | return nil, err 77 | } 78 | content := resp.String() 79 | 80 | re := regexp.MustCompile("captchaToken' value='(.+?)'") 81 | captchaToken := re.FindStringSubmatch(content)[1] 82 | 83 | re = regexp.MustCompile("lt = \"(.+?)\"") 84 | lt := re.FindStringSubmatch(content)[1] 85 | 86 | re = regexp.MustCompile("returnUrl = '(.+?)'") 87 | returnURL := re.FindStringSubmatch(content)[1] 88 | 89 | re = regexp.MustCompile("paramId = \"(.+?)\"") 90 | paramID := re.FindStringSubmatch(content)[1] 91 | 92 | re = regexp.MustCompile("reqId = \"(.+?)\"") 93 | reqID := re.FindStringSubmatch(content)[1] 94 | 95 | // RSA key should be wrapped with the comments. 96 | re = regexp.MustCompile("j_rsaKey\" value=\"(.+?)\"") 97 | jRsaKey := "-----BEGIN PUBLIC KEY-----\n" + re.FindStringSubmatch(content)[1] + "\n-----END PUBLIC KEY-----" 98 | 99 | return &AppLoginParams{ 100 | CaptchaToken: captchaToken, 101 | Lt: lt, 102 | ReturnURL: returnURL, 103 | ParamID: paramID, 104 | ReqID: reqID, 105 | jRsaKey: jRsaKey, 106 | }, nil 107 | } 108 | 109 | func (t *Telecom) appLogin(params *AppLoginParams, username, password string) (*AppLoginResult, error) { 110 | rsaUsername, _ := rsaEncrypt([]byte(params.jRsaKey), []byte(username)) 111 | rsaPassword, _ := rsaEncrypt([]byte(params.jRsaKey), []byte(password)) 112 | 113 | // Start to perform login. 114 | resp, err := t.R(). 115 | SetHeaders(map[string]string{ 116 | "Accept": "application/json;charset=UTF-8", 117 | "Content-Type": "application/x-www-form-urlencoded", 118 | "Referer": authPrefix + "/unifyAccountLogin.do", 119 | "Cookie": "LT=" + params.Lt, 120 | "X-Requested-With": "XMLHttpRequest", 121 | "REQID": params.ReqID, 122 | "lt": params.Lt, 123 | }). 124 | SetFormData(map[string]string{ 125 | "appKey": "8025431004", 126 | "accountType": "02", 127 | "userName": "{RSA}" + b64toHex(base64Encode(rsaUsername)), 128 | "password": "{RSA}" + b64toHex(base64Encode(rsaPassword)), 129 | "validateCode": "", 130 | "captchaToken": params.CaptchaToken, 131 | "returnUrl": params.ReturnURL, 132 | "mailSuffix": "@189.cn", 133 | "dynamicCheck": "FALSE", 134 | "clientType": "10020", 135 | "cb_SaveName": "1", 136 | "isOauth2": "false", 137 | "state": "", 138 | "paramId": params.ParamID, 139 | }). 140 | ForceContentType("application/json"). 141 | SetResult(&AppLoginResult{}). 142 | Post(authPrefix + "/loginSubmit.do") 143 | if err != nil { 144 | return nil, err 145 | } 146 | 147 | // Check login result. 148 | res := resp.Result().(*AppLoginResult) 149 | if res.Result != 0 || res.ToURL == "" { 150 | return nil, errors.New("login failed in telecom disk") 151 | } 152 | 153 | return res, nil 154 | } 155 | 156 | func (t *Telecom) createSession(jumpURL string) (*AppSessionResp, error) { 157 | resp, err := t.R(). 158 | SetHeader("Accept", "application/json;charset=UTF-8"). 159 | SetQueryParams(map[string]string{ 160 | "clientType": "TELEMAC", 161 | "version": "1.0.0", 162 | "channelId": "web_cloud.189.cn", 163 | "redirectURL": url.QueryEscape(jumpURL), 164 | }). 165 | SetResult(&AppSessionResp{}). 166 | Get(apiPrefix + "/getSessionForPC.action") 167 | if err != nil { 168 | return nil, err 169 | } 170 | 171 | // Check the session result. 172 | res := resp.Result().(*AppSessionResp) 173 | if res.ResCode != 0 { 174 | return nil, errors.New("failed to acquire session") 175 | } 176 | 177 | return res, nil 178 | } 179 | 180 | func (t *Telecom) createToken(sessionKey string) (*AccessTokenResp, error) { 181 | timestamp := timeStamp() 182 | signParams := map[string]string{ 183 | "Timestamp": timestamp, 184 | "sessionKey": sessionKey, 185 | "AppKey": "601102120", 186 | } 187 | resp, err := t.R(). 188 | SetQueryParam("sessionKey", sessionKey). 189 | SetHeaders(map[string]string{ 190 | "AppKey": "601102120", 191 | "Signature": signatureOfMd5(signParams), 192 | "Sign-Type": "1", 193 | "Accept": "application/json", 194 | "Timestamp": timestamp, 195 | }). 196 | SetResult(&AccessTokenResp{}). 197 | Get(apiPrefix + "/open/oauth2/getAccessTokenBySsKey.action") 198 | if err != nil { 199 | return nil, err 200 | } 201 | 202 | return resp.Result().(*AccessTokenResp), nil 203 | } 204 | 205 | func (t *Telecom) refreshCookies(sessionKey string) error { 206 | _, err := t.R(). 207 | SetHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,ja;q=0.7"). 208 | SetQueryParams(map[string]string{ 209 | "sessionKey": sessionKey, 210 | "redirectUrl": "main.action%%23recycle", 211 | }). 212 | Get(webPrefix + "/ssoLogin.action") 213 | 214 | return err 215 | } 216 | 217 | // timeStamp is used to return the telecom required time str. 218 | func timeStamp() string { 219 | return strconv.FormatInt(time.Now().UTC().UnixNano()/1e6, 10) 220 | } 221 | -------------------------------------------------------------------------------- /internal/driver/telecom/response.go: -------------------------------------------------------------------------------- 1 | package telecom 2 | 3 | type ( 4 | // All the request and response structs for the telecom login. 5 | 6 | AppLoginParams struct { 7 | CaptchaToken string 8 | Lt string 9 | ReturnURL string 10 | ParamID string 11 | ReqID string 12 | jRsaKey string 13 | } 14 | 15 | AppLoginResult struct { 16 | Result int `json:"result"` 17 | Msg string `json:"msg"` 18 | ToURL string `json:"toUrl"` 19 | } 20 | 21 | AppSessionResp struct { 22 | ResCode int `json:"res_code"` 23 | ResMessage string `json:"res_message"` 24 | AccessToken string `json:"accessToken"` 25 | FamilySessionKey string `json:"familySessionKey"` 26 | FamilySessionSecret string `json:"familySessionSecret"` 27 | GetFileDiffSpan int `json:"getFileDiffSpan"` 28 | GetUserInfoSpan int `json:"getUserInfoSpan"` 29 | IsSaveName string `json:"isSaveName"` 30 | KeepAlive int `json:"keepAlive"` 31 | LoginName string `json:"loginName"` 32 | RefreshToken string `json:"refreshToken"` 33 | SessionKey string `json:"sessionKey"` 34 | SessionSecret string `json:"sessionSecret"` 35 | } 36 | 37 | AccessTokenResp struct { 38 | // The expiry time for token, default 30 days. 39 | ExpiresIn int64 `json:"expiresIn"` 40 | AccessToken string `json:"accessToken"` 41 | } 42 | 43 | AppLoginToken struct { 44 | SessionKey string `json:"sessionKey"` 45 | SessionSecret string `json:"sessionSecret"` 46 | FamilySessionKey string `json:"familySessionKey"` 47 | FamilySessionSecret string `json:"familySessionSecret"` 48 | AccessToken string `json:"accessToken"` 49 | RefreshToken string `json:"refreshToken"` 50 | SskAccessToken string `json:"sskAccessToken"` 51 | SskAccessTokenExpiresIn int64 `json:"sskAccessTokenExpiresIn"` 52 | RsaPublicKey string `json:"rsaPublicKey"` 53 | } 54 | ) 55 | 56 | type ( 57 | // All the request and response structs for the telecom share link query. 58 | 59 | ShareInfo struct { 60 | ResCode int `json:"res_code"` 61 | ResMessage string `json:"res_message"` 62 | AccessCode string `json:"accessCode"` 63 | ExpireTime int `json:"expireTime"` 64 | ExpireType int `json:"expireType"` 65 | FileID string `json:"fileId"` 66 | FileName string `json:"fileName"` 67 | FileSize int `json:"fileSize"` 68 | IsFolder bool `json:"isFolder"` 69 | NeedAccessCode int `json:"needAccessCode"` 70 | ShareDate int64 `json:"shareDate"` 71 | ShareID int64 `json:"shareId"` 72 | ShareMode int `json:"shareMode"` 73 | ShareType int `json:"shareType"` 74 | } 75 | 76 | ShareFile struct { 77 | CreateDate string `json:"createDate"` 78 | FileCata int `json:"fileCata"` 79 | ID int64 `json:"id"` 80 | LastOpTime string `json:"lastOpTime"` 81 | Md5 string `json:"md5"` 82 | MediaType int `json:"mediaType"` 83 | Name string `json:"name"` 84 | Rev string `json:"rev"` 85 | Size int64 `json:"size"` 86 | StarLabel int `json:"starLabel"` 87 | } 88 | 89 | ShareFolder struct { 90 | CreateDate string `json:"createDate"` 91 | FileCata int `json:"fileCata"` 92 | FileListSize int `json:"fileListSize"` 93 | ID int64 `json:"id"` 94 | LastOpTime string `json:"lastOpTime"` 95 | Name string `json:"name"` 96 | ParentID int64 `json:"parentId"` 97 | Rev string `json:"rev"` 98 | StarLabel int `json:"starLabel"` 99 | } 100 | 101 | ShareFiles struct { 102 | ResCode int `json:"res_code"` 103 | ResMessage string `json:"res_message"` 104 | ExpireTime int `json:"expireTime"` 105 | ExpireType int `json:"expireType"` 106 | FileListAO struct { 107 | Count int `json:"count"` 108 | FileList []ShareFile `json:"fileList"` 109 | FileListSize int64 `json:"fileListSize"` 110 | FolderList []ShareFolder `json:"folderList"` 111 | } `json:"fileListAO"` 112 | LastRev int64 `json:"lastRev"` 113 | } 114 | 115 | ShareLink struct { 116 | ResCode int `json:"res_code"` 117 | ResMessage string `json:"res_message"` 118 | FileDownloadURL string `json:"fileDownloadUrl"` 119 | } 120 | ) 121 | -------------------------------------------------------------------------------- /internal/driver/telecom/share.go: -------------------------------------------------------------------------------- 1 | package telecom 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strconv" 7 | "strings" 8 | 9 | "github.com/bookstairs/bookhunter/internal/log" 10 | ) 11 | 12 | // ShareFiles will resolve the telecom-shared link. 13 | func (t *Telecom) ShareFiles(accessURL, accessCode string) (*ShareInfo, []ShareFile, error) { 14 | log.Debugf("Download telecom file from %s -- %s", accessURL, accessCode) 15 | 16 | // Get share info. 17 | info, err := t.shareInfo(accessURL) 18 | if err != nil { 19 | return nil, nil, err 20 | } 21 | 22 | // Reclusive get the shared files. 23 | var files []ShareFile 24 | if info.IsFolder { 25 | files, err = t.listShareFolders(accessCode, info.FileID, info.FileID, info.ShareID, info.ShareMode) 26 | } else { 27 | files, err = t.listShareFiles(accessCode, info.FileID, info.ShareID, info.ShareMode) 28 | } 29 | if err != nil { 30 | return nil, nil, err 31 | } 32 | 33 | return info, files, nil 34 | } 35 | 36 | func (t *Telecom) DownloadURL(shareCode, shareID, fileID string) (string, error) { 37 | resp, err := t.R(). 38 | SetHeaders(map[string]string{ 39 | "accept": "application/json;charset=UTF-8", 40 | "origin": "https://cloud.189.cn", 41 | "Referer": "https://cloud.189.cn/web/share?code=" + shareCode, 42 | }). 43 | SetQueryParams(map[string]string{ 44 | "fileId": fileID, 45 | "shareId": shareID, 46 | "dt": "1", 47 | }). 48 | SetResult(&ShareLink{}). 49 | Get(webPrefix + "/api/open/file/getFileDownloadUrl.action") 50 | if err != nil { 51 | return "", err 52 | } 53 | link := resp.Result().(*ShareLink) 54 | 55 | return link.FileDownloadURL, nil 56 | } 57 | 58 | func (t *Telecom) DownloadFile(url string) (io.ReadCloser, error) { 59 | resp, err := t.R(). 60 | SetDoNotParseResponse(true). 61 | Get(url) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | return resp.RawBody(), nil 67 | } 68 | 69 | // ShareCode extract the share code. 70 | func (t *Telecom) ShareCode(accessURL string) (string, error) { 71 | if idx := strings.LastIndex(accessURL, "/"); idx > 0 { 72 | rs := []rune(accessURL) 73 | return string(rs[idx+1:]), nil 74 | } else { 75 | return "", fmt.Errorf("invalid share link, couldn't find share code: %s", accessURL) 76 | } 77 | } 78 | 79 | func (t *Telecom) shareInfo(accessURL string) (*ShareInfo, error) { 80 | shareCode, err := t.ShareCode(accessURL) 81 | if err != nil { 82 | return nil, err 83 | } 84 | resp, err := t.R(). 85 | SetHeaders(map[string]string{ 86 | "accept": "application/json;charset=UTF-8", 87 | "origin": "https://cloud.189.cn", 88 | "Referer": "https://cloud.189.cn/web/share?code=" + shareCode, 89 | }). 90 | SetQueryParam("shareCode", shareCode). 91 | SetResult(&ShareInfo{}). 92 | Get(webPrefix + "/api/open/share/getShareInfoByCode.action") 93 | if err != nil { 94 | return nil, err 95 | } 96 | 97 | return resp.Result().(*ShareInfo), nil 98 | } 99 | 100 | func (t *Telecom) listShareFiles(code, fileID string, shareID int64, mode int) ([]ShareFile, error) { 101 | resp, err := t.R(). 102 | SetQueryParams(map[string]string{ 103 | "fileId": fileID, 104 | "shareId": strconv.FormatInt(shareID, 10), 105 | "shareMode": strconv.Itoa(mode), 106 | "accessCode": code, 107 | "isFolder": "false", 108 | "iconOption": "5", 109 | "pageNum": "1", 110 | "pageSize": "10", 111 | }). 112 | SetResult(&ShareFiles{}). 113 | Get(webPrefix + "/api/open/share/listShareDir.action") 114 | if err != nil { 115 | return nil, err 116 | } 117 | res := resp.Result().(*ShareFiles) 118 | 119 | return res.FileListAO.FileList, nil 120 | } 121 | 122 | func (t *Telecom) listShareFolders(code, fileID, shareDirFileID string, shareID int64, mode int) ([]ShareFile, error) { 123 | resp, err := t.R(). 124 | SetQueryParams(map[string]string{ 125 | "fileId": fileID, 126 | "shareDirFileId": shareDirFileID, 127 | "shareId": strconv.FormatInt(shareID, 10), 128 | "shareMode": strconv.Itoa(mode), 129 | "accessCode": code, 130 | "isFolder": "true", 131 | "orderBy": "lastOpTime", 132 | "descending": "true", 133 | "iconOption": "5", 134 | "pageNum": "1", 135 | "pageSize": "60", 136 | }). 137 | SetResult(&ShareFiles{}). 138 | Get(webPrefix + "/api/open/share/listShareDir.action") 139 | if err != nil { 140 | return nil, err 141 | } 142 | 143 | res := resp.Result().(*ShareFiles).FileListAO 144 | files := res.FileList 145 | 146 | for _, folder := range res.FolderList { 147 | id := strconv.FormatInt(folder.ID, 10) 148 | children, err := t.listShareFolders(code, id, shareDirFileID, shareID, mode) 149 | if err != nil { 150 | return nil, err 151 | } 152 | 153 | files = append(files, children...) 154 | } 155 | 156 | return files, nil 157 | } 158 | -------------------------------------------------------------------------------- /internal/fetcher/common.go: -------------------------------------------------------------------------------- 1 | package fetcher 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http" 7 | "strings" 8 | 9 | "github.com/go-resty/resty/v2" 10 | 11 | "github.com/bookstairs/bookhunter/internal/client" 12 | "github.com/bookstairs/bookhunter/internal/file" 13 | ) 14 | 15 | var ( 16 | ErrOverrideRedirectHandler = errors.New("couldn't override the existed redirect handler") 17 | ErrFileNotExist = errors.New("current file does not exist") 18 | ) 19 | 20 | type Category string // The fetcher service identity. 21 | 22 | const ( 23 | Talebook Category = "talebook" 24 | SoBooks Category = "sobooks" 25 | Telegram Category = "telegram" 26 | K12 Category = "k12" 27 | Hsu Category = "hsu" 28 | ) 29 | 30 | // Config is used to define a common config for a specified fetcher service. 31 | type Config struct { 32 | Category Category // The identity of the fetcher service. 33 | Formats []file.Format // The formats that the user wants. 34 | Keywords []string // The keywords that the user wants. 35 | Extract bool // Extract the archives after download. 36 | DownloadPath string // The path for storing the file. 37 | InitialBookID int64 // The book id start to download. 38 | Rename bool // Rename the file by using book ID. 39 | Thread int // The number of download threads. 40 | RateLimit int // Request per minute for a thread. 41 | Retry int // The retry times for a failed download. 42 | SkipError bool // Continue to download the next book if the current book download failed. 43 | processFile string // Define the download process. 44 | 45 | // The extra configuration for a custom fetcher services. 46 | Properties map[string]string 47 | 48 | *client.Config 49 | } 50 | 51 | // Property will require an existed property from the config. 52 | func (c *Config) Property(name string) string { 53 | if v, ok := c.Properties[name]; ok { 54 | return v 55 | } 56 | return "" 57 | } 58 | 59 | func (c *Config) SetRedirect(redirect func(*http.Request, []*http.Request) error) error { 60 | if c.Config.Redirect != nil { 61 | return ErrOverrideRedirectHandler 62 | } 63 | c.Config.Redirect = resty.RedirectPolicyFunc(redirect) 64 | 65 | return nil 66 | } 67 | 68 | // ParseFormats will create the format array from the string slice. 69 | func ParseFormats(formats []string) ([]file.Format, error) { 70 | var fs []file.Format 71 | for _, format := range formats { 72 | f, err := ParseFormat(format) 73 | if err != nil { 74 | return nil, err 75 | } 76 | fs = append(fs, f) 77 | } 78 | return fs, nil 79 | } 80 | 81 | // ParseFormat will create the format from the string. 82 | func ParseFormat(format string) (file.Format, error) { 83 | f := file.Format(strings.ToLower(format)) 84 | if !IsValidFormat(f) { 85 | return "", fmt.Errorf("invalid format %s", format) 86 | } 87 | return f, nil 88 | } 89 | 90 | // IsValidFormat judge if this format was supported. 91 | func IsValidFormat(format file.Format) bool { 92 | switch format { 93 | case file.EPUB: 94 | return true 95 | case file.MOBI: 96 | return true 97 | case file.AZW: 98 | return true 99 | case file.AZW3: 100 | return true 101 | case file.PDF: 102 | return true 103 | case file.ZIP: 104 | return true 105 | default: 106 | return false 107 | } 108 | } 109 | 110 | // New create a fetcher service for downloading books. 111 | func New(c *Config) (Fetcher, error) { 112 | s, err := newService(c) 113 | if err != nil { 114 | return nil, err 115 | } 116 | 117 | return &fetcher{Config: c, service: s}, nil 118 | } 119 | -------------------------------------------------------------------------------- /internal/fetcher/fetcher.go: -------------------------------------------------------------------------------- 1 | package fetcher 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "strconv" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | "github.com/bookstairs/bookhunter/internal/driver" 14 | "github.com/bookstairs/bookhunter/internal/file" 15 | "github.com/bookstairs/bookhunter/internal/log" 16 | "github.com/bookstairs/bookhunter/internal/progress" 17 | ) 18 | 19 | const ( 20 | defaultProgressFile = "progress.db" 21 | ) 22 | 23 | // Fetcher exposes the download method to the command line. 24 | type Fetcher interface { 25 | // Download the books from the given service. 26 | Download() error 27 | } 28 | 29 | // fetcher is the basic common download service the multiple thread support. 30 | type fetcher struct { 31 | *Config 32 | service service 33 | progress progress.Progress 34 | creator file.Creator 35 | errs chan error 36 | } 37 | 38 | // Download the books from the given service. 39 | func (f *fetcher) Download() error { 40 | // Create the config path. 41 | configPath, err := f.ConfigPath() 42 | if err != nil { 43 | return err 44 | } 45 | 46 | // Query the total download amount from the given service. 47 | size, err := f.service.size() 48 | if err != nil { 49 | return err 50 | } 51 | log.Infof("Successfully query the download content counts: %d", size) 52 | 53 | // Create download progress with rate limit. 54 | if f.processFile == "" { 55 | if len(f.Keywords) == 0 { 56 | f.processFile = defaultProgressFile 57 | } else { 58 | // Avoid the download progress overloading. 59 | f.processFile = strconv.FormatInt(time.Now().Unix(), 10) + defaultProgressFile 60 | } 61 | } 62 | rate := f.RateLimit * f.Thread 63 | f.progress, err = progress.NewProgress(f.InitialBookID, size, rate, filepath.Join(configPath, f.processFile)) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | // Create the download directory if it's not existed. 69 | err = os.MkdirAll(f.DownloadPath, 0o755) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | // Create the file creator. 75 | f.creator = file.NewCreator(f.Rename, f.DownloadPath, f.Formats, f.Extract) 76 | 77 | // Create the download thread and save the files. 78 | f.errs = make(chan error, f.Thread) 79 | defer close(f.errs) 80 | 81 | var wait sync.WaitGroup 82 | for i := 0; i < f.Thread; i++ { 83 | wait.Add(1) 84 | go func() { 85 | defer wait.Done() 86 | f.startDownload() 87 | }() 88 | } 89 | wait.Wait() 90 | 91 | // Acquire the download errors. 92 | select { 93 | case err := <-f.errs: 94 | return err 95 | default: 96 | log.Debug("All the fetch thread have been finished.") 97 | } 98 | 99 | return nil 100 | } 101 | 102 | // startDownload will start a download thread. 103 | func (f *fetcher) startDownload() { //nolint:gocyclo 104 | thread: 105 | for { 106 | bookID := f.progress.AcquireBookID() 107 | if bookID == progress.NoBookToDownload { 108 | // Finish this thread. 109 | log.Debugf("No book to download in [%s] service.", f.Category) 110 | break thread 111 | } 112 | 113 | // Start downloading the given book ID. 114 | // The error will be sent to the channel. 115 | 116 | // Acquire the available file formats 117 | formats, err := f.service.formats(bookID) 118 | if err != nil { 119 | f.errs <- err 120 | break thread 121 | } 122 | log.Debugf("Book id %d formats: %v.", bookID, formats) 123 | 124 | // Filter the formats. 125 | formats = f.filterFormats(formats) 126 | if len(formats) == 0 { 127 | log.Warnf("[%d/%d] No downloadable files found.", bookID, f.progress.Size()) 128 | } 129 | 130 | // Filter the name, skip the progress if the name isn't the desired one. 131 | if len(formats) != 0 && len(f.Keywords) != 0 { 132 | formats = f.filterNames(formats) 133 | if len(formats) == 0 { 134 | log.Warnf("[%d/%d] The files found by the given keywords", bookID, f.progress.Size()) 135 | // No need to save the download progress. 136 | continue 137 | } 138 | } 139 | 140 | // Download the file by formats one by one. 141 | for format, share := range formats { 142 | err := f.downloadFile(bookID, format, share) 143 | for retry := 0; err != nil && !errors.Is(err, ErrFileNotExist) && retry < f.Retry; retry++ { 144 | fmt.Printf("Download book id %d failed: %v, retry (%d/%d)\n", bookID, err, retry, f.Retry) 145 | err = f.downloadFile(bookID, format, share) 146 | } 147 | 148 | if err != nil && !errors.Is(err, ErrFileNotExist) { 149 | fmt.Printf("Download book id %d failed: %v\n", bookID, err) 150 | if !f.SkipError { 151 | f.errs <- err 152 | break thread 153 | } 154 | } 155 | } 156 | 157 | // Save the download progress 158 | err = f.progress.SaveBookID(bookID) 159 | if err != nil { 160 | f.errs <- err 161 | break thread 162 | } 163 | } 164 | } 165 | 166 | // downloadFile in a thread. 167 | func (f *fetcher) downloadFile(bookID int64, format file.Format, share driver.Share) error { 168 | f.progress.TakeRateLimit() 169 | log.Debugf("Start download book id %d, format %s, share %v.", bookID, format, share) 170 | // Create the file writer. 171 | writer, err := f.creator.NewWriter(bookID, f.progress.Size(), share.FileName, share.SubPath, format, share.Size) 172 | if err != nil { 173 | return err 174 | } 175 | defer func() { _ = writer.Close() }() 176 | 177 | // Write file content. 178 | return f.service.fetch(bookID, format, share, writer) 179 | } 180 | 181 | // filterFormats will find the valid formats by user configure. 182 | func (f *fetcher) filterFormats(formats map[file.Format]driver.Share) map[file.Format]driver.Share { 183 | fs := make(map[file.Format]driver.Share) 184 | for format, share := range formats { 185 | for _, vf := range f.Formats { 186 | if format == vf { 187 | fs[format] = share 188 | break 189 | } 190 | } 191 | } 192 | return fs 193 | } 194 | 195 | func (f *fetcher) filterNames(formats map[file.Format]driver.Share) map[file.Format]driver.Share { 196 | fs := make(map[file.Format]driver.Share) 197 | for format, share := range formats { 198 | if matchKeywords(share.FileName, f.Keywords) { 199 | fs[format] = share 200 | } 201 | } 202 | return fs 203 | } 204 | 205 | func matchKeywords(title string, keywords []string) bool { 206 | for _, keyword := range keywords { 207 | // Should we support the regular expression? 208 | if strings.Contains(title, keyword) { 209 | return true 210 | } 211 | } 212 | 213 | return false 214 | } 215 | -------------------------------------------------------------------------------- /internal/fetcher/hsu.go: -------------------------------------------------------------------------------- 1 | package fetcher 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "sort" 7 | "strconv" 8 | 9 | "github.com/bookstairs/bookhunter/internal/client" 10 | "github.com/bookstairs/bookhunter/internal/driver" 11 | "github.com/bookstairs/bookhunter/internal/file" 12 | ) 13 | 14 | // HsuBookMeta is used for representing book information. 15 | // https://book.hsu.life/api/series/all-v2 16 | type HsuBookMeta struct { 17 | ID int `json:"id"` 18 | Name string `json:"name"` 19 | OriginalName string `json:"originalName"` 20 | LocalizedName string `json:"localizedName"` 21 | SortName string `json:"sortName"` 22 | Pages int `json:"pages"` 23 | CoverImageLocked bool `json:"coverImageLocked"` 24 | PagesRead int `json:"pagesRead"` 25 | LatestReadDate string `json:"latestReadDate"` 26 | LastChapterAdded string `json:"lastChapterAdded"` 27 | UserRating int `json:"userRating"` 28 | HasUserRated bool `json:"hasUserRated"` 29 | Format int `json:"format"` 30 | Created string `json:"created"` 31 | NameLocked bool `json:"nameLocked"` 32 | SortNameLocked bool `json:"sortNameLocked"` 33 | LocalizedNameLocked bool `json:"localizedNameLocked"` 34 | WordCount int `json:"wordCount"` 35 | LibraryID int `json:"libraryId"` 36 | LibraryName string `json:"libraryName"` 37 | MinHoursToRead int `json:"minHoursToRead"` 38 | MaxHoursToRead int `json:"maxHoursToRead"` 39 | AvgHoursToRead int `json:"avgHoursToRead"` 40 | FolderPath string `json:"folderPath"` 41 | LastFolderScanned string `json:"lastFolderScanned"` 42 | } 43 | 44 | type HsuBookMetaReq struct { 45 | Statements []struct { 46 | Comparison int `json:"comparison"` 47 | Value string `json:"value"` 48 | Field int `json:"field"` 49 | } `json:"statements"` 50 | Combination int `json:"combination"` 51 | LimitTo int `json:"limitTo"` 52 | SortOptions struct { 53 | IsAscending bool `json:"isAscending"` 54 | SortField int `json:"sortField"` 55 | } `json:"sortOptions"` 56 | } 57 | 58 | // HsuLoginReq is used in POST https://book.hsu.life/api/account/login 59 | type HsuLoginReq struct { 60 | Username string `json:"username"` 61 | Password string `json:"password"` 62 | APIKey string `json:"apiKey"` 63 | } 64 | 65 | // HsuLoginResp is used in https://book.hsu.life/api/account/login response 66 | type HsuLoginResp struct { 67 | Username string `json:"username"` 68 | Email string `json:"email"` 69 | Token string `json:"token"` 70 | RefreshToken string `json:"refreshToken"` 71 | APIKey string `json:"apiKey"` 72 | Preferences struct { 73 | ReadingDirection int `json:"readingDirection"` 74 | ScalingOption int `json:"scalingOption"` 75 | PageSplitOption int `json:"pageSplitOption"` 76 | ReaderMode int `json:"readerMode"` 77 | LayoutMode int `json:"layoutMode"` 78 | EmulateBook bool `json:"emulateBook"` 79 | BackgroundColor string `json:"backgroundColor"` 80 | SwipeToPaginate bool `json:"swipeToPaginate"` 81 | AutoCloseMenu bool `json:"autoCloseMenu"` 82 | ShowScreenHints bool `json:"showScreenHints"` 83 | BookReaderMargin int `json:"bookReaderMargin"` 84 | BookReaderLineSpacing int `json:"bookReaderLineSpacing"` 85 | BookReaderFontSize int `json:"bookReaderFontSize"` 86 | BookReaderFontFamily string `json:"bookReaderFontFamily"` 87 | BookReaderTapToPaginate bool `json:"bookReaderTapToPaginate"` 88 | BookReaderReadingDirection int `json:"bookReaderReadingDirection"` 89 | BookReaderWritingStyle int `json:"bookReaderWritingStyle"` 90 | Theme struct { 91 | ID int `json:"id"` 92 | Name string `json:"name"` 93 | NormalizedName string `json:"normalizedName"` 94 | FileName string `json:"fileName"` 95 | IsDefault bool `json:"isDefault"` 96 | Provider int `json:"provider"` 97 | Created string `json:"created"` 98 | LastModified string `json:"lastModified"` 99 | CreatedUtc string `json:"createdUtc"` 100 | LastModifiedUtc string `json:"lastModifiedUtc"` 101 | } `json:"theme"` 102 | BookReaderThemeName string `json:"bookReaderThemeName"` 103 | BookReaderLayoutMode int `json:"bookReaderLayoutMode"` 104 | BookReaderImmersiveMode bool `json:"bookReaderImmersiveMode"` 105 | GlobalPageLayoutMode int `json:"globalPageLayoutMode"` 106 | BlurUnreadSummaries bool `json:"blurUnreadSummaries"` 107 | PromptForDownloadSize bool `json:"promptForDownloadSize"` 108 | NoTransitions bool `json:"noTransitions"` 109 | CollapseSeriesRelationships bool `json:"collapseSeriesRelationships"` 110 | ShareReviews bool `json:"shareReviews"` 111 | Locale string `json:"locale"` 112 | } `json:"preferences"` 113 | AgeRestriction struct { 114 | AgeRating int `json:"ageRating"` 115 | IncludeUnknowns bool `json:"includeUnknowns"` 116 | } `json:"ageRestriction"` 117 | KavitaVersion string `json:"kavitaVersion"` 118 | } 119 | 120 | type hsuService struct { 121 | config *Config 122 | books []*HsuBookMeta 123 | *client.Client 124 | } 125 | 126 | // formatMapping is defined in https://github.com/Kareadita/Kavita/blob/develop/UI/Web/src/app/_models/manga-format.ts 127 | var formatMapping = map[int]file.Format{ 128 | 1: file.ZIP, 129 | 3: file.EPUB, 130 | 4: file.PDF, 131 | } 132 | 133 | func newHsuService(config *Config) (service, error) { 134 | // Create the resty client for HTTP handing. 135 | c, err := client.New(config.Config) 136 | if err != nil { 137 | return nil, err 138 | } 139 | 140 | resp, err := c.R(). 141 | SetBody(&HsuLoginReq{ 142 | Username: config.Property("username"), 143 | Password: config.Property("password"), 144 | APIKey: "", 145 | }). 146 | SetResult(&HsuLoginResp{}). 147 | ForceContentType("application/json"). 148 | Post("/api/account/login") 149 | 150 | if err != nil { 151 | return nil, err 152 | } 153 | 154 | token := resp.Result().(*HsuLoginResp).Token 155 | if token == "" { 156 | return nil, fmt.Errorf("invalid login credential") 157 | } 158 | c.SetAuthToken(token) 159 | 160 | // Download books. 161 | resp, err = c.R(). 162 | SetBody(&HsuBookMetaReq{ 163 | Statements: []struct { 164 | Comparison int `json:"comparison"` 165 | Value string `json:"value"` 166 | Field int `json:"field"` 167 | }{ 168 | { 169 | Comparison: 0, 170 | Value: "", 171 | Field: 1, 172 | }, 173 | }, 174 | Combination: 1, 175 | LimitTo: 0, 176 | SortOptions: struct { 177 | IsAscending bool `json:"isAscending"` 178 | SortField int `json:"sortField"` 179 | }{ 180 | IsAscending: true, 181 | SortField: 1, 182 | }, 183 | }). 184 | SetResult(&[]HsuBookMeta{}). 185 | Post("/api/series/all-v2") 186 | 187 | if err != nil { 188 | return nil, err 189 | } 190 | 191 | metas := *resp.Result().(*[]HsuBookMeta) 192 | sort.Slice(metas, func(i, j int) bool { 193 | return metas[i].ID < metas[j].ID 194 | }) 195 | 196 | // Create a better slice for holding the books. 197 | books := make([]*HsuBookMeta, metas[len(metas)-1].ID) 198 | for i := range metas { 199 | meta := metas[i] 200 | books[meta.ID-1] = &meta 201 | } 202 | 203 | return &hsuService{config: config, Client: c, books: books}, nil 204 | } 205 | 206 | func (h *hsuService) size() (int64, error) { 207 | return int64(len(h.books)), nil 208 | } 209 | 210 | func (h *hsuService) formats(i int64) (map[file.Format]driver.Share, error) { 211 | book := h.books[i-1] 212 | 213 | if book != nil { 214 | if format, ok := formatMapping[book.Format]; ok { 215 | resp, err := h.R(). 216 | SetQueryParam("seriesId", strconv.Itoa(int(i))). 217 | Get("/api/download/series-size") 218 | if err != nil { 219 | return nil, err 220 | } 221 | 222 | fileSize := resp.String() 223 | if fileSize == "" { 224 | return nil, fmt.Errorf("you are not allowed to download books") 225 | } 226 | size, err := strconv.ParseInt(fileSize, 10, 64) 227 | if err != nil { 228 | return nil, err 229 | } 230 | 231 | return map[file.Format]driver.Share{ 232 | format: { 233 | FileName: book.Name, 234 | Size: size, 235 | }, 236 | }, nil 237 | } 238 | } 239 | 240 | return make(map[file.Format]driver.Share), nil 241 | } 242 | 243 | func (h *hsuService) fetch(i int64, _ file.Format, _ driver.Share, writer file.Writer) error { 244 | resp, err := h.R(). 245 | SetDoNotParseResponse(true). 246 | SetQueryParam("seriesId", strconv.Itoa(int(i))). 247 | Get("/api/download/series") 248 | if err != nil { 249 | return err 250 | } 251 | 252 | body := resp.RawBody() 253 | defer func() { _ = body.Close() }() 254 | 255 | // Save the download content info files. 256 | _, err = io.Copy(writer, body) 257 | return err 258 | } 259 | -------------------------------------------------------------------------------- /internal/fetcher/k12.go: -------------------------------------------------------------------------------- 1 | package fetcher 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "path/filepath" 7 | "strings" 8 | 9 | "github.com/bookstairs/bookhunter/internal/client" 10 | "github.com/bookstairs/bookhunter/internal/driver" 11 | "github.com/bookstairs/bookhunter/internal/file" 12 | ) 13 | 14 | const ( 15 | textBookMetadata = "https://s-file-2.ykt.cbern.com.cn/zxx/ndrs/resources/tch_material/version/data_version.json" 16 | downloadLinkTmpl = "https://r1-ndr.ykt.cbern.com.cn/edu_product/esp/assets_document/%s.pkg/pdf.pdf" 17 | ) 18 | 19 | func newK12Service(config *Config) (service, error) { 20 | c, err := client.New(config.Config) 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | return &k12Service{Client: c, config: config, metadata: map[int64]TextBook{}}, nil 26 | } 27 | 28 | type k12Service struct { 29 | *client.Client 30 | config *Config 31 | metadata map[int64]TextBook 32 | } 33 | 34 | func (k *k12Service) size() (int64, error) { 35 | resp, err := k.R(). 36 | SetResult(&MetadataSource{}). 37 | Get(textBookMetadata) 38 | if err != nil { 39 | return 0, err 40 | } 41 | 42 | res := resp.Result().(*MetadataSource) 43 | id := int64(1) 44 | 45 | for _, url := range strings.Split(res.URLs, ",") { 46 | books, err := k.downloadMetadata(url) 47 | if err != nil { 48 | return 0, err 49 | } 50 | 51 | for idx := range books { 52 | book := books[idx] 53 | k.metadata[id] = book 54 | id++ 55 | } 56 | } 57 | 58 | return int64(len(k.metadata)), nil 59 | } 60 | 61 | func (k *k12Service) downloadMetadata(url string) ([]TextBook, error) { 62 | resp, err := k.R().SetResult([]TextBook{}).Get(url) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | return *resp.Result().(*[]TextBook), nil 68 | } 69 | 70 | func (k *k12Service) formats(id int64) (map[file.Format]driver.Share, error) { 71 | book, ok := k.metadata[id] 72 | if !ok { 73 | return map[file.Format]driver.Share{}, nil 74 | } 75 | 76 | tags := make(map[string]string, len(book.TagList)) 77 | for _, tag := range book.TagList { 78 | tags[tag.TagID] = tag.TagName 79 | } 80 | 81 | subPath := "" 82 | for _, id := range strings.Split(book.TagPaths[0], "/") { 83 | if name := tags[id]; name != "" { 84 | if subPath == "" { 85 | subPath = name 86 | } else { 87 | subPath = filepath.Join(subPath, name) 88 | } 89 | } 90 | } 91 | 92 | return map[file.Format]driver.Share{ 93 | file.Format(book.CustomProperties.Format): { 94 | FileName: book.Title, 95 | SubPath: subPath, 96 | Size: book.CustomProperties.Size, 97 | URL: book.ID, 98 | Properties: nil, 99 | }, 100 | }, nil 101 | } 102 | 103 | func (k *k12Service) fetch(_ int64, _ file.Format, share driver.Share, writer file.Writer) error { 104 | resp, err := k.R().SetDoNotParseResponse(true).Get(fmt.Sprintf(downloadLinkTmpl, share.URL)) 105 | if err != nil { 106 | return err 107 | } 108 | 109 | body := resp.RawBody() 110 | defer func() { _ = body.Close() }() 111 | 112 | // Save the download content info files. 113 | _, err = io.Copy(writer, body) 114 | return err 115 | } 116 | 117 | type MetadataSource struct { 118 | URLs string `json:"urls"` 119 | } 120 | 121 | type TextBook struct { 122 | ID string `json:"id"` 123 | CustomProperties struct { 124 | AutoFillThumb bool `json:"auto_fill_thumb"` 125 | ExtProperties struct { 126 | CatalogType string `json:"catalog_type"` 127 | LibraryID string `json:"library_id"` 128 | SubCatalog string `json:"sub_catalog"` 129 | } `json:"ext_properties"` 130 | Format string `json:"format"` 131 | Height string `json:"height"` 132 | IsTop int `json:"is_top"` 133 | Providers []string `json:"providers"` 134 | ResSort int64 `json:"res_sort"` 135 | Resolution string `json:"resolution"` 136 | Size int64 `json:"size"` 137 | SysTransStatus string `json:"sys_trans_status"` 138 | Thumbnails []string `json:"thumbnails"` 139 | Width string `json:"width"` 140 | } `json:"custom_properties"` 141 | ResourceTypeCode string `json:"resource_type_code"` 142 | Language string `json:"language"` 143 | Provider string `json:"provider"` 144 | CreateTime string `json:"create_time"` 145 | UpdateTime string `json:"update_time"` 146 | TagList []struct { 147 | TagID string `json:"tag_id"` 148 | TagName string `json:"tag_name"` 149 | TagDimensionID string `json:"tag_dimension_id"` 150 | OrderNum int `json:"order_num"` 151 | } `json:"tag_list"` 152 | Status string `json:"status"` 153 | CreateContainerID string `json:"create_container_id"` 154 | ResourceTypeCodeName string `json:"resource_type_code_name"` 155 | OnlineTime string `json:"online_time"` 156 | TagPaths []string `json:"tag_paths"` 157 | ContainerID string `json:"container_id"` 158 | TenantID string `json:"tenant_id"` 159 | Title string `json:"title"` 160 | Label []string `json:"label"` 161 | Description string `json:"description"` 162 | ProviderList []struct { 163 | Name string `json:"name"` 164 | ID string `json:"id"` 165 | Type string `json:"type"` 166 | } `json:"provider_list"` 167 | } 168 | -------------------------------------------------------------------------------- /internal/fetcher/service.go: -------------------------------------------------------------------------------- 1 | package fetcher 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/bookstairs/bookhunter/internal/driver" 7 | "github.com/bookstairs/bookhunter/internal/file" 8 | ) 9 | 10 | // service is a real implementation for fetcher. 11 | type service interface { 12 | // size is the total download amount of this given service. 13 | size() (int64, error) 14 | 15 | // formats will query the available downloadable file formats. 16 | formats(int64) (map[file.Format]driver.Share, error) 17 | 18 | // fetch the given book ID. 19 | fetch(int64, file.Format, driver.Share, file.Writer) error 20 | } 21 | 22 | // newService is the endpoint for creating all the supported download service. 23 | func newService(c *Config) (service, error) { 24 | switch c.Category { 25 | case Talebook: 26 | return newTalebookService(c) 27 | case SoBooks: 28 | return newSobooksService(c) 29 | case Telegram: 30 | return newTelegramService(c) 31 | case K12: 32 | return newK12Service(c) 33 | case Hsu: 34 | return newHsuService(c) 35 | default: 36 | return nil, fmt.Errorf("no such fetcher service [%s] supported", c.Category) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /internal/fetcher/sobooks.go: -------------------------------------------------------------------------------- 1 | package fetcher 2 | 3 | import ( 4 | "crypto/tls" 5 | "io" 6 | "net/http" 7 | "net/url" 8 | "regexp" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/PuerkitoBio/goquery" 13 | 14 | "github.com/bookstairs/bookhunter/internal/client" 15 | "github.com/bookstairs/bookhunter/internal/driver" 16 | "github.com/bookstairs/bookhunter/internal/file" 17 | "github.com/bookstairs/bookhunter/internal/log" 18 | "github.com/bookstairs/bookhunter/internal/sobooks" 19 | ) 20 | 21 | var ( 22 | sobooksIDRe = regexp.MustCompile(`.*?/(\d+?).html`) 23 | ) 24 | 25 | type sobooksService struct { 26 | config *Config 27 | *client.Client 28 | driver driver.Driver 29 | } 30 | 31 | func newSobooksService(config *Config) (service, error) { 32 | // Create the resty client for HTTP handing. 33 | c, err := client.New(config.Config) 34 | // Set code for viewing hidden content 35 | c.SetCookie(&http.Cookie{ 36 | Name: "mpcode", 37 | Value: config.Property("code"), 38 | Path: "/", 39 | Domain: config.Host, 40 | }).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) //nolint:gosec 41 | 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | // Create the net disk driver. 47 | d, err := driver.New(config.Config, config.Properties) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | return &sobooksService{config: config, Client: c, driver: d}, nil 53 | } 54 | 55 | func (s *sobooksService) size() (int64, error) { 56 | resp, err := s.R(). 57 | Get("/") 58 | if err != nil { 59 | return 0, err 60 | } 61 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(resp.String())) 62 | if err != nil { 63 | return 0, err 64 | } 65 | 66 | lastID := -1 67 | 68 | // Find all the links is case of the website primary changed the theme. 69 | doc.Find("a").Each(func(i int, selection *goquery.Selection) { 70 | // This is a book link. 71 | link, exists := selection.Attr("href") 72 | if exists { 73 | // Extract bookId. 74 | match := sobooksIDRe.FindStringSubmatch(link) 75 | if len(match) > 0 { 76 | id, _ := strconv.Atoi(match[1]) 77 | if id > lastID { 78 | lastID = id 79 | } 80 | } 81 | } 82 | }) 83 | return int64(lastID), nil 84 | } 85 | 86 | func (s *sobooksService) formats(id int64) (map[file.Format]driver.Share, error) { 87 | resp, err := s.R(). 88 | SetPathParam("bookId", strconv.FormatInt(id, 10)). 89 | SetHeader("referer", s.BaseURL). 90 | Get("/books/{bookId}.html") 91 | if err != nil { 92 | return nil, err 93 | } 94 | if resp.IsError() { 95 | log.Debugf("The current book [%v] content does not exist ", id) 96 | return map[file.Format]driver.Share{}, nil 97 | } 98 | 99 | title, links, err := sobooks.ParseLinks(resp.String(), id) 100 | if err != nil { 101 | return map[file.Format]driver.Share{}, nil 102 | } 103 | 104 | res := make(map[file.Format]driver.Share) 105 | for source, link := range links { 106 | if source != s.driver.Source() { 107 | continue 108 | } 109 | 110 | shares, err := s.driver.Resolve(link.URL, link.Code) 111 | if err != nil { 112 | return nil, err 113 | } 114 | 115 | for _, share := range shares { 116 | if ext, has := file.LinkExtension(share.FileName); has { 117 | if IsValidFormat(ext) { 118 | res[ext] = share 119 | } else { 120 | log.Debugf("The file name %s don't have valid extension %s", share.FileName, ext) 121 | } 122 | } else { 123 | log.Debugf("The file name %s don't have the extension", share.FileName) 124 | } 125 | } 126 | } 127 | 128 | if link, ok := links[driver.DIRECT]; ok && len(res) == 0 { 129 | res[file.EPUB] = driver.Share{ 130 | FileName: title + ".epub", 131 | Size: 0, 132 | URL: link.URL, 133 | } 134 | } 135 | return res, nil 136 | } 137 | 138 | func (s *sobooksService) fetch(_ int64, _ file.Format, share driver.Share, writer file.Writer) error { 139 | u, err := url.Parse(share.URL) 140 | if err != nil { 141 | return err 142 | } 143 | resp, err := s.R(). 144 | SetDoNotParseResponse(true). 145 | Get(u.String()) 146 | if err != nil { 147 | return err 148 | } 149 | 150 | if resp.StatusCode() == 404 { 151 | return ErrFileNotExist 152 | } 153 | body := resp.RawBody() 154 | defer func() { _ = body.Close() }() 155 | 156 | // Save the download content info files. 157 | writer.SetSize(resp.RawResponse.ContentLength) 158 | _, err = io.Copy(writer, body) 159 | return err 160 | } 161 | -------------------------------------------------------------------------------- /internal/fetcher/talebook.go: -------------------------------------------------------------------------------- 1 | package fetcher 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "strconv" 9 | "strings" 10 | 11 | "github.com/bookstairs/bookhunter/internal/client" 12 | "github.com/bookstairs/bookhunter/internal/driver" 13 | "github.com/bookstairs/bookhunter/internal/file" 14 | "github.com/bookstairs/bookhunter/internal/log" 15 | "github.com/bookstairs/bookhunter/internal/talebook" 16 | ) 17 | 18 | var ( 19 | ErrTalebookNeedSignin = errors.New("need user account to download books") 20 | ErrEmptyTalebook = errors.New("couldn't find available books in talebook") 21 | 22 | redirectHandler = func(request *http.Request, requests []*http.Request) error { 23 | if request.URL.Path == "/login" { 24 | return ErrTalebookNeedSignin 25 | } 26 | return nil 27 | } 28 | ) 29 | 30 | type talebookService struct { 31 | config *Config 32 | *client.Client 33 | } 34 | 35 | func newTalebookService(config *Config) (service, error) { 36 | // Add login check in redirect handler. 37 | if err := config.SetRedirect(redirectHandler); err != nil { 38 | return nil, err 39 | } 40 | 41 | // Create the resty client for HTTP handing. 42 | c, err := client.New(config.Config) 43 | if err != nil { 44 | return nil, err 45 | } 46 | 47 | // Start to sign in if required. 48 | username := config.Property("username") 49 | password := config.Property("password") 50 | if username != "" && password != "" { 51 | log.Info("You have provided user information, start to login.") 52 | resp, err := c.R(). 53 | SetFormData(map[string]string{ 54 | "username": username, 55 | "password": password, 56 | }). 57 | SetResult(&talebook.LoginResp{}). 58 | ForceContentType("application/json"). 59 | Post("/api/user/sign_in") 60 | 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | result := resp.Result().(*talebook.LoginResp) 66 | if result.Err != talebook.SuccessStatus { 67 | return nil, errors.New(result.Msg) 68 | } 69 | 70 | log.Info("Login success. Save cookies into file.") 71 | } 72 | 73 | return &talebookService{config: config, Client: c}, nil 74 | } 75 | 76 | func (t *talebookService) size() (int64, error) { 77 | resp, err := t.R(). 78 | SetResult(&talebook.BooksResp{}). 79 | Get("/api/recent") 80 | if err != nil { 81 | return 0, err 82 | } 83 | 84 | result := resp.Result().(*talebook.BooksResp) 85 | if result.Err != talebook.SuccessStatus { 86 | return 0, errors.New(result.Msg) 87 | } 88 | 89 | bookID := int64(0) 90 | for _, book := range result.Books { 91 | if book.ID > bookID { 92 | bookID = book.ID 93 | } 94 | } 95 | 96 | if bookID == 0 { 97 | return 0, ErrEmptyTalebook 98 | } 99 | 100 | return bookID, nil 101 | } 102 | 103 | func (t *talebookService) formats(id int64) (map[file.Format]driver.Share, error) { 104 | resp, err := t.R(). 105 | SetResult(&talebook.BookResp{}). 106 | SetPathParam("bookID", strconv.FormatInt(id, 10)). 107 | Get("/api/book/{bookID}") 108 | if err != nil { 109 | return nil, err 110 | } 111 | 112 | result := resp.Result().(*talebook.BookResp) 113 | switch result.Err { 114 | case talebook.SuccessStatus: 115 | formats := make(map[file.Format]driver.Share) 116 | for _, f := range result.Book.Files { 117 | format, err := ParseFormat(strings.ToLower(f.Format)) 118 | if err != nil { 119 | return nil, err 120 | } 121 | formats[format] = driver.Share{ 122 | FileName: fmt.Sprintf("%s.%s", result.Book.Title, format), 123 | URL: f.Href, 124 | } 125 | } 126 | return formats, nil 127 | case talebook.BookNotFoundStatus: 128 | return nil, nil 129 | default: 130 | return nil, errors.New(result.Msg) 131 | } 132 | } 133 | 134 | func (t *talebookService) fetch(_ int64, _ file.Format, share driver.Share, writer file.Writer) error { 135 | resp, err := t.R(). 136 | SetDoNotParseResponse(true). 137 | Get(share.URL) 138 | if err != nil { 139 | return err 140 | } 141 | body := resp.RawBody() 142 | defer func() { _ = body.Close() }() 143 | 144 | // Save the download content info files. 145 | _, err = io.Copy(writer, body) 146 | return err 147 | } 148 | -------------------------------------------------------------------------------- /internal/fetcher/telegram.go: -------------------------------------------------------------------------------- 1 | package fetcher 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "strconv" 7 | "strings" 8 | "time" 9 | 10 | "github.com/gotd/td/tg" 11 | 12 | "github.com/bookstairs/bookhunter/internal/driver" 13 | "github.com/bookstairs/bookhunter/internal/file" 14 | "github.com/bookstairs/bookhunter/internal/telegram" 15 | ) 16 | 17 | func newTelegramService(config *Config) (service, error) { 18 | // Create the session file. 19 | path, err := config.ConfigPath() 20 | if err != nil { 21 | return nil, err 22 | } 23 | sessionPath := filepath.Join(path, "session.db") 24 | if refresh, _ := strconv.ParseBool(config.Property("reLogin")); refresh { 25 | _ = os.Remove(sessionPath) 26 | } 27 | 28 | channelID := config.Property("channelID") 29 | mobile := config.Property("mobile") 30 | appID, _ := strconv.ParseInt(config.Property("appID"), 10, 64) 31 | appHash := config.Property("appHash") 32 | 33 | // Change the process file name. 34 | config.processFile = strings.ReplaceAll(channelID, "/", "_") + ".db" 35 | if len(config.Keywords) != 0 { 36 | config.processFile = strconv.FormatInt(time.Now().Unix(), 10) + config.processFile 37 | } 38 | 39 | tel, err := telegram.New(channelID, mobile, appID, appHash, sessionPath, config.Proxy) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | return &telegramService{config: config, telegram: tel}, nil 45 | } 46 | 47 | type telegramService struct { 48 | config *Config 49 | telegram *telegram.Telegram 50 | info *telegram.ChannelInfo 51 | } 52 | 53 | func (s *telegramService) size() (int64, error) { 54 | info, err := s.telegram.ChannelInfo() 55 | if err != nil { 56 | return 0, err 57 | } 58 | s.info = info 59 | 60 | return info.LastMsgID, nil 61 | } 62 | 63 | func (s *telegramService) formats(id int64) (map[file.Format]driver.Share, error) { 64 | files, err := s.telegram.ParseMessage(s.info, id) 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | res := make(map[file.Format]driver.Share) 70 | for _, f := range files { 71 | res[f.Format] = driver.Share{ 72 | FileName: f.Name, 73 | Size: f.Size, 74 | Properties: map[string]any{ 75 | "fileID": f.ID, 76 | "document": f.Document, 77 | }, 78 | } 79 | } 80 | 81 | return res, nil 82 | } 83 | 84 | func (s *telegramService) fetch(_ int64, f file.Format, share driver.Share, writer file.Writer) error { 85 | o := &telegram.File{ 86 | ID: share.Properties["fileID"].(int64), 87 | Name: share.FileName, 88 | Format: f, 89 | Size: share.Size, 90 | Document: share.Properties["document"].(*tg.InputDocumentFileLocation), 91 | } 92 | 93 | return s.telegram.DownloadFile(o, writer) 94 | } 95 | -------------------------------------------------------------------------------- /internal/file/decompress.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "fmt" 7 | "io" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | 12 | "golang.org/x/text/encoding/simplifiedchinese" 13 | "golang.org/x/text/transform" 14 | ) 15 | 16 | var encoding = simplifiedchinese.GB18030 17 | 18 | // decompress - extract zip file. 19 | func (p *writer) decompress() error { 20 | r, err := zip.OpenReader(p.filePath()) 21 | if err != nil { 22 | return err 23 | } 24 | defer func() { 25 | if err := r.Close(); err != nil { 26 | panic(err) 27 | } 28 | }() 29 | 30 | _ = os.MkdirAll(p.download, 0o755) 31 | 32 | for _, f := range r.File { 33 | if err := p.extractAndWriteFile(f); err != nil { 34 | return err 35 | } 36 | } 37 | 38 | return nil 39 | } 40 | 41 | // Closure to address file descriptors issue with all the deferred Close() methods. 42 | func (p *writer) extractAndWriteFile(f *zip.File) error { 43 | rc, err := f.Open() 44 | if err != nil { 45 | return err 46 | } 47 | defer func() { 48 | if err := rc.Close(); err != nil { 49 | panic(err) 50 | } 51 | }() 52 | 53 | filename := encodingFilename(f.Name) 54 | ext, ok := Extension(filename) 55 | if !ok || !p.formats[ext] { 56 | // No need to extract this file. Skip. 57 | return nil 58 | } 59 | 60 | path, err := sanitizeArchivePath(p.download, filename) 61 | if err != nil { 62 | return err 63 | } 64 | 65 | if !strings.HasPrefix(path, filepath.Clean(p.download)+string(os.PathSeparator)) { 66 | return fmt.Errorf("%s: illegal file path", path) 67 | } 68 | 69 | if f.FileInfo().IsDir() { 70 | _ = os.MkdirAll(path, 0o755) 71 | } else { 72 | mode := f.FileHeader.Mode() 73 | if mode&os.ModeType == os.ModeSymlink { 74 | data, err := io.ReadAll(rc) 75 | if err != nil { 76 | return err 77 | } 78 | _ = writeSymbolicLink(path, string(data)) 79 | } else { 80 | _ = os.MkdirAll(filepath.Dir(path), 0o755) 81 | _ = os.Remove(path) 82 | outFile, err := os.Create(path) 83 | if err != nil { 84 | return err 85 | } 86 | defer func() { 87 | if err := outFile.Close(); err != nil { 88 | panic(err) 89 | } 90 | }() 91 | 92 | // G110: Potential DoS vulnerability via decompression bomb. 93 | for { 94 | _, err := io.CopyN(outFile, rc, bytes.MinRead) 95 | if err != nil { 96 | if err == io.EOF { 97 | break 98 | } 99 | return err 100 | } 101 | } 102 | } 103 | } 104 | 105 | return nil 106 | } 107 | 108 | func writeSymbolicLink(filePath string, targetPath string) error { 109 | err := os.MkdirAll(filepath.Dir(filePath), 0o755) 110 | if err != nil { 111 | return err 112 | } 113 | 114 | err = os.Symlink(targetPath, filePath) 115 | if err != nil { 116 | return err 117 | } 118 | 119 | return nil 120 | } 121 | 122 | // sanitizeArchivePath sanitize archive file pathing from "G305: Zip Slip vulnerability" 123 | func sanitizeArchivePath(d, t string) (v string, err error) { 124 | v = filepath.Join(d, t) 125 | if strings.HasPrefix(v, filepath.Clean(d)) { 126 | return v, nil 127 | } 128 | 129 | return "", fmt.Errorf("%s: %s", "content filepath is tainted", t) 130 | } 131 | 132 | // encodingFilename will convert the GBK into UTF-8 and escape invalid characters. 133 | func encodingFilename(name string) string { 134 | i := bytes.NewReader([]byte(name)) 135 | decoder := transform.NewReader(i, encoding.NewDecoder()) 136 | content, err := io.ReadAll(decoder) 137 | if err != nil { 138 | // Fallback to default UTF-8 encoding 139 | return escape(name) 140 | } else { 141 | return escape(string(content)) 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /internal/file/formats.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "net/url" 5 | "strings" 6 | "unicode" 7 | ) 8 | 9 | type Format string // The supported file extension. 10 | 11 | const ( 12 | EPUB Format = "epub" 13 | MOBI Format = "mobi" 14 | AZW Format = "azw" 15 | AZW3 Format = "azw3" 16 | PDF Format = "pdf" 17 | ZIP Format = "zip" 18 | ) 19 | 20 | // The Archive will return if this format is an archive. 21 | func (f Format) Archive() bool { 22 | return f == ZIP 23 | } 24 | 25 | func isLetter(s string) bool { 26 | for _, r := range s { 27 | if !unicode.IsLetter(r) { 28 | return false 29 | } 30 | } 31 | return true 32 | } 33 | 34 | // LinkExtension the file extension from the link. 35 | func LinkExtension(link string) (Format, bool) { 36 | u, err := url.Parse(link) 37 | if err != nil { 38 | return "", false 39 | } 40 | return Extension(u.Path) 41 | } 42 | 43 | func Extension(filename string) (Format, bool) { 44 | start := strings.LastIndex(filename, ".") + 1 45 | ext := filename[start:] 46 | 47 | if isLetter(ext) { 48 | return Format(strings.ToLower(ext)), true 49 | } 50 | return "", false 51 | } 52 | -------------------------------------------------------------------------------- /internal/file/replacer.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | package file 4 | 5 | import "strings" 6 | 7 | var replacer = strings.NewReplacer( 8 | `/`, empty, 9 | `\`, empty, 10 | `*`, empty, 11 | `:`, empty, 12 | `"`, empty, 13 | ) 14 | -------------------------------------------------------------------------------- /internal/file/replacer_windows.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import "strings" 4 | 5 | var replacer = strings.NewReplacer( 6 | `<`, empty, 7 | `>`, empty, 8 | `/`, empty, 9 | `\`, empty, 10 | `%`, empty, 11 | `*`, empty, 12 | `:`, empty, 13 | `|`, empty, 14 | `"`, empty, 15 | `?`, empty, 16 | ) 17 | -------------------------------------------------------------------------------- /internal/file/writer.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "path/filepath" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/schollz/progressbar/v3" 11 | 12 | "github.com/bookstairs/bookhunter/internal/log" 13 | ) 14 | 15 | const ( 16 | maxLength = 60 17 | empty = " " 18 | ) 19 | 20 | // escape the filename in *nix like systems and limit the max name size. 21 | func escape(filename string) string { 22 | filename = replacer.Replace(filename) 23 | 24 | if name := []rune(filename); len(name) > maxLength { 25 | return string(name[0:maxLength]) 26 | } else { 27 | return filename 28 | } 29 | } 30 | 31 | func NewCreator(rename bool, downloadPath string, formats []Format, extract bool) Creator { 32 | fs := make(map[Format]bool) 33 | for _, format := range formats { 34 | fs[format] = true 35 | } 36 | 37 | return &creator{rename: rename, downloadPath: downloadPath, formats: fs, extract: extract} 38 | } 39 | 40 | type Creator interface { 41 | NewWriter(id, total int64, name, subPath string, format Format, size int64) (Writer, error) 42 | } 43 | 44 | type creator struct { 45 | rename bool 46 | extract bool 47 | formats map[Format]bool 48 | downloadPath string 49 | } 50 | 51 | func (c *creator) NewWriter(id, total int64, name, subPath string, format Format, size int64) (Writer, error) { 52 | // Rename if it was required. 53 | filename := strconv.FormatInt(id, 10) 54 | if c.rename { 55 | filename = filename + "." + string(format) 56 | } else if strings.HasSuffix(name, "."+string(format)) { 57 | filename = name 58 | } else { 59 | filename = name + "." + string(format) 60 | } 61 | 62 | // Escape the file name for avoiding the illegal characters. 63 | // Ref: https://en.wikipedia.org/wiki/Filename#Reserved_characters_and_words 64 | filename = escape(filename) 65 | 66 | // Create the download path. 67 | downloadPath := c.downloadPath 68 | if subPath != "" { 69 | downloadPath = filepath.Join(downloadPath, subPath) 70 | err := os.MkdirAll(downloadPath, 0o755) 71 | if err != nil { 72 | return nil, err 73 | } 74 | } 75 | 76 | // Generate the file path. 77 | path := filepath.Join(downloadPath, filename) 78 | 79 | // Remove the exist file. 80 | if _, err := os.Stat(path); err == nil { 81 | if err := os.Remove(path); err != nil { 82 | return nil, err 83 | } 84 | } 85 | 86 | // Add download progress, no need to close. 87 | bar := log.NewProgressBar(id, total, name, size) 88 | 89 | // Create file io. and remember to close it manually. 90 | file, err := os.Create(path) 91 | if err != nil { 92 | return nil, err 93 | } 94 | 95 | return &writer{ 96 | file: file, 97 | name: filename, 98 | download: downloadPath, 99 | extract: c.extract && format.Archive(), 100 | formats: c.formats, 101 | bar: bar, 102 | }, nil 103 | } 104 | 105 | type Writer interface { 106 | io.Writer 107 | io.Closer 108 | SetSize(int64) 109 | } 110 | 111 | type writer struct { 112 | file *os.File 113 | name string 114 | download string 115 | formats map[Format]bool 116 | extract bool 117 | bar *progressbar.ProgressBar 118 | } 119 | 120 | func (p *writer) Close() error { 121 | _ = p.bar.Close() 122 | err := p.file.Close() 123 | if err != nil { 124 | return err 125 | } 126 | 127 | // Extract the file if user enabled this. 128 | if p.extract { 129 | if err := p.decompress(); err != nil { 130 | log.Fatal(err) 131 | return nil 132 | } 133 | 134 | // Remove the compress files. 135 | _ = os.Remove(p.filePath()) 136 | } 137 | 138 | return err 139 | } 140 | 141 | func (p *writer) filePath() string { 142 | return filepath.Join(p.download, p.name) 143 | } 144 | 145 | func (p *writer) Write(b []byte) (n int, err error) { 146 | _, _ = p.bar.Write(b) 147 | return p.file.Write(b) 148 | } 149 | 150 | func (p *writer) SetSize(i int64) { 151 | p.bar.ChangeMax64(i) 152 | } 153 | -------------------------------------------------------------------------------- /internal/log/console.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "github.com/k0kubun/go-ansi" 9 | "github.com/mitchellh/colorstring" 10 | ) 11 | 12 | var ( 13 | EnableDebug = false // EnableDebug will enable the disabled debug log level. 14 | 15 | ansiStdout = ansi.NewAnsiStdout() 16 | 17 | debug = colorstring.Color("[dark_gray][DEBUG][reset]") 18 | info = colorstring.Color("[green][INFO] [reset]") 19 | warn = colorstring.Color("[yellow][WARN] [reset]") 20 | fatal = colorstring.Color("[red][FATAL][reset]") 21 | ) 22 | 23 | // Debugf would print the log with in debug level. The debug was disabled by default. 24 | // You should use EnableDebug to enable it. 25 | func Debugf(format string, v ...any) { 26 | if EnableDebug { 27 | printLog(debug, fmt.Sprintf(format, v...)) 28 | } 29 | } 30 | 31 | // Debug would print the log with in debug level. The debug was disabled by default. 32 | // // You should use EnableDebug to enable it. 33 | func Debug(v ...any) { 34 | if EnableDebug { 35 | printLog(debug, v...) 36 | } 37 | } 38 | 39 | // Infof would print the log with info level. 40 | func Infof(format string, v ...any) { 41 | printLog(info, fmt.Sprintf(format, v...)) 42 | } 43 | 44 | // Info would print the log with info level. 45 | func Info(v ...any) { 46 | printLog(info, v...) 47 | } 48 | 49 | // Warnf would print the log with warn level. 50 | func Warnf(format string, v ...any) { 51 | printLog(warn, fmt.Sprintf(format, v...)) 52 | } 53 | 54 | // Warn would print the log with warn level. 55 | func Warn(v ...any) { 56 | printLog(warn, v...) 57 | } 58 | 59 | // Fatalf would print the log with fatal level. And exit the program. 60 | func Fatalf(format string, v ...any) { 61 | printLog(fatal, fmt.Sprintf(format, v...)) 62 | } 63 | 64 | // Fatal would print the log with fatal level. And exit the program. 65 | func Fatal(v ...any) { 66 | printLog(fatal, v...) 67 | } 68 | 69 | // Exit will print the error with fatal level and os.Exit if the error isn't nil. 70 | func Exit(err error) { 71 | if err != nil { 72 | Fatal(err.Error()) 73 | os.Exit(-1) 74 | } 75 | } 76 | 77 | // printLog would print a colorful log level and log time. 78 | func printLog(level string, args ...any) { 79 | _, _ = fmt.Fprintln(ansiStdout, logTime(), level, fmt.Sprint(args...)) 80 | } 81 | 82 | // logTime will print the current time 83 | func logTime() string { 84 | return time.Now().Format("06/01/02 15:04:05") 85 | } 86 | -------------------------------------------------------------------------------- /internal/log/printer.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "os" 5 | "reflect" 6 | 7 | "github.com/jedib0t/go-pretty/v6/table" 8 | ) 9 | 10 | var DefaultHead = table.Row{"Config Key", "Config Value"} 11 | 12 | type Printer interface { 13 | Title(title string) Printer // Title adds the table title. 14 | Head(heads ...any) Printer // Head adds the table head. 15 | Row(fields ...any) Printer // Row add a row to the table. 16 | AllowZeroValue() Printer // AllowZeroValue The row will be printed if it contains zero value. 17 | Print() // Print would print a table-like message from the given config. 18 | } 19 | 20 | type tablePrinter struct { 21 | title string 22 | heads []any 23 | rows [][]any 24 | allowZero bool 25 | } 26 | 27 | func (t *tablePrinter) Title(title string) Printer { 28 | t.title = title 29 | return t 30 | } 31 | 32 | func (t *tablePrinter) Head(heads ...any) Printer { 33 | t.heads = heads 34 | return t 35 | } 36 | 37 | func (t *tablePrinter) Row(fields ...any) Printer { 38 | if len(fields) > 0 { 39 | t.rows = append(t.rows, fields) 40 | } 41 | return t 42 | } 43 | 44 | func (t *tablePrinter) AllowZeroValue() Printer { 45 | t.allowZero = true 46 | return t 47 | } 48 | 49 | func (t *tablePrinter) Print() { 50 | w := table.NewWriter() 51 | w.SetOutputMirror(os.Stdout) 52 | w.SetTitle(t.title) 53 | if len(t.heads) > 0 { 54 | w.AppendHeader(t.heads) 55 | } 56 | 57 | for _, row := range t.rows { 58 | appendRow(w, row, t.allowZero) 59 | } 60 | 61 | w.Render() 62 | } 63 | 64 | func appendRow(writer table.Writer, row []any, allowZero bool) { 65 | if len(row) == 1 { 66 | writer.AppendRow(row) 67 | } else { 68 | zero := true 69 | for _, r := range row[1:] { 70 | v := reflect.ValueOf(r) 71 | if !v.IsZero() { 72 | zero = false 73 | break 74 | } 75 | } 76 | if !zero || allowZero { 77 | writer.AppendRow(row) 78 | } 79 | } 80 | } 81 | 82 | // NewPrinter will return a printer for table-like logs. 83 | func NewPrinter() Printer { 84 | return &tablePrinter{} 85 | } 86 | -------------------------------------------------------------------------------- /internal/log/progress.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "fmt" 5 | "unicode/utf8" 6 | 7 | "github.com/k0kubun/go-ansi" 8 | "github.com/schollz/progressbar/v3" 9 | ) 10 | 11 | // NewProgressBar is used to print beautiful download progress. 12 | func NewProgressBar(index, total int64, filename string, bytes int64) *progressbar.ProgressBar { 13 | // Trim the filename size for better printing. 14 | if utf8.RuneCountInString(filename) > 30 { 15 | filename = string([]rune(filename)[:30]) + "..." 16 | } 17 | 18 | return progressbar.NewOptions64(bytes, 19 | progressbar.OptionSetWriter(ansi.NewAnsiStdout()), 20 | progressbar.OptionEnableColorCodes(true), 21 | progressbar.OptionShowBytes(true), 22 | progressbar.OptionSetWidth(15), 23 | progressbar.OptionOnCompletion(func() { 24 | fmt.Printf("\n") 25 | }), 26 | progressbar.OptionSetDescription(fmt.Sprintf("%s %s [%d/%d] %s", logTime(), info, index, total, filename)), 27 | progressbar.OptionSetTheme(progressbar.Theme{ 28 | Saucer: "[green]=[reset]", 29 | SaucerHead: "[green]>[reset]", 30 | SaucerPadding: " ", 31 | BarStart: "[", 32 | BarEnd: "]", 33 | }), 34 | ) 35 | } 36 | -------------------------------------------------------------------------------- /internal/progress/progress.go: -------------------------------------------------------------------------------- 1 | package progress 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "sync" 8 | "time" 9 | 10 | "github.com/bits-and-blooms/bitset" 11 | "go.uber.org/ratelimit" 12 | ) 13 | 14 | const NoBookToDownload = -1 15 | 16 | var ( 17 | ErrStartBookID = errors.New("the start book id should start from 1") 18 | ErrStartAndEndBookID = errors.New("start book id should below the available book id") 19 | ErrStorageFile = errors.New("couldn't create file for storing download process") 20 | ) 21 | 22 | type Progress interface { 23 | // TakeRateLimit would wait until the rate limit is available. 24 | TakeRateLimit() 25 | 26 | // AcquireBookID would find the book id from the assign array. 27 | AcquireBookID() int64 28 | 29 | // SaveBookID would save the download progress. 30 | SaveBookID(bookID int64) error 31 | 32 | // Finished would tell the called whether all the books have downloaded. 33 | Finished() bool 34 | 35 | // Size would return the book size. 36 | Size() int64 37 | } 38 | 39 | // bitProgress is a bit-based implementation with file persistence. 40 | type bitProgress struct { 41 | limit ratelimit.Limiter // The ratelimit for acquiring a book ID. 42 | progress *bitset.BitSet // progress is used for file Progress. 43 | assigned *bitset.BitSet // the assign status, memory based. 44 | lock *sync.Mutex // lock is used for concurrent request. 45 | file *os.File // The Progress file path for download progress. 46 | } 47 | 48 | // NewProgress Create a storage for save the download progress. 49 | func NewProgress(start, size int64, rate int, path string) (Progress, error) { 50 | if start < 1 { 51 | return nil, ErrStartBookID 52 | } 53 | if start > size { 54 | return nil, ErrStartAndEndBookID 55 | } 56 | 57 | var progress *bitset.BitSet 58 | var file *os.File 59 | 60 | startIndex := func(set *bitset.BitSet) { 61 | for i := uint(0); i < uint(start-1); i++ { 62 | set.Set(i) 63 | } 64 | } 65 | 66 | if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { 67 | // Create Progress related file 68 | if file, err = os.Create(path); err != nil { 69 | return nil, ErrStorageFile 70 | } else { 71 | // Enrich file content. 72 | progress = bitset.New(uint(size)) 73 | startIndex(progress) 74 | 75 | if err := saveStorage(file, progress); err != nil { 76 | return nil, err 77 | } 78 | } 79 | } else { 80 | // Load Progress from file. 81 | if file, err = os.OpenFile(path, os.O_RDWR, 0o644); err != nil { 82 | return nil, err 83 | } 84 | if progress, err = loadStorage(file); err != nil { 85 | return nil, err 86 | } 87 | 88 | // Recalculate start index. 89 | startIndex(progress) 90 | 91 | // Support book update, increase the progress size. 92 | if progress.Len() < uint(size) { 93 | p := bitset.New(uint(size)) 94 | progress.Copy(p) 95 | progress = p 96 | } 97 | } 98 | 99 | assigned := bitset.New(progress.Len()) 100 | progress.Copy(assigned) 101 | 102 | // Create ratelimit 103 | limit := ratelimit.New(rate, ratelimit.Per(time.Minute)) 104 | 105 | return &bitProgress{ 106 | limit: limit, 107 | progress: progress, 108 | assigned: assigned, 109 | lock: new(sync.Mutex), 110 | file: file, 111 | }, nil 112 | } 113 | 114 | func saveStorage(file *os.File, progress *bitset.BitSet) error { 115 | bytes, err := progress.MarshalBinary() 116 | if err != nil { 117 | return err 118 | } 119 | 120 | _, err = file.WriteAt(bytes, 0) 121 | return err 122 | } 123 | 124 | func loadStorage(file *os.File) (*bitset.BitSet, error) { 125 | set := new(bitset.BitSet) 126 | if _, err := set.ReadFrom(file); err != nil { 127 | return nil, err 128 | } 129 | 130 | return set, nil 131 | } 132 | 133 | // TakeRateLimit block until the rate meets the given config. 134 | func (storage *bitProgress) TakeRateLimit() { 135 | storage.limit.Take() 136 | } 137 | 138 | // AcquireBookID would find the book id from the assign array. 139 | func (storage *bitProgress) AcquireBookID() int64 { 140 | storage.lock.Lock() 141 | defer storage.lock.Unlock() 142 | 143 | for i := uint(0); i < storage.assigned.Len(); i++ { 144 | if !storage.assigned.Test(i) { 145 | storage.assigned.Set(i) 146 | return int64(i + 1) 147 | } 148 | } 149 | 150 | return NoBookToDownload 151 | } 152 | 153 | // SaveBookID would save the download progress. 154 | func (storage *bitProgress) SaveBookID(bookID int64) error { 155 | storage.lock.Lock() 156 | defer storage.lock.Unlock() 157 | 158 | if bookID > int64(storage.progress.Len()) { 159 | return fmt.Errorf("invalid book id: %d", bookID) 160 | } 161 | 162 | i := uint(bookID - 1) 163 | storage.assigned.Set(i) 164 | storage.progress.Set(i) 165 | 166 | _ = saveStorage(storage.file, storage.progress) 167 | 168 | return nil 169 | } 170 | 171 | // Finished would tell the called whether all the books have downloaded. 172 | func (storage *bitProgress) Finished() bool { 173 | return storage.progress.Count() == storage.progress.Len() 174 | } 175 | 176 | // Size would return the book size. 177 | func (storage *bitProgress) Size() int64 { 178 | return int64(storage.progress.Len()) 179 | } 180 | -------------------------------------------------------------------------------- /internal/progress/progress_test.go: -------------------------------------------------------------------------------- 1 | package progress 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | "strconv" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func tempFile() string { 13 | tempDir := os.TempDir() 14 | return path.Join(tempDir, "acquire-book-id_"+strconv.FormatInt(time.Now().Unix(), 10)) 15 | } 16 | 17 | func TestProgress_AcquireBookID(t *testing.T) { 18 | file := tempFile() 19 | defer func() { _ = os.Remove(file) }() 20 | 21 | s, err := NewProgress(1, 10000, 1000000, file) 22 | if err != nil { 23 | t.Errorf("Error in creating Progress: %v", err) 24 | } 25 | 26 | now := time.Now().UnixMilli() 27 | for i := 0; i < 1000; i++ { 28 | bookID := s.AcquireBookID() 29 | if bookID != int64(i+1) { 30 | t.Errorf("The book id doesn't match the desired id.") 31 | } 32 | } 33 | fmt.Println("Total time for acquiring 1000 book IDs: ", time.Now().UnixMilli()-now, "ms") 34 | } 35 | 36 | func TestProgress_SaveBookID(t *testing.T) { 37 | file := tempFile() 38 | defer func() { _ = os.Remove(file) }() 39 | 40 | s, err := NewProgress(1, 1000, 1000000, file) 41 | if err != nil { 42 | t.Errorf("Error in creating Progress: %v", err) 43 | } 44 | 45 | now := time.Now().UnixMilli() 46 | for i := 0; i < 500; i++ { 47 | bookID := s.AcquireBookID() 48 | if bookID != int64(i+1) { 49 | t.Errorf("The book id doesn't match the desired id.") 50 | } 51 | 52 | err = s.SaveBookID(bookID) 53 | if err != nil { 54 | t.Errorf("Error in saving download book id: %v", err) 55 | } 56 | } 57 | fmt.Println("Total time for saving 500 book IDs: ", time.Now().UnixMilli()-now, "ms") 58 | 59 | s2, err := NewProgress(1, 1000, 10000, file) 60 | if err != nil { 61 | t.Errorf("Error in creating Progress: %v", err) 62 | } 63 | 64 | bookID := s2.AcquireBookID() 65 | if bookID != 501 { 66 | t.Errorf("Error in acquire book id from Progress file. Book id should be %d, but it's %d", 501, bookID) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /internal/sobooks/metadata.go: -------------------------------------------------------------------------------- 1 | package sobooks 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/PuerkitoBio/goquery" 9 | 10 | "github.com/bookstairs/bookhunter/internal/driver" 11 | "github.com/bookstairs/bookhunter/internal/log" 12 | ) 13 | 14 | type LinkType string 15 | 16 | var ( 17 | // driveNamings is the chinese name mapping of the drive's provider. 18 | driveNamings = map[driver.Source]string{ 19 | driver.ALIYUN: "阿里", 20 | driver.CTFILE: "城通", 21 | driver.BAIDU: "百度", 22 | driver.QUARK: "夸克", 23 | driver.LANZOU: "蓝奏", 24 | driver.TELECOM: "天翼", 25 | driver.DIRECT: "备份", 26 | } 27 | dateRe = regexp.MustCompile(`(?m)(\d{4})-(\d{2})-(\d{2})`) 28 | linkRe = regexp.MustCompile(`(?m)https://sobooks\.cc/go\.html\?url=(.*?)"(.*?[：:]\s?(\w+))?`) 29 | ) 30 | 31 | type BookLink struct { 32 | URL string // The url to access the download page. 33 | Code string // The passcode for querying the file content. 34 | } 35 | 36 | // ParseLinks will find all the available link in the different driver. 37 | func ParseLinks(content string, id int64) (title string, links map[driver.Source]BookLink, err error) { 38 | // Find all the links. 39 | links = map[driver.Source]BookLink{} 40 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(content)) 41 | if err != nil { 42 | return "", links, err 43 | } 44 | 45 | dateText := doc.Find("div.bookinfo > ul > li:nth-child(5)").Text() 46 | 47 | if !dateRe.MatchString(dateText) { 48 | log.Fatal("not found book date ", id) 49 | return "", map[driver.Source]BookLink{}, fmt.Errorf("not found book date %v", id) 50 | } 51 | submatch := dateRe.FindStringSubmatch(dateText) 52 | year := submatch[1] 53 | month := submatch[2] 54 | titleDom := doc.Find(".article-title>a") 55 | title = titleDom.Text() 56 | 57 | // default link 58 | links[driver.DIRECT] = BookLink{URL: fmt.Sprintf("https://sobooks.cloud/%s/%s/%d.epub", year, month, id)} 59 | 60 | html, err := doc.Find(".e-secret").Html() 61 | if err != nil { 62 | return "", links, err 63 | } 64 | split := strings.Split(html, "
") 65 | 66 | for _, s := range split { 67 | for linkType, name := range driveNamings { 68 | if strings.Contains(s, name) { 69 | match := linkRe.FindStringSubmatch(s) 70 | if len(match) > 2 { 71 | links[linkType] = BookLink{ 72 | URL: match[1], 73 | Code: match[3], 74 | } 75 | } 76 | break 77 | } 78 | } 79 | } 80 | return title, links, nil 81 | } 82 | -------------------------------------------------------------------------------- /internal/sobooks/metadata_test.go: -------------------------------------------------------------------------------- 1 | package sobooks 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strconv" 7 | "testing" 8 | 9 | "github.com/go-resty/resty/v2" 10 | ) 11 | 12 | func TestParseSobooksUrl(t *testing.T) { 13 | client := resty.New().SetBaseURL("https://sobooks.cc") 14 | 15 | // unsupported 14320 11000 16 | // 20081 16899 16240 17 | id := int64(18021) 18 | resp, err := client.R(). 19 | SetCookie(&http.Cookie{ 20 | Name: "mpcode", 21 | Value: "244152", 22 | Path: "/", 23 | Domain: "sobooks.cc", 24 | }). 25 | SetPathParam("bookId", strconv.FormatInt(id, 10)). 26 | SetHeader("referer", client.BaseURL). 27 | Get("/books/{bookId}.html") 28 | if err != nil { 29 | t.Error(err) 30 | } 31 | _, links, _ := ParseLinks(resp.String(), id) 32 | fmt.Printf("%v", links) 33 | } 34 | -------------------------------------------------------------------------------- /internal/talebook/response.go: -------------------------------------------------------------------------------- 1 | package talebook 2 | 3 | const ( 4 | SuccessStatus = "ok" 5 | BookNotFoundStatus = "not_found" 6 | ) 7 | 8 | // CommonResp is the base response for all the requests. 9 | type CommonResp struct { 10 | Err string `json:"err"` 11 | } 12 | 13 | // LoginResp is used for login action. 14 | type LoginResp struct { 15 | CommonResp 16 | Msg string `json:"msg"` 17 | } 18 | 19 | // BookResp stands for default book information 20 | type BookResp struct { 21 | CommonResp 22 | Msg string `json:"msg"` 23 | KindleSender string `json:"kindle_sender"` 24 | Book struct { 25 | ID int `json:"id"` 26 | Title string `json:"title"` 27 | Files []struct { 28 | Format string `json:"format"` 29 | Size int64 `json:"size"` 30 | Href string `json:"href"` 31 | } `json:"files"` 32 | } `json:"book"` 33 | } 34 | 35 | // BooksResp is used to return recent books. 36 | type BooksResp struct { 37 | CommonResp 38 | Msg string `json:"msg"` 39 | Title string `json:"title"` 40 | Total int64 `json:"total"` 41 | Books []struct { 42 | ID int64 `json:"id"` 43 | } `json:"books"` 44 | } 45 | -------------------------------------------------------------------------------- /internal/telegram/auth.go: -------------------------------------------------------------------------------- 1 | package telegram 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "os" 9 | "strings" 10 | 11 | "github.com/gotd/td/telegram/auth" 12 | "github.com/gotd/td/tg" 13 | "golang.org/x/term" 14 | ) 15 | 16 | // Authentication is used for log into the telegram with a session support. 17 | // Every telegram execution will require this method. 18 | func (t *Telegram) Authentication() error { 19 | // Setting up authentication flow helper based on terminal auth. 20 | flow := auth.NewFlow(&terminalAuth{mobile: t.mobile}, auth.SendCodeOptions{}) 21 | if err := t.client.Auth().IfNecessary(t.ctx, flow); err != nil { 22 | return err 23 | } 24 | 25 | status, _ := t.client.Auth().Status(t.ctx) 26 | if !status.Authorized { 27 | return errors.New("failed to login, please check you login info or refresh the session by --refresh") 28 | } 29 | 30 | return nil 31 | } 32 | 33 | // terminalAuth implements authentication via terminal. 34 | type terminalAuth struct { 35 | mobile string 36 | } 37 | 38 | func (t *terminalAuth) Phone(_ context.Context) (string, error) { 39 | // Make the mobile number has the country code as the prefix. 40 | addCountryCode := func(mobile string) string { 41 | if strings.HasPrefix(mobile, "+") { 42 | return mobile 43 | } else if strings.HasPrefix(mobile, "86") { 44 | return "+" + mobile 45 | } else if mobile != "" { 46 | return "+86" + mobile 47 | } else { 48 | return "" 49 | } 50 | } 51 | 52 | if t.mobile == "" { 53 | fmt.Print("Enter Phone Number (+86): ") 54 | phone, err := bufio.NewReader(os.Stdin).ReadString('\n') 55 | if err != nil { 56 | return "", err 57 | } 58 | t.mobile = addCountryCode(phone) 59 | } 60 | 61 | return t.mobile, nil 62 | } 63 | 64 | func (t *terminalAuth) Password(_ context.Context) (string, error) { 65 | fmt.Print("Enter 2FA password: ") 66 | bytePwd, err := term.ReadPassword(0) 67 | if err != nil { 68 | return "", err 69 | } 70 | return strings.TrimSpace(string(bytePwd)), nil 71 | } 72 | 73 | func (t *terminalAuth) AcceptTermsOfService(_ context.Context, tos tg.HelpTermsOfService) error { 74 | return &auth.SignUpRequired{TermsOfService: tos} 75 | } 76 | 77 | func (t *terminalAuth) SignUp(_ context.Context) (auth.UserInfo, error) { 78 | return auth.UserInfo{}, errors.New("signup call is not expected") 79 | } 80 | 81 | func (t *terminalAuth) Code(_ context.Context, _ *tg.AuthSentCode) (string, error) { 82 | fmt.Print("Enter code: ") 83 | code, err := bufio.NewReader(os.Stdin).ReadString('\n') 84 | if err != nil { 85 | return "", err 86 | } 87 | return strings.TrimSpace(code), nil 88 | } 89 | -------------------------------------------------------------------------------- /internal/telegram/channel.go: -------------------------------------------------------------------------------- 1 | package telegram 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/gotd/td/tg" 9 | 10 | "github.com/bookstairs/bookhunter/internal/log" 11 | ) 12 | 13 | func (t *Telegram) ChannelInfo() (*ChannelInfo, error) { 14 | var channelID int64 15 | var accessHash int64 16 | var err error 17 | 18 | // Get the real channelID and accessHash. 19 | if strings.HasPrefix(t.channelID, "joinchat/") { 20 | channelID, accessHash, err = t.privateChannelInfo(strings.TrimPrefix(t.channelID, "joinchat/")) 21 | } else { 22 | channelID, accessHash, err = t.publicChannelInfo(t.channelID) 23 | } 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | // Query the last message ID. 29 | lastMsgID, err := t.queryLastMsgID(channelID, accessHash) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | return &ChannelInfo{ 35 | ID: channelID, 36 | AccessHash: accessHash, 37 | LastMsgID: lastMsgID, 38 | }, nil 39 | } 40 | 41 | // privateChannelInfo queries access hash for the private channel. 42 | func (t *Telegram) privateChannelInfo(hash string) (id int64, access int64, err error) { 43 | invite, err := t.client.API().MessagesCheckChatInvite(t.ctx, hash) 44 | if err != nil { 45 | return 46 | } 47 | 48 | switch v := invite.(type) { 49 | case *tg.ChatInviteAlready: 50 | if channel, ok := v.GetChat().(*tg.Channel); ok { 51 | id = channel.ID 52 | access = channel.AccessHash 53 | return 54 | } 55 | case *tg.ChatInvitePeek: 56 | if channel, ok := v.GetChat().(*tg.Channel); ok { 57 | id = channel.ID 58 | access = channel.AccessHash 59 | return 60 | } 61 | case *tg.ChatInvite: 62 | log.Warn("You haven't join this private channel, plz join it manually.") 63 | } 64 | 65 | err = errors.New("couldn't find access hash") 66 | return 67 | } 68 | 69 | // publicChannelInfo queries the public channel by its name. 70 | func (t *Telegram) publicChannelInfo(name string) (id, access int64, err error) { 71 | username, err := t.client.API().ContactsResolveUsername(t.ctx, &tg.ContactsResolveUsernameRequest{Username: name}) 72 | if err != nil { 73 | return 74 | } 75 | 76 | if len(username.Chats) == 0 { 77 | err = fmt.Errorf("you are not belong to channel: %s", name) 78 | return 79 | } 80 | 81 | for _, chat := range username.Chats { 82 | // Try to find the related channel. 83 | if channel, ok := chat.(*tg.Channel); ok { 84 | id = channel.ID 85 | access = channel.AccessHash 86 | return 87 | } 88 | } 89 | 90 | err = fmt.Errorf("couldn't find channel id and hash for channel: %s", name) 91 | return 92 | } 93 | 94 | // queryLastMsgID from the given channel info. 95 | func (t *Telegram) queryLastMsgID(channelID, access int64) (int64, error) { 96 | request := &tg.MessagesSearchRequest{ 97 | Peer: &tg.InputPeerChannel{ 98 | ChannelID: channelID, 99 | AccessHash: access, 100 | }, 101 | Filter: &tg.InputMessagesFilterEmpty{}, 102 | Q: "", 103 | OffsetID: -1, 104 | Limit: 1, 105 | } 106 | 107 | last := -1 108 | search, err := t.client.API().MessagesSearch(t.ctx, request) 109 | if err != nil { 110 | return 0, err 111 | } 112 | 113 | channelInfo, ok := search.(*tg.MessagesChannelMessages) 114 | if !ok { 115 | return 0, err 116 | } 117 | 118 | for _, msg := range channelInfo.Messages { 119 | if msg != nil { 120 | last = msg.GetID() 121 | break 122 | } 123 | } 124 | 125 | if last <= 0 { 126 | return 0, errors.New("couldn't find last message id") 127 | } 128 | 129 | return int64(last), nil 130 | } 131 | -------------------------------------------------------------------------------- /internal/telegram/common.go: -------------------------------------------------------------------------------- 1 | package telegram 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/gotd/contrib/bg" 7 | "github.com/gotd/contrib/middleware/floodwait" 8 | "github.com/gotd/td/session" 9 | "github.com/gotd/td/telegram" 10 | "github.com/gotd/td/telegram/dcs" 11 | "github.com/gotd/td/tg" 12 | 13 | "github.com/bookstairs/bookhunter/internal/file" 14 | ) 15 | 16 | type ( 17 | Telegram struct { 18 | channelID string 19 | mobile string 20 | appID int64 21 | appHash string 22 | client *telegram.Client 23 | ctx context.Context 24 | } 25 | 26 | ChannelInfo struct { 27 | ID int64 28 | AccessHash int64 29 | LastMsgID int64 30 | } 31 | 32 | // File is the file info from the telegram channel. 33 | File struct { 34 | ID int64 35 | Name string 36 | Format file.Format 37 | Size int64 38 | Document *tg.InputDocumentFileLocation 39 | } 40 | ) 41 | 42 | // New will create a telegram client. 43 | func New(channelID, mobile string, appID int64, appHash string, sessionPath, proxy string) (*Telegram, error) { 44 | // Create the http proxy dial. 45 | dialFunc, err := createProxy(proxy) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | // Create the backend telegram client. 51 | client := telegram.NewClient( 52 | int(appID), 53 | appHash, 54 | telegram.Options{ 55 | Resolver: dcs.Plain(dcs.PlainOptions{Dial: dialFunc}), 56 | SessionStorage: &session.FileStorage{Path: sessionPath}, 57 | Middlewares: []telegram.Middleware{ 58 | floodwait.NewSimpleWaiter().WithMaxRetries(uint(3)), 59 | }, 60 | }, 61 | ) 62 | 63 | ctx := context.Background() 64 | _, err = bg.Connect(client, bg.WithContext(ctx)) // No need to close this client. 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | t := &Telegram{ 70 | ctx: ctx, 71 | channelID: channelID, 72 | mobile: mobile, 73 | appID: appID, 74 | appHash: appHash, 75 | client: client, 76 | } 77 | 78 | if err := t.Authentication(); err != nil { 79 | return nil, err 80 | } 81 | 82 | return t, nil 83 | } 84 | -------------------------------------------------------------------------------- /internal/telegram/download.go: -------------------------------------------------------------------------------- 1 | package telegram 2 | 3 | import ( 4 | "io" 5 | "math" 6 | 7 | "github.com/gotd/td/telegram/downloader" 8 | "github.com/gotd/td/tg" 9 | 10 | "github.com/bookstairs/bookhunter/internal/file" 11 | ) 12 | 13 | func (t *Telegram) DownloadFile(f *File, writer io.Writer) error { 14 | tool := downloader.NewDownloader() 15 | thread := int(math.Ceil(float64(f.Size) / (512 * 1024))) 16 | _, err := tool.Download(t.client.API(), f.Document).WithThreads(thread).Stream(t.ctx, writer) 17 | 18 | return err 19 | } 20 | 21 | // ParseMessage will parse the given message id. 22 | func (t *Telegram) ParseMessage(info *ChannelInfo, msgID int64) ([]File, error) { 23 | var files []File 24 | // This API is translated from official C++ client. 25 | api := t.client.API() 26 | history, err := api.MessagesSearch(t.ctx, &tg.MessagesSearchRequest{ 27 | Peer: &tg.InputPeerChannel{ 28 | ChannelID: info.ID, 29 | AccessHash: info.AccessHash, 30 | }, 31 | Filter: &tg.InputMessagesFilterEmpty{}, 32 | Q: "", 33 | OffsetID: int(msgID), 34 | Limit: 1, 35 | AddOffset: -1, 36 | }) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | messages := history.(*tg.MessagesChannelMessages) 42 | for i := len(messages.Messages) - 1; i >= 0; i-- { 43 | message := messages.Messages[i] 44 | if f, ok := parseFile(message); ok { 45 | files = append(files, *f) 46 | } 47 | } 48 | 49 | return files, nil 50 | } 51 | 52 | func parseFile(message tg.MessageClass) (*File, bool) { 53 | if message == nil { 54 | return nil, false 55 | } 56 | msg, ok := message.(*tg.Message) 57 | if !ok { 58 | return nil, false 59 | } 60 | if msg.Media == nil { 61 | return nil, false 62 | } 63 | s, ok := msg.Media.(*tg.MessageMediaDocument) 64 | if !ok { 65 | return nil, false 66 | } 67 | document := s.Document.(*tg.Document) 68 | fileName := "" 69 | for _, attribute := range document.Attributes { 70 | x, ok := attribute.(*tg.DocumentAttributeFilename) 71 | if ok { 72 | fileName = x.FileName 73 | } 74 | } 75 | if fileName == "" { 76 | return nil, false 77 | } 78 | format, _ := file.Extension(fileName) 79 | 80 | return &File{ 81 | ID: int64(msg.ID), 82 | Name: fileName, 83 | Format: format, 84 | Size: document.Size, 85 | Document: document.AsInputDocumentFileLocation(), 86 | }, true 87 | } 88 | -------------------------------------------------------------------------------- /internal/telegram/proxy.go: -------------------------------------------------------------------------------- 1 | package telegram 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "net" 8 | "net/http" 9 | "net/url" 10 | 11 | "github.com/gotd/td/telegram/dcs" 12 | "golang.org/x/net/proxy" 13 | 14 | "github.com/bookstairs/bookhunter/internal/log" 15 | ) 16 | 17 | // Register Dialer Type for HTTP & HTTPS Proxy in golang. 18 | func init() { 19 | proxy.RegisterDialerType("http", newHTTPProxy) 20 | proxy.RegisterDialerType("https", newHTTPProxy) 21 | } 22 | 23 | // This file is used to manually create a proxy with the arguments and system environment. 24 | 25 | // createProxy is used to create a dcs.DialFunc for the telegram to send request. 26 | // We don't support MTProxy now. 27 | func createProxy(proxyURL string) (dcs.DialFunc, error) { 28 | if proxyURL != "" { 29 | log.Debugf("Try to manually create the proxy through %s", proxyURL) 30 | 31 | u, err := url.Parse(proxyURL) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | dialer, err := proxy.FromURL(u, Direct) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | return func(ctx context.Context, network, addr string) (net.Conn, error) { 42 | return dialContext(ctx, dialer, network, addr) 43 | }, nil 44 | } 45 | 46 | // Fallback to default proxy with environment support. 47 | dialer := proxy.FromEnvironment() 48 | return func(ctx context.Context, network, addr string) (net.Conn, error) { 49 | return dialContext(ctx, dialer, network, addr) 50 | }, nil 51 | } 52 | 53 | // Copied from golang.org/x/net/proxy/dial.go 54 | func dialContext(ctx context.Context, d proxy.Dialer, network, address string) (net.Conn, error) { 55 | var ( 56 | conn net.Conn 57 | done = make(chan struct{}, 1) 58 | err error 59 | ) 60 | go func() { 61 | conn, err = d.Dial(network, address) 62 | close(done) 63 | if conn != nil && ctx.Err() != nil { 64 | _ = conn.Close() 65 | } 66 | }() 67 | select { 68 | case <-ctx.Done(): 69 | err = ctx.Err() 70 | case <-done: 71 | } 72 | return conn, err 73 | } 74 | 75 | type direct struct{} 76 | 77 | // Direct is a direct proxy: one that makes network connections directly. 78 | var Direct = direct{} 79 | 80 | func (direct) Dial(network, addr string) (net.Conn, error) { 81 | return net.Dial(network, addr) 82 | } 83 | 84 | // httpProxy is an HTTP / HTTPS connection proxy. 85 | type httpProxy struct { 86 | host string 87 | haveAuth bool 88 | username string 89 | password string 90 | forward proxy.Dialer 91 | } 92 | 93 | func newHTTPProxy(uri *url.URL, forward proxy.Dialer) (proxy.Dialer, error) { 94 | s := new(httpProxy) 95 | s.host = uri.Host 96 | s.forward = forward 97 | if uri.User != nil { 98 | s.haveAuth = true 99 | s.username = uri.User.Username() 100 | s.password, _ = uri.User.Password() 101 | } 102 | 103 | return s, nil 104 | } 105 | 106 | func (s *httpProxy) Dial(_, addr string) (net.Conn, error) { 107 | // Dial and create the https client connection. 108 | c, err := s.forward.Dial("tcp", s.host) 109 | if err != nil { 110 | return nil, err 111 | } 112 | 113 | // HACK. http.ReadRequest also does this. 114 | reqURL, err := url.Parse("http://" + addr) 115 | if err != nil { 116 | _ = c.Close() 117 | return nil, err 118 | } 119 | reqURL.Scheme = "" 120 | 121 | req, err := http.NewRequestWithContext(context.Background(), "CONNECT", reqURL.String(), http.NoBody) 122 | if err != nil { 123 | _ = c.Close() 124 | return nil, err 125 | } 126 | req.Close = false 127 | if s.haveAuth { 128 | req.SetBasicAuth(s.username, s.password) 129 | } 130 | 131 | err = req.Write(c) 132 | if err != nil { 133 | _ = c.Close() 134 | return nil, err 135 | } 136 | 137 | resp, err := http.ReadResponse(bufio.NewReader(c), req) 138 | if err != nil { 139 | _ = resp.Body.Close() 140 | _ = c.Close() 141 | return nil, err 142 | } 143 | _ = resp.Body.Close() 144 | if resp.StatusCode != 200 { 145 | _ = c.Close() 146 | err = fmt.Errorf("connect server using proxy error, StatusCode [%d]", resp.StatusCode) 147 | return nil, err 148 | } 149 | 150 | return c, nil 151 | } 152 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/bookstairs/bookhunter/cmd" 5 | ) 6 | 7 | // main the entrypoint for the downloader. 8 | func main() { 9 | cmd.Execute() 10 | } 11 | -------------------------------------------------------------------------------- /scripts/goimports.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## This shell script is used with https://github.com/TekWizely/pre-commit-golang 4 | # You should add it to your .pre-commit-config.yaml file with the options like 5 | # 6 | # - repo: https://github.com/tekwizely/pre-commit-golang 7 | # rev: v1.0.0-rc.1 8 | # hooks: 9 | # - id: my-cmd 10 | # name: goimports 11 | # alias: goimports 12 | # args: [ scripts/goimports.sh, github.com/syhily/hobbit ] 13 | 14 | module="$1" 15 | file="$2" 16 | 17 | # Detect the running OS. 18 | COMMAND="sed" 19 | if [[ $OSTYPE == 'darwin'* ]]; then 20 | # macOS have to use the gsed which can be installed by `brew install gsed`. 21 | COMMAND="gsed" 22 | fi 23 | 24 | # Detect the command. 25 | command -v $COMMAND >/dev/null 2>&1 || { echo >&2 "Require ${COMMAND} but it's not installed. Aborting."; exit 1; } 26 | command -v goimports >/dev/null 2>&1 || { echo >&2 "Require goimports but it's not installed. Aborting."; exit 1; } 27 | 28 | # Remove all the import spaces in staging golang files. 29 | REPLACEMENT=$(cat <<-END 30 | ' 31 | /^import (/,/)/ { 32 | /^$/ d 33 | } 34 | ' 35 | END 36 | ) 37 | bash -c "${COMMAND} -i ${REPLACEMENT} ${file}" 38 | 39 | # Format the staging golang files. 40 | goimports -l -d -local "${module}" -w "${file}" 41 | --------------------------------------------------------------------------------