├── .gitignore
├── LICENSE.md
├── README.md
├── downloader
    ├── downloader.go
    ├── progress.go
    ├── progress_test.go
    └── proxy.go
├── example
    └── example.go
├── go.mod
├── kemono
    ├── fetch.go
    ├── kemono.go
    ├── type.go
    └── utils.go
├── main
    ├── args.go
    ├── cookie
    │   ├── chromium
    │   │   ├── chrome.go
    │   │   ├── chrome_unix.go
    │   │   └── chrome_windows.go
    │   ├── cookie_test.go
    │   ├── cookies.go
    │   ├── firefox
    │   │   └── firefox.go
    │   └── utils
    │   │   ├── helper.go
    │   │   ├── linux
    │   │       └── env.go
    │   │   ├── map.go
    │   │   └── windows_dpapi.go
    ├── cookies_unix.go
    ├── cookies_windows.go
    ├── cookies_windows_no_cookies_detected.go
    ├── main.go
    └── path.go
├── term
    ├── README.md
    ├── terminal.go
    ├── terminal_posix.go
    ├── terminal_unix.go
    └── terminal_windows.go
└── utils
    ├── format.go
    └── helper.go


/.gitignore:
--------------------------------------------------------------------------------
 1 | # Binaries for programs and plugins
 2 | *.exe
 3 | *.exe~
 4 | *.dll
 5 | *.so
 6 | *.dylib
 7 | 
 8 | # Test binary, built with `go test -c`
 9 | *.test
10 | 
11 | # Output of the go coverage tool, specifically when used with LiteIDE
12 | *.out
13 | .download/
14 | 
15 | # IDE directories and files
16 | .idea/
17 | 


--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2023 elvis972602
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Kemono-scraper
  2 | 
  3 | A simple downloader to download images from kemono.su
  4 | 
  5 | ## Flag option
  6 | 
  7 | ### Cookie file
  8 | 
  9 | only needed if you want to download favorite creators or posts
 10 | 
 11 | `--cookie PATH` cookie file, default is cookies.txt (value separate by whitespace) syntax:
 12 | 
 13 | | Domain        | Include subdomains | Path | Secure | Expiry     | Name        | Value   |
 14 | |---------------|--------------------|------|--------|------------|-------------|---------|
 15 | | .kemono.su    | FALSE              | /    | TRUE   | 1706755572 | kemono_auth | <value> |
 16 | 
 17 | you can get cookies easily by using Chrome extension [Get cookies.txt LOCALLY](https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc)
 18 | 
 19 | ### Windows
 20 | 
 21 | Windows can detect the cookie file automatically (Not supported in no_cookies_detection version)
 22 | 
 23 | `--cookie-browser string` which browser to use, default is chrome (supported: chrome, firefox, edge , opera, vivaldi)
 24 | 
 25 | ### Download Options
 26 | 
 27 | `--link [<urls>]`: download link, separate by comma
 28 | 
 29 | `--creator [<service>:<id>]`: download creators, separate by comma
 30 | 
 31 | `--banner bool`: download banner, default is false (kemono only)
 32 | 
 33 | `--fav-site string`: specify the website to get favorites from (kemono or coomer), separated by comma
 34 | 
 35 | `--fav-creator bool`: download favorite creator, default is false
 36 | 
 37 | `--fav-post bool` download favorite post, default is false
 38 | 
 39 | ### Post Filter Options
 40 | 
 41 | `--first int`: download first n post
 42 | 
 43 | `--last int`: download last n post
 44 | 
 45 | `--date YYYYMMDD`: download post on date
 46 | 
 47 | `--date-before YYYYMMDD`: download post before date
 48 | 
 49 | `--date-after YYYYMMDD`: download post after date
 50 | 
 51 | `--update YYYYMMDD`: download post updated on date
 52 | 
 53 | `--update-before YYYYMMDD`: download post updated before date
 54 | 
 55 | `--update-after YYYYMMDD`: download post updated after date
 56 | 
 57 | ### Image Filter Options
 58 | 
 59 | `--extension-only [<ext>]`: download post with extension, separate by comma
 60 | 
 61 | `--extension-exclude [<ext>]`: download post without extension, separate by comma
 62 | 
 63 | `--max-size string`: download post with size less than max-size (e.g. 1 MB, 1KB, 1.5 gb, etc.)
 64 | 
 65 | `--min-size string`: download post with size greater than min-size (e.g. 1 MB, 1KB, 1.5 gb, etc.)
 66 | 
 67 | ## Downloader options
 68 | 
 69 | `--output PATH`: output path
 70 | 
 71 | `--template <tags>`: The template for customizing download paths, where you can use the following keywords to specify different parts of the path:
 72 | 
 73 | - `<ks:service>`: creator service
 74 | - `<ks:creator>`: creator name
 75 | - `<ks:post>`: post title
 76 | - `<ks:index>`: file index
 77 | - `<ks:filename>`: file name
 78 | - `<ks:filehash>`: file hash
 79 | - `<ks:extension>`: file extension
 80 | 
 81 | For example:
 82 | 
 83 | `[<ks:service>] <ks:creator>/<ks:post>/<ks:index>-<ks:filename><ks:extension>`
 84 | 
 85 | `--image-template <tags>` The template for customizing image file, `--template` should be set first.
 86 | 
 87 | `--video-template <tags>` The template for customizing video file, `--template` should be set first.
 88 | 
 89 | `--audio-template <tags>` The template for customizing audio file, `--template` should be set first.
 90 | 
 91 | `--archive-template <tags>` The template for customizing archive file, `--template` should be set first.
 92 | 
 93 | `--content bool`: download content, default is false
 94 | 
 95 | `--overwrite bool`: overwrite existing file
 96 | 
 97 | `--async bool`: download posts asynchronously, may cause the file order is not the same as the post order, can be used with --with-prefix-number, default false
 98 | 
 99 | `--max-download-parallel int`: max download file concurrent, default is 3, async mode only
100 | 
101 | `--with-prefix-number bool`: add prefix number to file name `<order>_<filename>`, default false
102 | 
103 | `--name-rule-only-index bool`: only use index as file name, default false
104 | 
105 | `--download-timeout int`: download timeout in seconds, default 1800
106 | 
107 | `--retry int`: retry times, default 3
108 | 
109 | `--retry-interval number`: retry interval in seconds, default 10. The number can be specified as either an int or float type
110 | 
111 | `--rate-limit int`: rate limit in request/s, default 2
112 | 
113 | `--proxy string`: proxy url, default is empty, support socks5, http, https (e.g. socks5://proxy:1080)
114 | 
115 | ## Config File
116 | 
117 | config file is in `./config.yaml`
118 | 
119 | Options in config file are the same as command-line flag options, but will be overridden by flags (if both exists).
120 | Usually used for setting the default settings for the scraper.
121 | 
122 | ```yaml
123 | banner: true
124 | async: true
125 | max-download-parallel: 5
126 | output: ./downloads
127 | template: "[<ks:service>] <ks:creator>/<ks:post>/<ks:filename><ks:extension>"
128 | image-template: "[<ks:service>] <ks:creator>/<ks:post>/<ks:index><ks:extension>"
129 | video-template: "[<ks:service>] <ks:creator>/<ks:post>/video/<ks:filename><ks:extension>"
130 | retry: 10
131 | retry-interval: 15
132 | # proxy: socks5://proxy:1080
133 | ```
134 | 
135 | ## Build from Source
136 | 
137 | Cloning the repository:
138 | 
139 | ```bash
140 | git clone https://github.com/elvis972602/Kemono-scraper
141 | cd Kemono-scraper/main
142 | ```
143 | 
144 | Download all the dependencies:
145 | 
146 | ```bash
147 | go mod tidy
148 | ```
149 | 
150 | Build the project:
151 | 
152 | ```bash
153 | go build
154 | ```
155 | 
156 | - No cookies detection:
157 | 
158 | ```bash
159 | go build -tags=no_cookies_detection
160 | ```
161 | 
162 | ## Features
163 | 
164 | With Kemono-scraper, you can implement a Downloader to take advantage of features such as multi-connection downloading, resume broken downloads, and more.
165 | 


--------------------------------------------------------------------------------
/downloader/downloader.go:
--------------------------------------------------------------------------------
  1 | package downloader
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"crypto/tls"
  6 | 	"errors"
  7 | 	"fmt"
  8 | 	"github.com/elvis972602/kemono-scraper/kemono"
  9 | 	"github.com/elvis972602/kemono-scraper/utils"
 10 | 	"html/template"
 11 | 	"io"
 12 | 	"net/http"
 13 | 	"os"
 14 | 	"path/filepath"
 15 | 	"strconv"
 16 | 	"sync"
 17 | 	"time"
 18 | )
 19 | 
 20 | const (
 21 | 	maxConcurrent           = 5
 22 | 	maxConnection           = 100
 23 | 	rateLimit               = 2
 24 | 	UserAgent               = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
 25 | 	Accept                  = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
 26 | 	AcceptEncoding          = "gzip, deflate, br"
 27 | 	AcceptLanguage          = "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7"
 28 | 	SecChUA                 = "\"Google Chrome\";v=\"111\", \"Not(A:Brand\";v=\"8\", \"Chromium\";v=\"111\""
 29 | 	SecChUAMobile           = "?0"
 30 | 	SecFetchDest            = "document"
 31 | 	SecFetchMode            = "navigate"
 32 | 	SecFetchSite            = "none"
 33 | 	SecFetchUser            = "?1"
 34 | 	UpgradeInsecureRequests = "1"
 35 | )
 36 | 
 37 | type Log interface {
 38 | 	Printf(format string, v ...interface{})
 39 | 	Print(s string)
 40 | 	SetStatus(s []string)
 41 | }
 42 | 
 43 | type Header map[string]string
 44 | 
 45 | type DownloadOption func(*downloader)
 46 | 
 47 | type downloader struct {
 48 | 	BaseURL string
 49 | 	// Max concurrent download
 50 | 	MaxConcurrent int
 51 | 
 52 | 	// Async download, download several files at the same time,
 53 | 	// may cause the file order is not the same as the post order
 54 | 	Async bool
 55 | 
 56 | 	OverWrite bool
 57 | 
 58 | 	maxSize int64
 59 | 
 60 | 	minSize int64
 61 | 
 62 | 	// SavePath return the path to save the file
 63 | 	SavePath func(creator kemono.Creator, post kemono.Post, i int, attachment kemono.File) string
 64 | 	// timeout
 65 | 	Timeout time.Duration
 66 | 
 67 | 	reteLimiter *utils.RateLimiter
 68 | 
 69 | 	Header Header
 70 | 
 71 | 	cookies []*http.Cookie
 72 | 
 73 | 	retry int
 74 | 
 75 | 	retryInterval time.Duration
 76 | 
 77 | 	content bool
 78 | 
 79 | 	progress *Progress
 80 | 
 81 | 	log Log
 82 | 
 83 | 	client *http.Client
 84 | }
 85 | 
 86 | func NewDownloader(options ...DownloadOption) kemono.Downloader {
 87 | 	// with default options
 88 | 	d := &downloader{
 89 | 		MaxConcurrent: maxConcurrent,
 90 | 		SavePath:      defaultSavePath,
 91 | 		Timeout:       300 * time.Second,
 92 | 		Async:         false,
 93 | 		OverWrite:     false,
 94 | 		maxSize:       1<<63 - 1,
 95 | 		minSize:       0,
 96 | 		reteLimiter:   utils.NewRateLimiter(rateLimit),
 97 | 		retry:         2,
 98 | 		client: &http.Client{
 99 | 			Transport: &http.Transport{
100 | 				TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
101 | 				MaxIdleConns:          maxConnection,
102 | 				MaxConnsPerHost:       maxConnection,
103 | 				MaxIdleConnsPerHost:   maxConnection,
104 | 				ResponseHeaderTimeout: 30 * time.Second,
105 | 			},
106 | 		},
107 | 	}
108 | 	for _, option := range options {
109 | 		option(d)
110 | 	}
111 | 	if d.BaseURL == "" {
112 | 		panic("base url is empty")
113 | 	}
114 | 	if !d.Async {
115 | 		d.MaxConcurrent = 1
116 | 	}
117 | 	if d.log == nil {
118 | 		panic("log is nil")
119 | 	}
120 | 
121 | 	d.progress = NewProgress(d.log)
122 | 	d.progress.Run(100 * time.Millisecond)
123 | 
124 | 	return d
125 | }
126 | 
127 | // BaseURL set the base url
128 | func BaseURL(baseURL string) DownloadOption {
129 | 	return func(d *downloader) {
130 | 		d.BaseURL = baseURL
131 | 	}
132 | }
133 | 
134 | // MaxConcurrent set the max concurrent download
135 | func MaxConcurrent(maxConcurrent int) DownloadOption {
136 | 	return func(d *downloader) {
137 | 		d.MaxConcurrent = maxConcurrent
138 | 	}
139 | }
140 | 
141 | // MaxSize set the max size of the file to download
142 | func MaxSize(maxSize int64) DownloadOption {
143 | 	return func(d *downloader) {
144 | 		d.maxSize = maxSize
145 | 	}
146 | }
147 | 
148 | // MinSize set the min size of the file to download
149 | func MinSize(minSize int64) DownloadOption {
150 | 	return func(d *downloader) {
151 | 		d.minSize = minSize
152 | 	}
153 | }
154 | 
155 | // Timeout set the timeout
156 | func Timeout(timeout time.Duration) DownloadOption {
157 | 	return func(d *downloader) {
158 | 		d.Timeout = timeout
159 | 	}
160 | }
161 | 
162 | // RateLimit limit the rate of download per second
163 | func RateLimit(n int) DownloadOption {
164 | 	return func(d *downloader) {
165 | 		d.reteLimiter = utils.NewRateLimiter(n)
166 | 	}
167 | }
168 | 
169 | func WithHeader(header Header) DownloadOption {
170 | 	return func(d *downloader) {
171 | 		d.Header = header
172 | 	}
173 | }
174 | 
175 | func WithCookie(cookies []*http.Cookie) DownloadOption {
176 | 	return func(d *downloader) {
177 | 		d.cookies = cookies
178 | 	}
179 | }
180 | 
181 | func WithProxy(proxy string) DownloadOption {
182 | 	return func(d *downloader) {
183 | 		AddProxy(proxy, d.client.Transport.(*http.Transport))
184 | 	}
185 | }
186 | 
187 | func SavePath(savePath func(creator kemono.Creator, post kemono.Post, i int, attachment kemono.File) string) DownloadOption {
188 | 	return func(d *downloader) {
189 | 		d.SavePath = savePath
190 | 	}
191 | }
192 | 
193 | // SetLog set the log
194 | func SetLog(log Log) DownloadOption {
195 | 	return func(d *downloader) {
196 | 		d.log = log
197 | 	}
198 | }
199 | 
200 | func defaultSavePath(creator kemono.Creator, post kemono.Post, i int, attachment kemono.File) string {
201 | 	var name string
202 | 	ext := filepath.Ext(attachment.Name)
203 | 	if ext == "" {
204 | 		ext = filepath.Ext(attachment.Path)
205 | 	}
206 | 	if ext == ".zip" {
207 | 		name = attachment.Name
208 | 	} else {
209 | 		name = filepath.Base(attachment.Path)
210 | 	}
211 | 	return fmt.Sprintf(filepath.Join("./download", "%s", "%s", "%s"), utils.ValidDirectoryName(creator.Name), utils.ValidDirectoryName(DirectoryName(post)), utils.ValidDirectoryName(name))
212 | }
213 | 
214 | // Async set the async download option
215 | func Async(async bool) DownloadOption {
216 | 	return func(d *downloader) {
217 | 		d.Async = async
218 | 	}
219 | }
220 | 
221 | // OverWrite set the overwrite option
222 | func OverWrite(overwrite bool) DownloadOption {
223 | 	return func(d *downloader) {
224 | 		d.OverWrite = overwrite
225 | 	}
226 | }
227 | 
228 | func Retry(retry int) DownloadOption {
229 | 	return func(d *downloader) {
230 | 		d.retry = retry
231 | 	}
232 | }
233 | 
234 | func RetryInterval(interval time.Duration) DownloadOption {
235 | 	return func(d *downloader) {
236 | 		d.retryInterval = interval
237 | 	}
238 | }
239 | 
240 | func WithContent(content bool) DownloadOption {
241 | 	return func(d *downloader) {
242 | 		d.content = content
243 | 	}
244 | }
245 | 
246 | func (d *downloader) Get(url string) (resp *http.Response, err error) {
247 | 	var (
248 | 		req *http.Request
249 | 	)
250 | 	if req, err = newGetRequest(context.Background(), d.Header, d.cookies, url); err != nil {
251 | 		return
252 | 	}
253 | 	return d.client.Do(req)
254 | }
255 | 
256 | func (d *downloader) WriteContent(creator kemono.Creator, post kemono.Post, content string) error {
257 | 	if !d.content {
258 | 		return nil
259 | 	}
260 | 	path := d.SavePath(creator, post, 0, kemono.File{Path: "content.html", Name: "content.html"})
261 | 	path = filepath.Join(filepath.Dir(path), "content.html")
262 | 	err := os.MkdirAll(filepath.Dir(path), 0755)
263 | 	if err != nil {
264 | 		return err
265 | 	}
266 | 	file, err := os.Create(path)
267 | 	contentTemplate := `<!DOCTYPE html>
268 | <html>
269 | <head>
270 |     <title>{{ .Title }}</title>
271 | </head>
272 | <body>
273 |     {{ .Content }}
274 | </body>
275 | </html>`
276 | 	tmpl, err := template.New("content").Parse(contentTemplate)
277 | 	if err != nil {
278 | 		return err
279 | 	}
280 | 	return tmpl.Execute(file, struct {
281 | 		Title   string
282 | 		Content template.HTML
283 | 	}{
284 | 		Title:   post.Title,
285 | 		Content: template.HTML(content),
286 | 	})
287 | }
288 | 
289 | func (d *downloader) Download(files <-chan kemono.FileWithIndex, creator kemono.Creator, post kemono.Post) <-chan error {
290 | 	var (
291 | 		wg    sync.WaitGroup
292 | 		errCh = make(chan error, len(files))
293 | 	)
294 | 
295 | 	for i := 0; i < d.MaxConcurrent; i++ {
296 | 		wg.Add(1)
297 | 		go func() {
298 | 			for {
299 | 				select {
300 | 
301 | 				case file, ok := <-files:
302 | 					// download file
303 | 					if ok {
304 | 						url := d.BaseURL + file.GetURL()
305 | 						hash, err := file.GetHash()
306 | 						if err != nil {
307 | 							hash = ""
308 | 						}
309 | 						savePath := d.SavePath(creator, post, file.Index, file.File)
310 | 
311 | 						if err := d.download(savePath, url, hash); err != nil {
312 | 							errCh <- err
313 | 							continue
314 | 						}
315 | 					}
316 | 				default:
317 | 					if len(files) == 0 {
318 | 						wg.Done()
319 | 						return
320 | 					}
321 | 				}
322 | 			}
323 | 		}()
324 | 	}
325 | 	wg.Wait()
326 | 	return errCh
327 | }
328 | 
329 | // download downloads the file from the url
330 | func (d *downloader) download(filePath, url, fileHash string) error {
331 | 	// check if the file exists
332 | 	var (
333 | 		complete bool
334 | 		err      error
335 | 	)
336 | 	if !d.OverWrite {
337 | 		complete, err = checkFileExitAndComplete(filePath, fileHash)
338 | 		if err != nil {
339 | 			err = errors.New("check file error: " + err.Error())
340 | 			return err
341 | 		}
342 | 		if complete {
343 | 			d.log.Printf("file %s already exists, skip", filePath)
344 | 			return nil
345 | 		}
346 | 	}
347 | 
348 | 	err = os.MkdirAll(filepath.Dir(filePath), os.ModePerm)
349 | 	if err != nil {
350 | 		err = errors.New("create directory error: " + err.Error())
351 | 		return err
352 | 	}
353 | 	// download the file
354 | 	if err := d.downloadFile(filePath, url); err != nil {
355 | 		//err = errors.New("download file error: " + err.Error())
356 | 		return err
357 | 	}
358 | 	time.Sleep(1 * time.Second)
359 | 	return nil
360 | }
361 | 
362 | // download the file from the url, and save to the file
363 | func (d *downloader) downloadFile(filePath, url string) error {
364 | 	d.reteLimiter.Token()
365 | 
366 | 	ctx, cancel := context.WithTimeout(context.Background(), d.Timeout)
367 | 	defer cancel()
368 | 
369 | 	req, err := newGetRequest(ctx, d.Header, d.cookies, url)
370 | 	if err != nil {
371 | 		return fmt.Errorf("new request error: %w", err)
372 | 	}
373 | 
374 | 	var get func() error
375 | 
376 | 	get = func() error {
377 | 		bar := NewProgressBar(fmt.Sprintf("%s", filepath.Base(filePath)), 0, 30)
378 | 		d.progress.AddBar(bar)
379 | 		defer func() {
380 | 			if !bar.IsDone() {
381 | 				d.progress.Failed(bar, fmt.Errorf("download failed"))
382 | 			}
383 | 		}()
384 | 
385 | 		resp, err := d.client.Do(req)
386 | 		if err != nil {
387 | 			return fmt.Errorf("failed to make request: %w", err)
388 | 		}
389 | 		defer resp.Body.Close()
390 | 
391 | 		// get content length
392 | 		contentLength, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
393 | 		if err != nil {
394 | 			return fmt.Errorf("failed to get content length: %w", err)
395 | 		}
396 | 		bar.Max = contentLength
397 | 
398 | 		if contentLength > d.maxSize || contentLength < d.minSize {
399 | 			d.progress.Cancel(bar, "size out of range")
400 | 			return nil
401 | 		}
402 | 
403 | 		// 429 too many requests
404 | 		if resp.StatusCode == http.StatusTooManyRequests {
405 | 			d.progress.Failed(bar, fmt.Errorf("http 429"))
406 | 			return fmt.Errorf("request too many times")
407 | 		}
408 | 
409 | 		if resp.StatusCode != http.StatusOK {
410 | 			d.progress.Failed(bar, fmt.Errorf("http %d", resp.StatusCode))
411 | 			return fmt.Errorf("failed to download file: %d", resp.StatusCode)
412 | 		}
413 | 
414 | 		tmpFilePath := filePath + ".tmp"
415 | 		tmpFile, err := os.Create(tmpFilePath)
416 | 		if err != nil {
417 | 			// delete the tmp file
418 | 			_ = os.Remove(tmpFilePath)
419 | 			return fmt.Errorf("create tmp file error: %w", err)
420 | 		}
421 | 
422 | 		defer func() {
423 | 			_ = tmpFile.Close()
424 | 			_ = os.Remove(tmpFilePath)
425 | 		}()
426 | 
427 | 		_, err = io.Copy(io.MultiWriter(tmpFile, bar), resp.Body)
428 | 		if err != nil {
429 | 			d.progress.Failed(bar, err)
430 | 			return fmt.Errorf("io copy error: %w", err)
431 | 		}
432 | 
433 | 		err = tmpFile.Close()
434 | 		if err != nil {
435 | 			return fmt.Errorf("close tmp file error: %w", err)
436 | 		}
437 | 
438 | 		// rename the tmp file to the file
439 | 		err = os.Rename(tmpFilePath, filePath)
440 | 		if err != nil {
441 | 			return fmt.Errorf("rename file error: %w", err)
442 | 		}
443 | 
444 | 		d.progress.Success(bar)
445 | 		return nil
446 | 	}
447 | 
448 | 	for i := 0; i < d.retry; i++ {
449 | 		err = get()
450 | 		if err == nil {
451 | 			return nil
452 | 		}
453 | 		d.log.Printf("download failed: %s, retry after %.1f seconds...", err.Error(), d.retryInterval.Seconds())
454 | 		time.Sleep(d.retryInterval)
455 | 	}
456 | 	return fmt.Errorf("failed to download file: %w", err)
457 | 
458 | }
459 | 
460 | // check if the file exists, if exists, check if the file is complete,and return the file
461 | // if the file is complete, return true
462 | func checkFileExitAndComplete(filePath, fileHash string) (complete bool, err error) {
463 | 	// check if the file exists
464 | 	var (
465 | 		h    []byte
466 | 		file *os.File
467 | 	)
468 | 	f, err := os.Stat(filePath)
469 | 	if err != nil {
470 | 		// un exists
471 | 		return false, nil
472 | 	} else if f != nil {
473 | 		// file exists, check if the file is complete
474 | 		file, err = os.OpenFile(filePath, os.O_RDWR, 0644)
475 | 		defer file.Close()
476 | 		if err != nil {
477 | 			err = fmt.Errorf("open file error: %w", err)
478 | 			return
479 | 		}
480 | 		h, err = utils.Hash(file)
481 | 		if err != nil {
482 | 			err = fmt.Errorf("get file hash error: %w", err)
483 | 			return
484 | 		}
485 | 		// check if the file is complete
486 | 		if fmt.Sprintf("%x", h) == fileHash {
487 | 			complete = true
488 | 			return
489 | 		}
490 | 	}
491 | 	return false, nil
492 | }
493 | 
494 | func newGetRequest(ctx context.Context, header Header, cookies []*http.Cookie, url string) (*http.Request, error) {
495 | 	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
496 | 	if err != nil {
497 | 		return nil, fmt.Errorf("failed to create request: %w", err)
498 | 	}
499 | 	// set headers
500 | 	for k, v := range header {
501 | 		req.Header.Set(k, v)
502 | 	}
503 | 	// set cookies
504 | 	for _, c := range cookies {
505 | 		req.AddCookie(c)
506 | 	}
507 | 	return req, nil
508 | }
509 | 
510 | func DirectoryName(p kemono.Post) string {
511 | 	return fmt.Sprintf("[%s] [%s] %s", p.Published.Format("20060102"), p.Id, p.Title)
512 | }
513 | 


--------------------------------------------------------------------------------
/downloader/progress.go:
--------------------------------------------------------------------------------
  1 | package downloader
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"github.com/elvis972602/kemono-scraper/utils"
  6 | 	"strings"
  7 | 	"sync"
  8 | 	"sync/atomic"
  9 | 	"time"
 10 | )
 11 | 
 12 | const (
 13 | 	DeepRed    = "\x1b[38;5;196m"
 14 | 	Red        = "\x1b[38;5;197m"
 15 | 	Green      = "\x1b[38;5;106m"
 16 | 	DeepYellow = "\x1b[38;5;178m"
 17 | 	Blue       = "\x1b[38;5;67m"
 18 | 	Purple     = "\x1b[38;5;133m"
 19 | 	Grey       = "\x1b[38;5;243m"
 20 | 	White      = "\x1b[38;5;251m"
 21 | )
 22 | 
 23 | const (
 24 | 	BarModeDownload = "Download"
 25 | 	BarModeCancel   = "Cancel"
 26 | 	BarModeFailed   = "Failed"
 27 | 	BarModeSuccess  = "Success"
 28 | )
 29 | 
 30 | type progressBar struct {
 31 | 	Start   time.Time
 32 | 	Content string
 33 | 	Max     int64
 34 | 	cur     int64
 35 | 	Length  int
 36 | 	done    bool
 37 | }
 38 | 
 39 | func NewProgressBar(content string, max int64, length int) *progressBar {
 40 | 	return &progressBar{Start: time.Now(), Content: content, Max: max, Length: length}
 41 | }
 42 | 
 43 | func (p *progressBar) Add(n int) {
 44 | 	p.Add64(int64(n))
 45 | }
 46 | 
 47 | func (p *progressBar) Add64(n int64) {
 48 | 	atomic.AddInt64(&p.cur, n)
 49 | }
 50 | 
 51 | func (p *progressBar) Set(n int) {
 52 | 	p.Set64(int64(n))
 53 | }
 54 | 
 55 | func (p *progressBar) Set64(n int64) {
 56 | 	atomic.StoreInt64(&p.cur, n)
 57 | }
 58 | 
 59 | func (p *progressBar) String(mode string) string {
 60 | 	//var process string
 61 | 	var pre float64
 62 | 	if p.Max == 0 {
 63 | 		pre = 0
 64 | 
 65 | 	} else {
 66 | 		pre = float64(p.cur) / float64(p.Max)
 67 | 	}
 68 | 	speed := int64(float64(p.cur) / time.Since(p.Start).Seconds())
 69 | 	if speed < 0 {
 70 | 		speed = 0
 71 | 	}
 72 | 	return buildProgressBar(utils.FormatDuration(int64(time.Since(p.Start))), mode, utils.FormatSize(speed), utils.FormatSize(p.Max), p.Content, pre, 30, mode)
 73 | }
 74 | 
 75 | func (p *progressBar) Done() {
 76 | 	p.done = true
 77 | }
 78 | 
 79 | func (p *progressBar) IsDone() bool {
 80 | 	return p.done
 81 | }
 82 | 
 83 | func (p *progressBar) Write(b []byte) (n int, err error) {
 84 | 	n = len(b)
 85 | 	p.Add(n)
 86 | 	return
 87 | }
 88 | 
 89 | type Progress struct {
 90 | 	progressBars []*progressBar
 91 | 	count        int
 92 | 	pre          int
 93 | 	lock         sync.Mutex
 94 | 	log          Log
 95 | }
 96 | 
 97 | func NewProgress(log Log) *Progress {
 98 | 	return &Progress{pre: 0, log: log}
 99 | }
100 | 
101 | func (p *Progress) AddBar(bar *progressBar) {
102 | 	p.lock.Lock()
103 | 	defer p.lock.Unlock()
104 | 	p.progressBars = append(p.progressBars, bar)
105 | }
106 | 
107 | func (p *Progress) Remove(bar *progressBar) {
108 | 	p.lock.Lock()
109 | 	defer p.lock.Unlock()
110 | 	for i, v := range p.progressBars {
111 | 		if v == bar {
112 | 			if i == len(p.progressBars)-1 {
113 | 				p.progressBars = p.progressBars[:i]
114 | 			} else {
115 | 				p.progressBars = append(p.progressBars[:i], p.progressBars[i+1:]...)
116 | 			}
117 | 		}
118 | 	}
119 | }
120 | 
121 | func (p *Progress) Success(bar *progressBar) {
122 | 	bar.Done()
123 | 	p.Remove(bar)
124 | 	p.SetStatus()
125 | 	p.Print(bar.String(BarModeSuccess))
126 | }
127 | 
128 | func (p *Progress) Failed(bar *progressBar, err error) {
129 | 	bar.Done()
130 | 	p.Remove(bar)
131 | 	p.SetStatus()
132 | 	p.Print(bar.String(BarModeFailed))
133 | 	p.Print(DeepRed + err.Error())
134 | }
135 | 
136 | func (p *Progress) Cancel(bar *progressBar, err string) {
137 | 	bar.Done()
138 | 	p.Remove(bar)
139 | 	p.SetStatus()
140 | 	p.Print(bar.String(BarModeCancel))
141 | 	p.Print(DeepRed + err)
142 | }
143 | 
144 | func (p *Progress) SetStatus() {
145 | 	var s []string
146 | 	p.lock.Lock()
147 | 	defer p.lock.Unlock()
148 | 	for i := 0; i < len(p.progressBars); i++ {
149 | 		s = append(s, p.progressBars[i].String(BarModeDownload))
150 | 	}
151 | 	if len(s) == 0 {
152 | 		s = append(s, "")
153 | 	}
154 | 	p.log.SetStatus(s)
155 | }
156 | 
157 | func (p *Progress) Print(s string) {
158 | 	p.log.Print(s)
159 | }
160 | 
161 | func (p *Progress) Run(interval time.Duration) {
162 | 	go func() {
163 | 		tick := time.NewTicker(interval)
164 | 		for {
165 | 			select {
166 | 			case <-tick.C:
167 | 				p.SetStatus()
168 | 			}
169 | 		}
170 | 	}()
171 | }
172 | 
173 | func buildProgressBar(timeStr, prefix, speed, sizeStr, filename string, percent float64, length int, mode string) string {
174 | 	var barColor string
175 | 	switch mode {
176 | 	case BarModeDownload:
177 | 		barColor = Red
178 | 	case BarModeCancel:
179 | 		barColor = Grey
180 | 	case BarModeFailed:
181 | 		barColor = DeepRed
182 | 	case BarModeSuccess:
183 | 		barColor = Green
184 | 	}
185 | 	var process strings.Builder
186 | 	process.WriteString(DeepYellow)
187 | 	process.WriteString(fmt.Sprintf("%9s", timeStr))
188 | 	process.WriteString(" ")
189 | 	process.WriteString(White)
190 | 	process.WriteString(fmt.Sprintf("%8s", prefix))
191 | 	process.WriteString(" ")
192 | 	completedChars := int(percent * float64(length))
193 | 	process.WriteString(barColor)
194 | 	for i := 0; i < completedChars; i++ {
195 | 		process.WriteString("━")
196 | 	}
197 | 	if mode == BarModeDownload {
198 | 		process.WriteString(Grey)
199 | 		if completedChars > 0 && completedChars < length {
200 | 			completedChars++
201 | 			process.WriteString("╺")
202 | 		}
203 | 	}
204 | 	for i := completedChars; i < length; i++ {
205 | 		process.WriteString("━")
206 | 	}
207 | 	process.WriteString(" ")
208 | 	process.WriteString(Purple)
209 | 	process.WriteString(fmt.Sprintf("%5.1f", percent*100))
210 | 	process.WriteString("%")
211 | 	process.WriteString(" ")
212 | 	process.WriteString(Blue)
213 | 	process.WriteString(fmt.Sprintf("%10s", speed))
214 | 	process.WriteString("/s")
215 | 	process.WriteString(" ")
216 | 	process.WriteString(White)
217 | 	process.WriteString(fmt.Sprintf("%9s", sizeStr))
218 | 	process.WriteString(" ")
219 | 	process.WriteString(Grey)
220 | 	process.WriteString(filename)
221 | 	return process.String()
222 | }
223 | 


--------------------------------------------------------------------------------
/downloader/progress_test.go:
--------------------------------------------------------------------------------
 1 | package downloader
 2 | 
 3 | import (
 4 | 	"testing"
 5 | )
 6 | 
 7 | func TestProgressBar_Builder(t *testing.T) {
 8 | 	// download
 9 | 	bar := NewProgressBar(
10 | 		"test",
11 | 		100,
12 | 		30,
13 | 	)
14 | 	for i := 0; i <= 100; i++ {
15 | 		t.Log(bar.String(BarModeDownload))
16 | 		bar.Add64(1)
17 | 	}
18 | 
19 | 	// success
20 | 	bar = NewProgressBar(
21 | 		"test",
22 | 		100,
23 | 		30,
24 | 	)
25 | 	for i := 0; i <= 100; i++ {
26 | 		t.Log(bar.String(BarModeSuccess))
27 | 		bar.Add64(1)
28 | 	}
29 | 
30 | 	// failed
31 | 	bar = NewProgressBar(
32 | 		"test",
33 | 		100,
34 | 		30,
35 | 	)
36 | 	for i := 0; i <= 100; i++ {
37 | 		t.Log(bar.String(BarModeFailed))
38 | 		bar.Add64(1)
39 | 	}
40 | 
41 | 	// cancel
42 | 	bar = NewProgressBar(
43 | 		"test",
44 | 		100,
45 | 		30,
46 | 	)
47 | 	for i := 0; i <= 100; i++ {
48 | 		t.Log(bar.String(BarModeCancel))
49 | 		bar.Add64(1)
50 | 	}
51 | 
52 | }
53 | 


--------------------------------------------------------------------------------
/downloader/proxy.go:
--------------------------------------------------------------------------------
 1 | package downloader
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"golang.org/x/net/proxy"
 6 | 	"net"
 7 | 	"net/http"
 8 | 	"net/url"
 9 | 	"strings"
10 | )
11 | 
12 | func parseProxyUrl(proxyUrlStr string) (proxyType, proxyAddr string) {
13 | 	u, err := url.Parse(proxyUrlStr)
14 | 	if err != nil {
15 | 		panic(err)
16 | 	}
17 | 	proxyType = strings.ToLower(u.Scheme)
18 | 	proxyAddr = u.Host
19 | 	return
20 | }
21 | 
22 | func AddProxy(proxyUrlStr string, transport *http.Transport) {
23 | 	proxyType, _ := parseProxyUrl(proxyUrlStr)
24 | 	proxyUrl, err := url.Parse(proxyUrlStr)
25 | 	if err != nil {
26 | 		panic(err)
27 | 	}
28 | 	switch proxyType {
29 | 	case "http", "https":
30 | 		transport.Proxy = http.ProxyURL(proxyUrl)
31 | 	case "socks5":
32 | 		var dialer proxy.Dialer
33 | 		dialer, err = proxy.FromURL(proxyUrl, proxy.Direct)
34 | 		if err != nil {
35 | 			panic(err)
36 | 		}
37 | 		transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
38 | 			return dialer.Dial(network, addr)
39 | 		}
40 | 	default:
41 | 		panic("unsupported proxy type: " + proxyType)
42 | 	}
43 | }
44 | 


--------------------------------------------------------------------------------
/example/example.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"os"
 6 | 	"path/filepath"
 7 | 	"time"
 8 | 
 9 | 	"github.com/elvis972602/kemono-scraper/downloader"
10 | 	"github.com/elvis972602/kemono-scraper/kemono"
11 | 	"github.com/elvis972602/kemono-scraper/term"
12 | 	"github.com/elvis972602/kemono-scraper/utils"
13 | )
14 | 
15 | func main() {
16 | 	t := term.NewTerminal(os.Stdout, os.Stderr, false)
17 | 
18 | 	d := downloader.NewDownloader(
19 | 		downloader.BaseURL("https://kemono.su"),
20 | 		// the amount of download at the same time
21 | 		downloader.MaxConcurrent(3),
22 | 		downloader.Timeout(300*time.Second),
23 | 		// async download, download several files at the same time,
24 | 		// may cause the file order is not the same as the post order
25 | 		// you can use save path rule to control it
26 | 		downloader.Async(true),
27 | 		// the file will order by name in <order>-<file name>
28 | 		downloader.SavePath(func(creator kemono.Creator, post kemono.Post, i int, attachment kemono.File) string {
29 | 			var name string
30 | 			if filepath.Ext(attachment.Name) == ".zip" {
31 | 				name = attachment.Name
32 | 			} else {
33 | 				name = fmt.Sprintf("%d-%s", i, attachment.Name)
34 | 			}
35 | 			return fmt.Sprintf(filepath.Join("./download", "%s", "%s", "%s"), utils.ValidDirectoryName(creator.Name), utils.ValidDirectoryName(post.Title), utils.ValidDirectoryName(name))
36 | 		}),
37 | 		downloader.WithHeader(downloader.Header{
38 | 			"User-Agent":      downloader.UserAgent,
39 | 			"Referer":         "https://kemono.su",
40 | 			"accept":          downloader.Accept,
41 | 			"accept-encoding": "gzip, deflate, br",
42 | 			"accept-language": "ja-JP;q=0.8,ja;q=0.7,en-US;q=0.6,en;q=0.5",
43 | 		}),
44 | 		downloader.RateLimit(2),
45 | 		downloader.Retry(5),
46 | 		downloader.RetryInterval(5*time.Second),
47 | 		downloader.SetLog(t),
48 | 	)
49 | 	user1 := kemono.NewCreator("service1", "123456")
50 | 	user2 := kemono.NewCreator("service2", "654321")
51 | 	K := kemono.NewKemono(
52 | 		kemono.WithUsers(user1, user2),
53 | 		kemono.WithUsersPair("service3", "987654"),
54 | 		kemono.WithBanner(true),
55 | 		kemono.WithPostFilter(
56 | 			kemono.ReleaseDateFilter(time.Now().AddDate(0, 0, -365), time.Now()),
57 | 		),
58 | 		kemono.WithAttachmentFilter(
59 | 			kemono.ExtensionFilter(".jpg", ".png", ".zip", ".gif"),
60 | 		),
61 | 		// a post filter for specific user
62 | 		kemono.WithUserPostFilter(user1, kemono.EditDateFilter(time.Now().AddDate(0, 0, -20), time.Now())),
63 | 		// an attachment filter for specific user
64 | 		kemono.WithUserAttachmentFilter(user2, func(i int, attachment kemono.File) bool {
65 | 			if i%2 == 0 {
66 | 				return false
67 | 			}
68 | 			return true
69 | 		}),
70 | 		kemono.SetDownloader(d),
71 | 		// if not set , use default log
72 | 		kemono.SetLog(t),
73 | 	)
74 | 	K.Start()
75 | }
76 | 


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
 1 | module github.com/elvis972602/kemono-scraper
 2 | 
 3 | go 1.18
 4 | 
 5 | require (
 6 | 	github.com/mattn/go-colorable v0.1.13
 7 | 	github.com/mattn/go-sqlite3 v1.14.16
 8 | 	github.com/zalando/go-keyring v0.2.2
 9 | 	golang.org/x/crypto v0.14.0
10 | 	golang.org/x/net v0.17.0
11 | 	golang.org/x/sys v0.13.0
12 | 	golang.org/x/term v0.13.0
13 | 	gopkg.in/yaml.v3 v3.0.1
14 | )
15 | 
16 | require (
17 | 	github.com/godbus/dbus/v5 v5.1.0 // indirect
18 | 	github.com/mattn/go-isatty v0.0.17 // indirect
19 | 	github.com/spf13/cast v1.5.1 // indirect
20 | )
21 | 
22 | replace github.com/mattn/go-colorable => github.com/elvis972602/go-colorable v0.0.0-20230322143039-2b733b5d5ca7
23 | 


--------------------------------------------------------------------------------
/kemono/fetch.go:
--------------------------------------------------------------------------------
  1 | package kemono
  2 | 
  3 | import (
  4 | 	"compress/gzip"
  5 | 	"encoding/json"
  6 | 	"fmt"
  7 | 	"io"
  8 | 	"io/ioutil"
  9 | 	"net/http"
 10 | 	"path/filepath"
 11 | 	"time"
 12 | 
 13 | 	"github.com/elvis972602/kemono-scraper/utils"
 14 | )
 15 | 
 16 | // FetchCreators fetch Creator list
 17 | func (k *Kemono) FetchCreators() (creators []Creator, err error) {
 18 | 	k.log.Print("fetching creator list...")
 19 | 	url := fmt.Sprintf("https://%s.su/api/v1/creators", k.Site)
 20 | 	resp, err := k.Downloader.Get(url)
 21 | 	if err != nil {
 22 | 		return nil, fmt.Errorf("fetch creator list error: %s", err)
 23 | 	}
 24 | 
 25 | 	reader, err := handleCompressedHTTPResponse(resp)
 26 | 	if err != nil {
 27 | 		return nil, err
 28 | 	}
 29 | 
 30 | 	data, err := ioutil.ReadAll(reader)
 31 | 	if err != nil {
 32 | 		return nil, fmt.Errorf("fetch creator list error: %s", err)
 33 | 	}
 34 | 	err = json.Unmarshal(data, &creators)
 35 | 	if err != nil {
 36 | 		return nil, fmt.Errorf("unmarshal creator list error: %s", err)
 37 | 	}
 38 | 	return
 39 | }
 40 | 
 41 | // FetchPosts fetch post list
 42 | func (k *Kemono) FetchPosts(service, id string) (posts []Post, err error) {
 43 | 	url := fmt.Sprintf("https://%s.su/api/v1/%s/user/%s", k.Site, service, id)
 44 | 	perUnit := 50
 45 | 	fetch := func(page int) (err error, finish bool) {
 46 | 		k.log.Printf("fetching post list page %d...", page)
 47 | 		purl := fmt.Sprintf("%s?o=%d", url, page*perUnit)
 48 | 
 49 | 		retryCount := 0
 50 | 		for retryCount < k.retry {
 51 | 			resp, err := k.Downloader.Get(purl)
 52 | 			if err != nil {
 53 | 				k.log.Printf("fetch post list error: %v", err)
 54 | 				time.Sleep(k.retryInterval)
 55 | 				retryCount++
 56 | 				continue
 57 | 			}
 58 | 
 59 | 			if resp.StatusCode != http.StatusOK {
 60 | 				k.log.Printf("fetch post list error: %s", resp.Status)
 61 | 				time.Sleep(k.retryInterval)
 62 | 				retryCount++
 63 | 				continue
 64 | 			}
 65 | 
 66 | 			reader, err := handleCompressedHTTPResponse(resp)
 67 | 			if err != nil {
 68 | 				return err, false
 69 | 			}
 70 | 
 71 | 			data, err := ioutil.ReadAll(reader)
 72 | 			if err != nil {
 73 | 				return fmt.Errorf("fetch post list error: %s", err), false
 74 | 			}
 75 | 			reader.Close()
 76 | 
 77 | 			var pr []PostRaw
 78 | 			err = json.Unmarshal(data, &pr)
 79 | 			if err != nil {
 80 | 				return fmt.Errorf("unmarshal post list error: %s", err), false
 81 | 			}
 82 | 			if len(pr) == 0 {
 83 | 				// final page
 84 | 				return nil, true
 85 | 			}
 86 | 			for _, p := range pr {
 87 | 				posts = append(posts, p.ParasTime())
 88 | 			}
 89 | 			return nil, false
 90 | 		}
 91 | 
 92 | 		return fmt.Errorf("fetch post list error: maximum retry count exceeded"), false
 93 | 	}
 94 | 
 95 | 	for i := 0; ; i++ {
 96 | 		err, finish := fetch(i)
 97 | 		if err != nil {
 98 | 			return nil, err
 99 | 		}
100 | 		if finish {
101 | 			break
102 | 		}
103 | 	}
104 | 	return
105 | }
106 | 
107 | // DownloadPosts download posts
108 | func (k *Kemono) DownloadPosts(creator Creator, posts []Post) (err error) {
109 | 	for _, post := range posts {
110 | 		k.log.Printf("download post: %s", utils.ValidDirectoryName(post.Title))
111 | 		if post.Content != "" {
112 | 			err = k.Downloader.WriteContent(creator, post, post.Content)
113 | 			if err != nil {
114 | 				k.log.Printf("write content error: %s", err)
115 | 			}
116 | 		}
117 | 		if len(post.Attachments) == 0 {
118 | 			// no attachment
119 | 			continue
120 | 		}
121 | 		attachmentsChan := make(chan FileWithIndex, len(post.Attachments))
122 | 		for _, a := range AddIndexToAttachments(post.Attachments) {
123 | 			attachmentsChan <- a
124 | 		}
125 | 		errChan := k.Downloader.Download(attachmentsChan, creator, post)
126 | 		for i := 0; i < len(errChan); i++ {
127 | 			err, ok := <-errChan
128 | 			if ok {
129 | 				k.log.Printf("download post error: %s", err)
130 | 				// TODO: record error...
131 | 			} else {
132 | 				break
133 | 			}
134 | 		}
135 | 	}
136 | 	return
137 | }
138 | 
139 | func handleCompressedHTTPResponse(resp *http.Response) (io.ReadCloser, error) {
140 | 	switch resp.Header.Get("Content-Encoding") {
141 | 	case "gzip":
142 | 		reader, err := gzip.NewReader(resp.Body)
143 | 		if err != nil {
144 | 			return nil, err
145 | 		}
146 | 		return reader, nil
147 | 	default:
148 | 		return resp.Body, nil
149 | 	}
150 | }
151 | 
152 | func AddIndexToAttachments(attachments []File) []FileWithIndex {
153 | 	var files []FileWithIndex
154 | 	images := 0
155 | 	others := 0
156 | 	for _, a := range attachments {
157 | 		ext := filepath.Ext(a.Name)
158 | 		if ext == "" {
159 | 			ext = filepath.Ext(a.Path)
160 | 		}
161 | 		if isImage(ext) {
162 | 			files = append(files, a.Index(images))
163 | 			images++
164 | 		} else {
165 | 			files = append(files, a.Index(others))
166 | 			others++
167 | 		}
168 | 	}
169 | 	return files
170 | }
171 | 


--------------------------------------------------------------------------------
/kemono/kemono.go:
--------------------------------------------------------------------------------
  1 | package kemono
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"log"
  6 | 	"net/http"
  7 | 	"os"
  8 | 	"path/filepath"
  9 | 	"time"
 10 | )
 11 | 
 12 | type Downloader interface {
 13 | 	Download(<-chan FileWithIndex, Creator, Post) <-chan error
 14 | 	Get(url string) (resp *http.Response, err error)
 15 | 	WriteContent(Creator, Post, string) error
 16 | }
 17 | 
 18 | type Log interface {
 19 | 	Printf(format string, v ...interface{})
 20 | 	Print(s string)
 21 | }
 22 | 
 23 | type DefaultLog struct {
 24 | 	log *log.Logger
 25 | }
 26 | 
 27 | func (d *DefaultLog) Printf(format string, v ...interface{}) {
 28 | 	d.log.Printf(format, v...)
 29 | }
 30 | 
 31 | func (d *DefaultLog) Print(s string) {
 32 | 	d.log.Print(s)
 33 | }
 34 | 
 35 | // Filter return true for continue, false for skip
 36 | 
 37 | type CreatorFilter func(i int, post Creator) bool
 38 | 
 39 | type PostFilter func(i int, post Post) bool
 40 | 
 41 | type AttachmentFilter func(i int, attachment File) bool
 42 | 
 43 | type Option func(*Kemono)
 44 | 
 45 | type Kemono struct {
 46 | 	// kemono or coomer ...
 47 | 	Site string
 48 | 	//download Banner
 49 | 	Banner bool
 50 | 	// All Creator
 51 | 	creators []Creator
 52 | 
 53 | 	// Creator filter
 54 | 	creatorFilters []CreatorFilter
 55 | 
 56 | 	// Post filter map[creator(<service>:<id>)][]PostFilter
 57 | 	postFilters map[string][]PostFilter
 58 | 
 59 | 	// Attachment filter map[creator(<service>:<id>)][]AttachmentFilter
 60 | 	attachmentFilters map[string][]AttachmentFilter
 61 | 
 62 | 	// Select a specific creator
 63 | 	// If not specified, all creators will be selected
 64 | 	users []Creator
 65 | 
 66 | 	// downloader
 67 | 	Downloader Downloader
 68 | 
 69 | 	log Log
 70 | 
 71 | 	retry int
 72 | 
 73 | 	retryInterval time.Duration
 74 | }
 75 | 
 76 | func NewKemono(options ...Option) *Kemono {
 77 | 	k := &Kemono{
 78 | 		Site:              "kemono",
 79 | 		Banner:            true,
 80 | 		postFilters:       make(map[string][]PostFilter),
 81 | 		attachmentFilters: make(map[string][]AttachmentFilter),
 82 | 		retry:             3,
 83 | 		retryInterval:     5 * time.Second,
 84 | 	}
 85 | 	for _, option := range options {
 86 | 		option(k)
 87 | 	}
 88 | 	// lazy initialize downloader
 89 | 	if k.Downloader == nil {
 90 | 		panic("Downloader is nil")
 91 | 	}
 92 | 	if k.log == nil {
 93 | 		k.log = &DefaultLog{log: log.New(os.Stdout, "", log.LstdFlags)}
 94 | 	}
 95 | 	return k
 96 | }
 97 | 
 98 | // WithDomain Set Site
 99 | func WithDomain(web string) Option {
100 | 	return func(k *Kemono) {
101 | 		k.Site = web
102 | 	}
103 | }
104 | 
105 | func WithBanner(banner bool) Option {
106 | 	return func(k *Kemono) {
107 | 		k.Banner = banner
108 | 	}
109 | }
110 | 
111 | // Custom the creator list
112 | func WithCreators(creators []Creator) Option {
113 | 	return func(k *Kemono) {
114 | 		k.creators = creators
115 | 	}
116 | }
117 | 
118 | // WithUsers Select a specific creator, if not specified, all creators will be selected
119 | func WithUsers(user ...Creator) Option {
120 | 	return func(k *Kemono) {
121 | 		for _, u := range user {
122 | 			exist := false
123 | 			for _, c := range k.users {
124 | 				if c.Service == u.Service && c.Id == u.Id {
125 | 					exist = true
126 | 					break
127 | 				}
128 | 			}
129 | 			if !exist {
130 | 				k.users = append(k.users, u)
131 | 			}
132 | 		}
133 | 	}
134 | }
135 | 
136 | // WithUsersPair Select a specific creator, if not specified, all creators will be selected
137 | func WithUsersPair(serviceIdPairs ...string) Option {
138 | 	return func(k *Kemono) {
139 | 		if len(serviceIdPairs)%2 != 0 {
140 | 			k.log.Printf("serviceIdPairs length must be even")
141 | 			return
142 | 		}
143 | 		for i := 0; i < len(serviceIdPairs); i += 2 {
144 | 			exist := false
145 | 			for _, c := range k.users {
146 | 				if c.Service == serviceIdPairs[i] && c.Id == serviceIdPairs[i+1] {
147 | 					exist = true
148 | 					break
149 | 				}
150 | 			}
151 | 			if !exist {
152 | 				k.users = append(k.users, NewCreator(serviceIdPairs[i], serviceIdPairs[i+1]))
153 | 			}
154 | 		}
155 | 	}
156 | }
157 | 
158 | // SetDownloader set Downloader
159 | func SetDownloader(downloader Downloader) Option {
160 | 	return func(k *Kemono) {
161 | 		k.Downloader = downloader
162 | 	}
163 | }
164 | 
165 | // SetLog set log
166 | func SetLog(log Log) Option {
167 | 	return func(k *Kemono) {
168 | 		k.log = log
169 | 	}
170 | }
171 | 
172 | // SetRetry set retry
173 | func SetRetry(retry int) Option {
174 | 	return func(k *Kemono) {
175 | 		k.retry = retry
176 | 	}
177 | }
178 | 
179 | // SetRetryInterval set retry interval
180 | func SetRetryInterval(retryInterval time.Duration) Option {
181 | 	return func(k *Kemono) {
182 | 		k.retryInterval = retryInterval
183 | 	}
184 | }
185 | 
186 | // WithCreatorFilter Creator filter
187 | func WithCreatorFilter(filter ...CreatorFilter) Option {
188 | 	return func(k *Kemono) {
189 | 		k.addCreatorFilter(filter...)
190 | 	}
191 | }
192 | 
193 | // WithPostFilter Post filter
194 | func WithPostFilter(filter ...PostFilter) Option {
195 | 	return func(k *Kemono) {
196 | 		k.addPostFilter(filter...)
197 | 	}
198 | }
199 | 
200 | func WithUserPostFilter(creator Creator, filter ...PostFilter) Option {
201 | 	return func(k *Kemono) {
202 | 		k.addUserPostFilter(creator.PairString(), filter...)
203 | 	}
204 | }
205 | 
206 | // WithAttachmentFilter Attachment filter
207 | func WithAttachmentFilter(filter ...AttachmentFilter) Option {
208 | 	return func(k *Kemono) {
209 | 		k.addAttachmentFilter(filter...)
210 | 	}
211 | }
212 | 
213 | func WithUserAttachmentFilter(creator Creator, filter ...AttachmentFilter) Option {
214 | 	return func(k *Kemono) {
215 | 		k.addUserAttachmentFilter(creator.PairString(), filter...)
216 | 	}
217 | }
218 | 
219 | // Start fetch and download
220 | func (k *Kemono) Start() error {
221 | 	// initialize the creators
222 | 	if len(k.creators) == 0 {
223 | 		// fetch creators from kemono
224 | 		cs, err := k.FetchCreators()
225 | 		if err != nil {
226 | 			return err
227 | 		}
228 | 		k.creators = cs
229 | 	}
230 | 
231 | 	//find creators
232 | 	if len(k.users) != 0 {
233 | 		var creators []Creator
234 | 		for _, user := range k.users {
235 | 			c, ok := FindCreator(k.creators, user.Id, user.Service)
236 | 			if !ok {
237 | 				k.log.Printf("Creator %s:%s not found", user.Service, user.Id)
238 | 				continue
239 | 			}
240 | 			creators = append(creators, c)
241 | 		}
242 | 		k.users = creators
243 | 	} else {
244 | 		k.users = k.creators
245 | 	}
246 | 
247 | 	// Filter selected creators
248 | 	k.users = k.FilterCreators(k.users)
249 | 
250 | 	// start download
251 | 	k.log.Printf("Start download %d creators", len(k.users))
252 | 	for _, creator := range k.users {
253 | 		// fetch posts
254 | 		posts, err := k.FetchPosts(creator.Service, creator.Id)
255 | 		if err != nil {
256 | 			return err
257 | 		}
258 | 		// filter posts
259 | 		posts = k.FilterPosts(posts)
260 | 
261 | 		// filter attachments
262 | 		for i, post := range posts {
263 | 			// download banner if banner is true or file is not image
264 | 			if (k.Banner || !isImage(filepath.Ext(post.File.Name))) && post.File.Path != "" {
265 | 				res := make([]File, len(post.Attachments)+1)
266 | 				copy(res[1:], post.Attachments)
267 | 				res[0] = post.File
268 | 				post.Attachments = res
269 | 			}
270 | 			posts[i].Attachments = k.FilterAttachments(fmt.Sprintf("%s:%s", post.Service, post.User), post.Attachments)
271 | 		}
272 | 
273 | 		// download posts
274 | 		err = k.DownloadPosts(creator, posts)
275 | 		if err != nil {
276 | 			return err
277 | 		}
278 | 	}
279 | 	return nil
280 | }
281 | 
282 | func (k *Kemono) addCreatorFilter(filter ...CreatorFilter) {
283 | 	k.creatorFilters = append(k.creatorFilters, filter...)
284 | }
285 | 
286 | func (k *Kemono) addPostFilter(filter ...PostFilter) {
287 | 	k.postFilters["*"] = append(k.postFilters["*"], filter...)
288 | }
289 | 
290 | func (k *Kemono) addUserPostFilter(user string, filter ...PostFilter) {
291 | 	k.postFilters[user] = append(k.postFilters[user], filter...)
292 | }
293 | 
294 | func (k *Kemono) addAttachmentFilter(filter ...AttachmentFilter) {
295 | 	k.attachmentFilters["*"] = append(k.attachmentFilters["*"], filter...)
296 | }
297 | 
298 | func (k *Kemono) addUserAttachmentFilter(user string, filter ...AttachmentFilter) {
299 | 	k.attachmentFilters[user] = append(k.attachmentFilters[user], filter...)
300 | }
301 | 
302 | func (k *Kemono) filterCreator(i int, creator Creator) bool {
303 | 	for _, filter := range k.creatorFilters {
304 | 		if !filter(i, creator) {
305 | 			return false
306 | 		}
307 | 	}
308 | 	return true
309 | }
310 | 
311 | func (k *Kemono) filterPost(i int, post Post) bool {
312 | 	for _, filter := range k.postFilters["*"] {
313 | 		if !filter(i, post) {
314 | 			return false
315 | 		}
316 | 	}
317 | 	for _, filter := range k.postFilters[fmt.Sprintf("%s:%s", post.Service, post.User)] {
318 | 		if !filter(i, post) {
319 | 			return false
320 | 		}
321 | 	}
322 | 	return true
323 | }
324 | 
325 | func (k *Kemono) filterAttachment(user string, i int, attachment File) bool {
326 | 	for _, filter := range k.attachmentFilters["*"] {
327 | 		if !filter(i, attachment) {
328 | 			return false
329 | 		}
330 | 	}
331 | 	for _, filter := range k.attachmentFilters[user] {
332 | 		if !filter(i, attachment) {
333 | 			return false
334 | 		}
335 | 	}
336 | 	return true
337 | }
338 | 
339 | func (k *Kemono) FilterCreators(creators []Creator) []Creator {
340 | 	var filteredCreators []Creator
341 | 	for i, creator := range creators {
342 | 		if k.filterCreator(i, creator) {
343 | 			filteredCreators = append(filteredCreators, creator)
344 | 		}
345 | 	}
346 | 	return filteredCreators
347 | }
348 | 
349 | func (k *Kemono) FilterPosts(posts []Post) []Post {
350 | 	var filteredPosts []Post
351 | 	for i, post := range posts {
352 | 		if k.filterPost(i, post) {
353 | 			filteredPosts = append(filteredPosts, post)
354 | 		}
355 | 	}
356 | 	return filteredPosts
357 | }
358 | 
359 | func (k *Kemono) FilterAttachments(user string, attachments []File) []File {
360 | 	var filteredAttachments []File
361 | 	for i, attachment := range attachments {
362 | 		if k.filterAttachment(user, i, attachment) {
363 | 			filteredAttachments = append(filteredAttachments, attachment)
364 | 		}
365 | 	}
366 | 	return filteredAttachments
367 | }
368 | 
369 | // ReleaseDateFilter A Post  filter that filters posts with release date
370 | func ReleaseDateFilter(from, to time.Time) PostFilter {
371 | 	return func(i int, post Post) bool {
372 | 		return post.Published.After(from) && post.Published.Before(to)
373 | 	}
374 | }
375 | 
376 | // ReleaseDateAfterFilter A Post  filter that filters posts with release date after
377 | func ReleaseDateAfterFilter(from time.Time) PostFilter {
378 | 	return func(i int, post Post) bool {
379 | 		return post.Published.After(from)
380 | 	}
381 | }
382 | 
383 | // ReleaseDateBeforeFilter A Post  filter that filters posts with release date before
384 | func ReleaseDateBeforeFilter(to time.Time) PostFilter {
385 | 	return func(i int, post Post) bool {
386 | 		return post.Published.Before(to)
387 | 	}
388 | }
389 | 
390 | // EditDateFilter A Post  filter that filters posts with edit date
391 | func EditDateFilter(from, to time.Time) PostFilter {
392 | 	return func(i int, post Post) bool {
393 | 		return post.Edited.After(from) && post.Edited.Before(to)
394 | 	}
395 | }
396 | 
397 | // EditDateAfterFilter A Post  filter that filters posts with edit date after
398 | func EditDateAfterFilter(from time.Time) PostFilter {
399 | 	return func(i int, post Post) bool {
400 | 		return post.Edited.After(from)
401 | 	}
402 | }
403 | 
404 | // EditDateBeforeFilter A Post  filter that filters posts with edit date before
405 | func EditDateBeforeFilter(to time.Time) PostFilter {
406 | 	return func(i int, post Post) bool {
407 | 		return post.Edited.Before(to)
408 | 	}
409 | }
410 | 
411 | func IdFilter(ids ...string) PostFilter {
412 | 	return func(i int, post Post) bool {
413 | 		for _, id := range ids {
414 | 			if id == post.Id {
415 | 				return true
416 | 			}
417 | 		}
418 | 		return false
419 | 	}
420 | }
421 | 
422 | // NumbFilter A Post filter that filters posts with a specific number
423 | func NumbFilter(f func(i int) bool) PostFilter {
424 | 	return func(i int, post Post) bool {
425 | 		return f(i)
426 | 	}
427 | }
428 | 
429 | // ExtensionFilter A attachmentFilter filter that filters attachments with a specific extension
430 | func ExtensionFilter(extension ...string) AttachmentFilter {
431 | 	return func(i int, attachment File) bool {
432 | 		ext := filepath.Ext(attachment.Name)
433 | 		for _, e := range extension {
434 | 			if ext == e {
435 | 				return true
436 | 			}
437 | 		}
438 | 		return false
439 | 	}
440 | }
441 | 
442 | // ExtensionExcludeFilter A attachmentFilter filter that filters attachments without a specific extension
443 | func ExtensionExcludeFilter(extension ...string) AttachmentFilter {
444 | 	return func(i int, attachment File) bool {
445 | 		ext := filepath.Ext(attachment.Name)
446 | 		for _, e := range extension {
447 | 			if ext == e {
448 | 				return false
449 | 			}
450 | 		}
451 | 		return true
452 | 	}
453 | }
454 | 


--------------------------------------------------------------------------------
/kemono/type.go:
--------------------------------------------------------------------------------
  1 | package kemono
  2 | 
  3 | import (
  4 | 	"encoding/json"
  5 | 	"fmt"
  6 | 	"github.com/elvis972602/kemono-scraper/utils"
  7 | 	"github.com/spf13/cast"
  8 | 	"net/url"
  9 | 	"path/filepath"
 10 | 	"time"
 11 | )
 12 | 
 13 | type Timestamp struct {
 14 | 	Time time.Time
 15 | }
 16 | 
 17 | func (t *Timestamp) UnmarshalJSON(b []byte) error {
 18 | 	var timestamp float64
 19 | 	err := json.Unmarshal(b, &timestamp)
 20 | 	if err != nil {
 21 | 		return err
 22 | 	}
 23 | 
 24 | 	t.Time = time.Unix(int64(timestamp), int64((timestamp-float64(int64(timestamp)))*1e9))
 25 | 	return nil
 26 | }
 27 | 
 28 | type Creator struct {
 29 | 	Favorited int       `json:"favorited"`
 30 | 	Id        string    `json:"id"`
 31 | 	Indexed   Timestamp `json:"indexed"`
 32 | 	Name      string    `json:"name"`
 33 | 	Service   string    `json:"service"`
 34 | 	Updated   Timestamp `json:"updated"`
 35 | }
 36 | 
 37 | // GetID get creator id
 38 | func (c Creator) GetID() string {
 39 | 	return c.Id
 40 | }
 41 | 
 42 | // GetService get creator Service
 43 | func (c Creator) GetService() string {
 44 | 	return c.Service
 45 | }
 46 | 
 47 | func (c Creator) PairString() string {
 48 | 	return fmt.Sprintf("%s:%s", c.Service, c.Id)
 49 | }
 50 | 
 51 | func NewCreator(service, id string) Creator {
 52 | 	return Creator{
 53 | 		Service: service,
 54 | 		Id:      id,
 55 | 	}
 56 | }
 57 | 
 58 | // FindCreator Get the Creator by ID and Service
 59 | func FindCreator(creators []Creator, id, service string) (Creator, bool) {
 60 | 	for _, creator := range creators {
 61 | 		if creator.Id == id && creator.Service == service {
 62 | 			return creator, true
 63 | 		}
 64 | 	}
 65 | 	return Creator{}, false
 66 | }
 67 | 
 68 | type File struct {
 69 | 	Name string `json:"name"`
 70 | 	Path string `json:"path"`
 71 | }
 72 | 
 73 | // GetURL return the url
 74 | func (f File) GetURL() string {
 75 | 	ext := filepath.Ext(f.Name)
 76 | 	name := f.Name[:len(f.Name)-len(ext)]
 77 | 	return fmt.Sprintf("%s?f=%s%s", f.Path, url.QueryEscape(name), ext)
 78 | }
 79 | 
 80 | // GetHash get hash from file path
 81 | func (f File) GetHash() (string, error) {
 82 | 	return utils.SplitHash(f.Path)
 83 | }
 84 | 
 85 | func (f File) Index(n int) FileWithIndex {
 86 | 	return FileWithIndex{
 87 | 		Index: n,
 88 | 		File:  f,
 89 | 	}
 90 | }
 91 | 
 92 | type FileWithIndex struct {
 93 | 	Index int
 94 | 	File
 95 | }
 96 | 
 97 | type PostRaw struct {
 98 | 	Added       string      `json:"added"`
 99 | 	Attachments []File      `json:"attachments"`
100 | 	Content     string      `json:"content"`
101 | 	Edited      string      `json:"edited"`
102 | 	Embed       interface{} `json:"embed"`
103 | 	File        File        `json:"file"`
104 | 	Id          string      `json:"id"`
105 | 	Published   string      `json:"published"`
106 | 	Service     string      `json:"service"`
107 | 	SharedFile  bool        `json:"shared_file"`
108 | 	Title       string      `json:"title"`
109 | 	User        string      `json:"user"`
110 | }
111 | 
112 | func (p PostRaw) ParasTime() Post {
113 | 	var post Post
114 | 	post.Added, _ = cast.StringToDate(p.Added)
115 | 	post.Edited, _ = cast.StringToDate(p.Edited)
116 | 	post.Published, _ = cast.StringToDate(p.Published)
117 | 	post.Id = p.Id
118 | 	post.Service = p.Service
119 | 	post.Title = p.Title
120 | 	post.User = p.User
121 | 	post.Content = p.Content
122 | 	post.Embed = p.Embed
123 | 	post.SharedFile = p.SharedFile
124 | 	post.File = p.File
125 | 	post.Attachments = p.Attachments
126 | 	return post
127 | }
128 | 
129 | type Post struct {
130 | 	Added       time.Time   `json:"added"`
131 | 	Attachments []File      `json:"attachments"`
132 | 	Content     string      `json:"content"`
133 | 	Edited      time.Time   `json:"edited"`
134 | 	Embed       interface{} `json:"embed"`
135 | 	File        File        `json:"file"`
136 | 	Id          string      `json:"id"`
137 | 	Published   time.Time   `json:"published"`
138 | 	Service     string      `json:"service"`
139 | 	SharedFile  bool        `json:"shared_file"`
140 | 	Title       string      `json:"title"`
141 | 	User        string      `json:"user"`
142 | }
143 | 
144 | // User a creator according to the service and id
145 | type User struct {
146 | 	Service string `json:"service"`
147 | 	Id      string `json:"id"`
148 | }
149 | 
150 | // GetID get user id
151 | func (u User) GetID() string {
152 | 	return u.Id
153 | }
154 | 
155 | // GetService get user Service
156 | func (u User) GetService() string {
157 | 	return u.Service
158 | }
159 | 
160 | type FavoriteCreator struct {
161 | 	FavedSeq int    `json:"faved_seq"`
162 | 	Id       string `json:"id"`
163 | 	Index    string `json:"index"`
164 | 	Name     string `json:"name"`
165 | 	Service  string `json:"service"`
166 | 	Update   string `json:"update"`
167 | }
168 | 
169 | var SiteMap = map[string]string{
170 | 	"patreon":       "kemono",
171 | 	"fanbox":        "kemono",
172 | 	"gumroad":       "kemono",
173 | 	"subscribestar": "kemono",
174 | 	"dlsite":        "kemono",
175 | 	"discord":       "kemono",
176 | 	"fantia":        "kemono",
177 | 	"boosty":        "kemono",
178 | 	"afdian":        "kemono",
179 | 	"onlyfans":      "coomer",
180 | 	"fansly":        "coomer",
181 | }
182 | 


--------------------------------------------------------------------------------
/kemono/utils.go:
--------------------------------------------------------------------------------
 1 | package kemono
 2 | 
 3 | func isImage(ext string) bool {
 4 | 	switch ext {
 5 | 	case ".apng", ".avif", ".bmp", ".gif", ".ico", ".cur", ".jpg", ".jpeg", ".jfif", ".pjpeg", ".pjp", ".png", ".svg", ".tif", ".tiff", ".webp", ".jpe":
 6 | 		return true
 7 | 	default:
 8 | 		return false
 9 | 	}
10 | }
11 | 


--------------------------------------------------------------------------------
/main/args.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"flag"
  5 | 	"fmt"
  6 | 	"github.com/mattn/go-colorable"
  7 | 	"gopkg.in/yaml.v3"
  8 | 	"io/ioutil"
  9 | 	"log"
 10 | 	"os"
 11 | 	"strings"
 12 | )
 13 | 
 14 | var (
 15 | 	help bool
 16 | 	// url link
 17 | 	link string
 18 | 	// site
 19 | 	site string
 20 | 	// creator
 21 | 	creator string
 22 | 	// download banner
 23 | 	banner bool
 24 | 
 25 | 	// post filter
 26 | 	// overwrite
 27 | 	overwrite bool
 28 | 	// first n posts
 29 | 	first int
 30 | 	// last n posts
 31 | 	last int
 32 | 	// date
 33 | 	date int
 34 | 	//date before
 35 | 	dateBefore int
 36 | 	//date after
 37 | 	dateAfter int
 38 | 	// update
 39 | 	update int
 40 | 	//update before
 41 | 	updateBefore int
 42 | 	//update after
 43 | 	updateAfter int
 44 | 	// extension only
 45 | 	extensionOnly string
 46 | 	// extension exclude
 47 | 	extensionExclude string
 48 | 
 49 | 	// download options
 50 | 	// output directory
 51 | 	output string
 52 | 	// path template
 53 | 	template string
 54 | 	// Image template
 55 | 	imageTemplate string
 56 | 	// video template
 57 | 	videoTemplate string
 58 | 	// audio template
 59 | 	audioTemplate string
 60 | 	// archive template
 61 | 	archiveTemplate string
 62 | 	// content
 63 | 	content bool
 64 | 	// async
 65 | 	async bool
 66 | 	// max size
 67 | 	maxSize string
 68 | 	// min size
 69 | 	minSize string
 70 | 	// with prefix number
 71 | 	withPrefixNumber bool
 72 | 	// name rule only index
 73 | 	nameRuleOnlyIndex bool
 74 | 	// download timeout
 75 | 	downloadTimeout int
 76 | 	// download retry
 77 | 	retry int
 78 | 	// download retry interval
 79 | 	retryInterval float64
 80 | 	// max download goroutine
 81 | 	maxDownloadParallel int
 82 | 	// request per second
 83 | 	rateLimit int
 84 | 	// proxy url
 85 | 	proxy string
 86 | 
 87 | 	// download favorite creator
 88 | 	favoriteCreator bool
 89 | 	// download favorite post
 90 | 	favoritePost bool
 91 | 	// cookie browser
 92 | 	cookieBrowser string
 93 | 	// cookie file
 94 | 	cookieFile string
 95 | )
 96 | 
 97 | var (
 98 | 	config      map[string]interface{}
 99 | 	passedFlags = make(map[string]bool)
100 | )
101 | 
102 | func init() {
103 | 	log.SetOutput(colorable.NewColorableStdout())
104 | 
105 | 	flag.BoolVar(&help, "help", false, "show all usage")
106 | 	flag.StringVar(&link, "link", "", "download link, should be same site, separate by comma")
107 | 	// if already have link, or creator, site will be ignored
108 | 	flag.StringVar(&site, "fav-site", "", "download favorite creator or post, separate by comma")
109 | 	flag.StringVar(&creator, "creator", "", "--creator <service>:<id>, separate by comma")
110 | 	flag.BoolVar(&banner, "banner", false, "if download banner")
111 | 	flag.BoolVar(&favoriteCreator, "fav-creator", false, "download favorite creator")
112 | 	flag.BoolVar(&favoritePost, "fav-post", false, "download favorite post")
113 | 	flag.StringVar(&cookieBrowser, "cookie-browser", "chrome", "cookie browser, windows only, support chrome, firefox, opera, edge, vivaldi,default is chrome, other system can use --cookie <file name>")
114 | 	flag.StringVar(&cookieFile, "cookie", "cookies.txt", "cookie file, default is cookies.txt (value separate by whitespace)\n"+
115 | 		"structure : +--------+--------------------+------+--------+--------+------+-------+\n"+
116 | 		"            | Domain | Include subdomains | Path | Secure | Expiry | Name | Value |\n"+
117 | 		"            +--------+--------------------+------+--------+--------+------+-------+")
118 | 	// filter
119 | 	flag.BoolVar(&overwrite, "overwrite", false, "if overwrite file")
120 | 	flag.IntVar(&first, "first", 0, "download first n posts")
121 | 	flag.IntVar(&last, "last", 0, "download last n posts")
122 | 	flag.IntVar(&date, "date", 0, "--date YYYYMMDD (notice: date in website is GMT+0)")
123 | 	flag.IntVar(&dateBefore, "date-before", 0, "--date-before YYYYMMDD, select posts before YYYYMMDD")
124 | 	flag.IntVar(&dateAfter, "date-after", 0, "--date-after YYYYMMDD, select posts after YYYYMMDD")
125 | 	flag.IntVar(&update, "update", 0, "--update YYYYMMDD (notice: date in website is GMT+0)")
126 | 	flag.IntVar(&updateBefore, "update-before", 0, "--update-before YYYYMMDD, select posts updated before YYYYMMDD")
127 | 	flag.IntVar(&updateAfter, "update-after", 0, "--update-after YYYYMMDD, select posts updated after YYYYMMDD")
128 | 	flag.StringVar(&extensionOnly, "extension-only", "", "--extension-only, select posts with only extension, separate by comma (e.g. --extension-only jpg,png)")
129 | 	flag.StringVar(&extensionExclude, "extension-exclude", "", "--extension-exclude, select posts without extension, separate by comma (e.g. --extension-exclude jpg,png)")
130 | 
131 | 	// download options
132 | 	flag.StringVar(&output, "output", "", "output directory")
133 | 	flag.StringVar(&template, "template", "", "default path template, e.g. <ks:creator>/<ks:post>/<ks:index>_<ks:filename><ks:extension>")
134 | 	flag.StringVar(&imageTemplate, "image-template", "", "image template, e.g. <ks:creator>/<ks:post>/<ks:index><ks:extension>")
135 | 	flag.StringVar(&videoTemplate, "video-template", "", "video template, e.g. <ks:creator>/<ks:post>/<ks:filename><ks:extension>")
136 | 	flag.StringVar(&audioTemplate, "audio-template", "", "audio template, e.g. <ks:creator>/<ks:post>/<ks:filename><ks:extension>")
137 | 	flag.StringVar(&archiveTemplate, "archive-template", "", "archive template, e.g. <ks:creator>/<ks:post>/<ks:filename><ks:extension>")
138 | 	flag.BoolVar(&content, "content", false, "if download post content")
139 | 	flag.BoolVar(&async, "async", false, "if download posts asynchronously, may cause the file order is not the same as the post order, can be used with --with-prefix-number, default false")
140 | 	flag.StringVar(&maxSize, "max-size", "", "max size, e.g. 10 MB, 1 GB")
141 | 	flag.StringVar(&minSize, "min-size", "", "min size, e.g. 10 MB, 1 GB")
142 | 	flag.BoolVar(&withPrefixNumber, "with-prefix-number", false, "if add prefix number to file name: <index>-<file name> (zip file name is not changed)")
143 | 	flag.BoolVar(&nameRuleOnlyIndex, "name-rule-only-index", false, "if use only index as file name(eg. 1.png, 2.png, ...)")
144 | 	flag.IntVar(&downloadTimeout, "download-timeout", 1800, "download timeout(second), default is 1800s")
145 | 	flag.IntVar(&retry, "retry", 3, "download retry, default is 3")
146 | 	flag.Float64Var(&retryInterval, "retry-interval", 10, "download retry interval(second), default is 10s")
147 | 	flag.IntVar(&maxDownloadParallel, "max-download-parallel", 3, "max download file concurrent, default is 3, async mode only")
148 | 	flag.IntVar(&rateLimit, "rate-limit", 2, "request per second, default is 2")
149 | 	flag.StringVar(&proxy, "proxy", "", "proxy url, e.g. http://proxy.com:8080")
150 | 	_, err := os.Stat("config.yaml")
151 | 
152 | 	if os.IsNotExist(err) {
153 | 		// file does not exist
154 | 		return
155 | 	}
156 | 	if err != nil {
157 | 		log.Printf("check config.yaml failed, %v", err)
158 | 		return
159 | 	}
160 | 	open, err := os.Open("config.yaml")
161 | 	if err != nil {
162 | 		log.Fatalf("open config file error: %v", err)
163 | 	}
164 | 	defer open.Close()
165 | 	bytes, err := ioutil.ReadAll(open)
166 | 	if err != nil {
167 | 		log.Fatalf("read config file error: %v", err)
168 | 	}
169 | 	err = yaml.Unmarshal(bytes, &config)
170 | 	if err != nil {
171 | 		log.Fatalf("unmarshal config file error: %v", err)
172 | 	}
173 | }
174 | 
175 | func setPassedFlags() {
176 | 	flag.Visit(func(f *flag.Flag) {
177 | 		passedFlags[f.Name] = true
178 | 	})
179 | }
180 | 
181 | // PrintDefaults same as flag.PrintDefaults(), but only print the flag with two hyphens and without default value
182 | func PrintDefaults() {
183 | 	flag.VisitAll(func(f *flag.Flag) {
184 | 		var b strings.Builder
185 | 		fmt.Fprintf(&b, "  --%s", f.Name) // Two spaces before -; see next two comments.
186 | 		name, usage := flag.UnquoteUsage(f)
187 | 		if len(name) > 0 {
188 | 			b.WriteString(" ")
189 | 			b.WriteString(name)
190 | 		}
191 | 		// Boolean flags of one ASCII letter are so common we
192 | 		// treat them specially, putting their usage on the same line.
193 | 		if b.Len() <= 4 { // space, space, '-', 'x'.
194 | 			b.WriteString("\t")
195 | 		} else {
196 | 			// Four spaces before the tab triggers good alignment
197 | 			// for both 4- and 8-space tab stops.
198 | 			b.WriteString("\n    \t")
199 | 		}
200 | 		b.WriteString(strings.ReplaceAll(usage, "\n", "\n    \t"))
201 | 		b.WriteString("\n")
202 | 		fmt.Fprint(flag.CommandLine.Output(), b.String())
203 | 	})
204 | }
205 | 


--------------------------------------------------------------------------------
/main/cookie/chromium/chrome.go:
--------------------------------------------------------------------------------
 1 | package chromium
 2 | 
 3 | import (
 4 | 	"github.com/elvis972602/kemono-scraper/main/cookie/utils"
 5 | 	"github.com/elvis972602/kemono-scraper/main/cookie/utils/linux"
 6 | 	"os"
 7 | 	"path/filepath"
 8 | 	"runtime"
 9 | )
10 | 
11 | type ChroniumBasedBrowser struct{}
12 | 
13 | type CookieDecryptor interface {
14 | 	Decrypt(encrypted []byte) ([]byte, error)
15 | }
16 | 
17 | func GetCookieDecryptor(browserRoot, keyringName string, keyring linux.LinuxKeyring) (CookieDecryptor, error) {
18 | 	return NewChromeCookieDecryptor(browserRoot, keyringName, keyring)
19 | }
20 | 
21 | func GetChromiumBasedBrowserSettings(browserName string) (browserDir string, keyringName string, supportsProfiles bool) {
22 | 	appDataLocal := os.Getenv("LOCALAPPDATA")
23 | 	appDataRoaming := os.Getenv("APPDATA")
24 | 	switch runtime.GOOS {
25 | 	case "windows":
26 | 		if browserName == "opera" {
27 | 			browserDir = filepath.Join(appDataRoaming, utils.WindowsBrowserDir[browserName])
28 | 		} else {
29 | 			browserDir = filepath.Join(appDataLocal, utils.WindowsBrowserDir[browserName])
30 | 		}
31 | 	case "darwin":
32 | 		panic("TODO: implement darwin")
33 | 	default:
34 | 		browserDir = filepath.Join(utils.ConfigHome(), utils.LinuxBrowserDir[browserName])
35 | 	}
36 | 
37 | 	keyringName = utils.KeyingName(browserName)
38 | 	supportsProfiles = browserName != "opera"
39 | 	return
40 | }
41 | 


--------------------------------------------------------------------------------
/main/cookie/chromium/chrome_unix.go:
--------------------------------------------------------------------------------
  1 | //go:build linux
  2 | 
  3 | package chromium
  4 | 
  5 | import (
  6 | 	"bytes"
  7 | 	"crypto/aes"
  8 | 	"crypto/cipher"
  9 | 	"crypto/sha1"
 10 | 	"errors"
 11 | 	"github.com/elvis972602/kemono-scraper/main/cookie/utils/linux"
 12 | 	"golang.org/x/crypto/pbkdf2"
 13 | 	"log"
 14 | )
 15 | 
 16 | const (
 17 | 	salt   = "saltysalt"
 18 | 	secret = "peanuts"
 19 | )
 20 | 
 21 | var (
 22 | 	// 16 bytes
 23 | 	iv = []byte("                ")
 24 | )
 25 | 
 26 | type ChromeCookieDecryptor struct {
 27 | 	v10Key []byte
 28 | 	v11Key []byte
 29 | 	v10    int
 30 | 	v11    int
 31 | 	other  int
 32 | }
 33 | 
 34 | func NewChromeCookieDecryptor(browserRoot, keyringName string, keyring linux.LinuxKeyring) (*ChromeCookieDecryptor, error) {
 35 | 	l := &ChromeCookieDecryptor{
 36 | 		v10:   0,
 37 | 		v11:   0,
 38 | 		other: 0,
 39 | 	}
 40 | 	l.v10Key = pbkdf2Sha1([]byte(secret), []byte(salt), 1, 16)
 41 | 	password, err := getLinuxKeyringPassword(keyringName, keyring)
 42 | 	if err != nil {
 43 | 		return nil, err
 44 | 	}
 45 | 	if password != nil {
 46 | 		l.v11Key = pbkdf2Sha1(password, []byte(salt), 1, 16)
 47 | 	}
 48 | 	return l, nil
 49 | }
 50 | 
 51 | func (d *ChromeCookieDecryptor) Decrypt(encrypted []byte) ([]byte, error) {
 52 | 	version := encrypted[:3]
 53 | 	ciphertext := encrypted[3:]
 54 | 	if bytesEqual(version, []byte("v10")) {
 55 | 		// vxx (3 bytes)
 56 | 		// nonce (12 bytes)
 57 | 		// ciphertext (variable)
 58 | 		// tag (16 bytes)
 59 | 		d.v10++
 60 | 		if d.v10Key == nil {
 61 | 			return nil, errors.New("v10 key is nil")
 62 | 		}
 63 | 
 64 | 		return aesCBCDecrypt(ciphertext, d.v10Key, iv)
 65 | 	} else {
 66 | 		d.v11++
 67 | 		if d.v11Key == nil {
 68 | 			return nil, errors.New("v11 key is nil")
 69 | 		}
 70 | 		return aesCBCDecrypt(ciphertext, d.v11Key, iv)
 71 | 	}
 72 | }
 73 | 
 74 | func getLinuxKeyringPassword(keyringName string, keyring linux.LinuxKeyring) ([]byte, error) {
 75 | 	if !linux.LinuxKeyringNames[keyring] {
 76 | 		keyring = linux.ChooseLinuxKeyring()
 77 | 	}
 78 | 	log.Println("Using keyring:", keyring)
 79 | 	if keyring == linux.KWALLET {
 80 | 		return linux.GetKWalletPassword(keyringName)
 81 | 	} else if keyring == linux.GNOMEKEYRING {
 82 | 		return linux.GetGnomeKeyringPassword(keyringName)
 83 | 	} else if keyring == linux.BASICTEXT {
 84 | 		// all store as v10
 85 | 		return nil, nil
 86 | 	}
 87 | 	return nil, errors.New("unknown keyring")
 88 | }
 89 | 
 90 | func pbkdf2Sha1(password, salt []byte, iterations, keyLen int) []byte {
 91 | 	return pbkdf2.Key(password, salt, iterations, keyLen, sha1.New)
 92 | }
 93 | 
 94 | func pkcs7Padding(ciphertext []byte, blockSize int) []byte {
 95 | 	padding := blockSize - len(ciphertext)%blockSize
 96 | 	padtext := bytes.Repeat([]byte{byte(padding)}, padding)
 97 | 	return append(ciphertext, padtext...)
 98 | }
 99 | 
100 | func pkcs7UnPadding(origData []byte) []byte {
101 | 	length := len(origData)
102 | 	unpadding := int(origData[length-1])
103 | 	return origData[:(length - unpadding)]
104 | }
105 | 
106 | func aesCBCEncrypt(plaintext []byte, key, iv []byte) ([]byte, error) {
107 | 	block, err := aes.NewCipher(key)
108 | 	if err != nil {
109 | 		return nil, err
110 | 	}
111 | 	blockSize := block.BlockSize()
112 | 	plaintext = pkcs7Padding(plaintext, blockSize)
113 | 	blockMode := cipher.NewCBCEncrypter(block, iv)
114 | 	crypted := make([]byte, len(plaintext))
115 | 	blockMode.CryptBlocks(crypted, plaintext)
116 | 	return crypted, nil
117 | }
118 | 
119 | func aesCBCDecrypt(ciphertext []byte, key, iv []byte) ([]byte, error) {
120 | 	block, err := aes.NewCipher(key)
121 | 	if err != nil {
122 | 		return nil, err
123 | 	}
124 | 	blockSize := block.BlockSize()
125 | 	blockMode := cipher.NewCBCDecrypter(block, iv[:blockSize])
126 | 	origData := make([]byte, len(ciphertext))
127 | 	blockMode.CryptBlocks(origData, ciphertext)
128 | 	origData = pkcs7UnPadding(origData)
129 | 	return origData, nil
130 | }
131 | 


--------------------------------------------------------------------------------
/main/cookie/chromium/chrome_windows.go:
--------------------------------------------------------------------------------
  1 | //go:build windows
  2 | 
  3 | package chromium
  4 | 
  5 | import (
  6 | 	"bytes"
  7 | 	"crypto/aes"
  8 | 	"crypto/cipher"
  9 | 	"encoding/base64"
 10 | 	"encoding/json"
 11 | 	"errors"
 12 | 	"fmt"
 13 | 	"github.com/elvis972602/kemono-scraper/main/cookie/utils"
 14 | 	"github.com/elvis972602/kemono-scraper/main/cookie/utils/linux"
 15 | 	"log"
 16 | 	"os"
 17 | )
 18 | 
 19 | type ChromeCookieDecryptor struct {
 20 | 	v10Key []byte
 21 | 	v10    int
 22 | 	v11    int
 23 | }
 24 | 
 25 | func NewChromeCookieDecryptor(browserRoot, keyringName string, keyring linux.LinuxKeyring) (*ChromeCookieDecryptor, error) {
 26 | 	k, err := getWindowsV10Key(browserRoot)
 27 | 	if err != nil {
 28 | 		return nil, err
 29 | 	}
 30 | 	return &ChromeCookieDecryptor{
 31 | 		v10Key: k,
 32 | 		v10:    0,
 33 | 		v11:    0,
 34 | 	}, nil
 35 | }
 36 | 
 37 | func (d *ChromeCookieDecryptor) Decrypt(encrypted []byte) ([]byte, error) {
 38 | 	version := encrypted[:3]
 39 | 	ciphertext := encrypted[3:]
 40 | 	if bytesEqual(version, []byte("v10")) {
 41 | 		// vxx (3 bytes)
 42 | 		// nonce (12 bytes)
 43 | 		// ciphertext (variable)
 44 | 		// tag (16 bytes)
 45 | 		d.v10++
 46 | 		if d.v10Key == nil {
 47 | 			return nil, errors.New("v10 key is nil")
 48 | 		}
 49 | 		nonceLength := 96 / 8
 50 | 		authenticationTagLength := 16
 51 | 
 52 | 		rawCiphertext := ciphertext
 53 | 		nonce := rawCiphertext[:nonceLength]
 54 | 		ciphertextWithTag := rawCiphertext[nonceLength:]
 55 | 		// authenticationTag 16 bytes
 56 | 		_ = ciphertextWithTag[len(ciphertext)-authenticationTagLength:]
 57 | 
 58 | 		return decryptAESGCM(ciphertextWithTag, d.v10Key, nonce)
 59 | 	} else {
 60 | 		d.v11++
 61 | 		return utils.DecryptWindowsDpapi(encrypted)
 62 | 	}
 63 | }
 64 | 
 65 | // byte array equal
 66 | func bytesEqual(a, b []byte) bool {
 67 | 	if len(a) != len(b) {
 68 | 		return false
 69 | 	}
 70 | 	for i := range a {
 71 | 		if a[i] != b[i] {
 72 | 			return false
 73 | 		}
 74 | 	}
 75 | 	return true
 76 | }
 77 | 
 78 | func decryptAESGCM(ciphertext, key, nonce []byte) ([]byte, error) {
 79 | 	block, err := aes.NewCipher(key)
 80 | 	if err != nil {
 81 | 		return nil, fmt.Errorf("could not create AES cipher: %v", err)
 82 | 	}
 83 | 
 84 | 	aesgcm, err := cipher.NewGCM(block)
 85 | 	if err != nil {
 86 | 		return nil, fmt.Errorf("could not create AES GCM: %v", err)
 87 | 	}
 88 | 
 89 | 	plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil)
 90 | 	if err != nil {
 91 | 		return nil, fmt.Errorf("could not Decrypt AES GCM: %v", err)
 92 | 	}
 93 | 
 94 | 	return plaintext, nil
 95 | }
 96 | 
 97 | func getWindowsV10Key(browserRoot string) ([]byte, error) {
 98 | 	path := utils.FindMostRecentlyUsedFile(browserRoot, "Local State")
 99 | 	if path == "" {
100 | 		return nil, errors.New("could not find Local State file")
101 | 	}
102 | 	log.Println("found local state file")
103 | 	file, err := os.Open(path)
104 | 	if err != nil {
105 | 		return nil, fmt.Errorf("could not open local state file: %v", err)
106 | 	}
107 | 	defer file.Close()
108 | 	data := make(map[string]interface{})
109 | 	if err := json.NewDecoder(file).Decode(&data); err != nil {
110 | 		return nil, fmt.Errorf("could not decode local state file: %v", err)
111 | 	}
112 | 	base64Key, ok := data["os_crypt"].(map[string]interface{})["encrypted_key"].(string)
113 | 	if !ok {
114 | 		return nil, errors.New("could not find encrypted key in local state file")
115 | 	}
116 | 	encryptedKey, err := base64.StdEncoding.DecodeString(base64Key)
117 | 	if err != nil {
118 | 		return nil, fmt.Errorf("could not base64 decode encrypted key: %v", err)
119 | 	}
120 | 	prefix := []byte("DPAPI")
121 | 	if !bytes.HasPrefix(encryptedKey, prefix) {
122 | 		return nil, errors.New("encrypted key does not have DPAPI prefix")
123 | 	}
124 | 	return utils.DecryptWindowsDpapi(encryptedKey[len(prefix):])
125 | }
126 | 


--------------------------------------------------------------------------------
/main/cookie/cookie_test.go:
--------------------------------------------------------------------------------
 1 | package cookie
 2 | 
 3 | import (
 4 | 	"testing"
 5 | )
 6 | 
 7 | func Test_Chrome_Cookies(t *testing.T) {
 8 | 	c := NewCookies()
 9 | 	err := c.ReadCookies("chrome", "", 0)
10 | 	if err != nil {
11 | 		t.Fatalf("error: %v", err)
12 | 	}
13 | 	_ = c.GetCookies()
14 | }
15 | 
16 | func Test_Firefox_Cookies(t *testing.T) {
17 | 	c := NewCookies()
18 | 	err := c.ReadCookies("firefox", "", 0)
19 | 	if err != nil {
20 | 		t.Fatalf("error: %v", err)
21 | 	}
22 | 	_ = c.GetCookies()
23 | }
24 | 
25 | func Test_Opera_Cookies(t *testing.T) {
26 | 	c := NewCookies()
27 | 	err := c.ReadCookies("opera", "", 0)
28 | 	if err != nil {
29 | 		t.Fatalf("error: %v", err)
30 | 	}
31 | 	_ = c.GetCookies()
32 | }
33 | 
34 | func Test_Edge_Cookies(t *testing.T) {
35 | 	c := NewCookies()
36 | 	err := c.ReadCookies("edge", "", 0)
37 | 	if err != nil {
38 | 		t.Fatalf("error: %v", err)
39 | 	}
40 | 	_ = c.GetCookies()
41 | }
42 | 


--------------------------------------------------------------------------------
/main/cookie/cookies.go:
--------------------------------------------------------------------------------
  1 | package cookie
  2 | 
  3 | import (
  4 | 	"database/sql"
  5 | 	"fmt"
  6 | 	"github.com/elvis972602/kemono-scraper/main/cookie/chromium"
  7 | 	"github.com/elvis972602/kemono-scraper/main/cookie/firefox"
  8 | 	"github.com/elvis972602/kemono-scraper/main/cookie/utils"
  9 | 	"github.com/elvis972602/kemono-scraper/main/cookie/utils/linux"
 10 | 	"io/ioutil"
 11 | 	"log"
 12 | 	"net/http"
 13 | 	"os"
 14 | 	"path"
 15 | 	"path/filepath"
 16 | 	"time"
 17 | 
 18 | 	_ "github.com/mattn/go-sqlite3"
 19 | )
 20 | 
 21 | type Cookies struct {
 22 | 	cookies []*http.Cookie
 23 | }
 24 | 
 25 | func NewCookies() *Cookies {
 26 | 	return &Cookies{}
 27 | }
 28 | 
 29 | func (c *Cookies) GetCookies() []*http.Cookie {
 30 | 	return c.cookies
 31 | }
 32 | 
 33 | func (c *Cookies) ReadCookies(browserName, profile string, keyring linux.LinuxKeyring) error {
 34 | 	browserName, profile, keyring = parseBrowserSpecification(browserName, profile, keyring)
 35 | 	return c.extractCookiesFromBrowser(browserName, profile, keyring)
 36 | }
 37 | 
 38 | func (c *Cookies) extractCookiesFromBrowser(browserName, profile string, keying linux.LinuxKeyring) error {
 39 | 	if browserName == "firefox" {
 40 | 		return c.extractFireFoxCookies(profile)
 41 | 	} else if browserName == "safari" {
 42 | 		// TODO: extract cookies from safari
 43 | 		panic("TODO: implement safari")
 44 | 	} else if utils.ChromiumBasedBrowsers[browserName] {
 45 | 		return c.extractChromiumBasedCookies(browserName, profile, keying)
 46 | 	} else {
 47 | 		return fmt.Errorf("browser %s not supported", browserName)
 48 | 	}
 49 | 	return nil
 50 | }
 51 | 
 52 | func (c *Cookies) extractChromiumBasedCookies(browserName, profile string, keyring linux.LinuxKeyring) error {
 53 | 	var searchRoot string
 54 | 	browserDir, keyringName, supportsProfiles := chromium.GetChromiumBasedBrowserSettings(browserName)
 55 | 
 56 | 	if profile == "" {
 57 | 		searchRoot = browserDir
 58 | 	} else if filepath.IsAbs(profile) {
 59 | 		searchRoot = browserDir
 60 | 		if supportsProfiles {
 61 | 			browserDir = filepath.Dir(profile)
 62 | 		} else {
 63 | 			browserDir = profile
 64 | 		}
 65 | 	} else {
 66 | 		if supportsProfiles {
 67 | 			searchRoot = filepath.Join(browserDir, profile)
 68 | 		} else {
 69 | 			// no profiles, so profile is the path to the browser
 70 | 			searchRoot = browserDir
 71 | 		}
 72 | 	}
 73 | 
 74 | 	cookieDatabasePath := utils.FindMostRecentlyUsedFile(searchRoot, "Cookies")
 75 | 	if cookieDatabasePath == "" {
 76 | 		return fmt.Errorf("could not find cookies database for %s", browserName)
 77 | 	}
 78 | 
 79 | 	log.Println("found cookie database")
 80 | 
 81 | 	decryptor, err := chromium.GetCookieDecryptor(searchRoot, keyringName, keyring)
 82 | 	if err != nil {
 83 | 		return fmt.Errorf("could not get cookie decryptor: %w", err)
 84 | 	}
 85 | 
 86 | 	tmpdir, err := os.MkdirTemp(".", "cookies")
 87 | 	if err != nil {
 88 | 		return fmt.Errorf("could not create temporary directory: %w", err)
 89 | 	}
 90 | 	defer os.RemoveAll(tmpdir)
 91 | 
 92 | 	db, err := openDataBaseCopy(cookieDatabasePath, tmpdir)
 93 | 	if err != nil {
 94 | 		return fmt.Errorf("could not open database copy: %w", err)
 95 | 	}
 96 | 	defer db.Close()
 97 | 
 98 | 	rows, err := db.Query("SELECT host_key, name, value, encrypted_value, path, expires_utc, is_secure FROM cookies")
 99 | 	if err != nil {
100 | 		return fmt.Errorf("could not query cookies: %w", err)
101 | 	}
102 | 	cookies := make([]*http.Cookie, 0)
103 | 	defer rows.Close()
104 | 	count := 0
105 | 	for rows.Next() {
106 | 		count++
107 | 		var (
108 | 			host, name, value, encryptedValue, p string
109 | 			expires, isSecure                    int
110 | 		)
111 | 		if err = rows.Scan(&host, &name, &value, &encryptedValue, &p, &expires, &isSecure); err != nil {
112 | 			return fmt.Errorf("could not scan row: %w", err)
113 | 		}
114 | 		_, cookie, err := processChromeCookie(decryptor, host, name, value, encryptedValue, p, expires, isSecure)
115 | 		if err != nil {
116 | 			err = fmt.Errorf("could not process cookie: %w", err)
117 | 			return err
118 | 		}
119 | 		cookies = append(cookies, cookie)
120 | 	}
121 | 
122 | 	c.cookies = cookies
123 | 
124 | 	return nil
125 | }
126 | 
127 | func (c *Cookies) extractFireFoxCookies(profile string) error {
128 | 	var searchRoot string
129 | 	if profile == "" {
130 | 		searchRoot = firefox.BrowserDir()
131 | 	} else if filepath.IsAbs(profile) {
132 | 		searchRoot = profile
133 | 	} else {
134 | 		searchRoot = filepath.Join(firefox.BrowserDir(), profile)
135 | 	}
136 | 	cookieDatabasePath := utils.FindMostRecentlyUsedFile(searchRoot, "cookies.sqlite")
137 | 	if cookieDatabasePath == "" {
138 | 		return fmt.Errorf("could not find cookies database for firefox")
139 | 	}
140 | 
141 | 	log.Println("found cookie database")
142 | 
143 | 	tmpdir, err := os.MkdirTemp(".", "cookies")
144 | 	if err != nil {
145 | 		return fmt.Errorf("could not create temporary directory: %w", err)
146 | 	}
147 | 	defer os.RemoveAll(tmpdir)
148 | 
149 | 	db, err := openDataBaseCopy(cookieDatabasePath, tmpdir)
150 | 	if err != nil {
151 | 		return fmt.Errorf("could not open database copy: %w", err)
152 | 	}
153 | 	defer db.Close()
154 | 
155 | 	rows, err := db.Query("SELECT host, name, value, path, expiry, isSecure FROM moz_cookies")
156 | 	if err != nil {
157 | 		return fmt.Errorf("could not query cookies: %w", err)
158 | 	}
159 | 	cookies := make([]*http.Cookie, 0)
160 | 	defer rows.Close()
161 | 	count := 0
162 | 	for rows.Next() {
163 | 		count++
164 | 		var (
165 | 			host, name, value, p string
166 | 			expires, isSecure    int
167 | 		)
168 | 		if err = rows.Scan(&host, &name, &value, &p, &expires, &isSecure); err != nil {
169 | 			return fmt.Errorf("could not scan row: %w", err)
170 | 		}
171 | 		cookie := &http.Cookie{
172 | 			Name:     name,
173 | 			Value:    value,
174 | 			Path:     p,
175 | 			Domain:   host,
176 | 			Secure:   isSecure == 1,
177 | 			HttpOnly: true,
178 | 			Expires:  time.Unix(int64(expires), 0),
179 | 		}
180 | 		cookies = append(cookies, cookie)
181 | 	}
182 | 	c.cookies = cookies
183 | 	return nil
184 | }
185 | 
186 | func parseBrowserSpecification(browserName, profile string, keyring linux.LinuxKeyring) (string, string, linux.LinuxKeyring) {
187 | 	if !utils.SupportedBrowsers[browserName] {
188 | 		log.Fatal("browser " + browserName + " not supported")
189 | 	}
190 | 	if !linux.LinuxKeyringNames[keyring] {
191 | 		log.Fatal(fmt.Sprintf("keyring %d not supported", keyring))
192 | 	}
193 | 	if profile != "" {
194 | 		sbsPath, err := filepath.Abs(profile)
195 | 		if err != nil {
196 | 			profile = sbsPath
197 | 		}
198 | 	}
199 | 	return browserName, profile, keyring
200 | }
201 | 
202 | type CookieDecryptor interface {
203 | 	Decrypt(encrypted []byte) ([]byte, error)
204 | }
205 | 
206 | func openDataBaseCopy(dbPath, tmpdir string) (*sql.DB, error) {
207 | 	cpPath := path.Join(tmpdir, "temporary.sqlite")
208 | 	data, err := ioutil.ReadFile(dbPath)
209 | 	if err != nil {
210 | 		return nil, fmt.Errorf("could not read database file: %w", err)
211 | 	}
212 | 
213 | 	err = ioutil.WriteFile(cpPath, data, 0644)
214 | 	if err != nil {
215 | 		return nil, fmt.Errorf("could not write database file: %w", err)
216 | 	}
217 | 	return sql.Open("sqlite3", cpPath)
218 | }
219 | 
220 | func processChromeCookie(decryptor CookieDecryptor, host_key, name, value, encryptedValue, path string, expires_utc, is_secure int) (isEncrypted bool, cookie *http.Cookie, err error) {
221 | 	isEncrypted = value == "" && encryptedValue != ""
222 | 	if isEncrypted {
223 | 		decryptedValue, err := decryptor.Decrypt([]byte(encryptedValue))
224 | 		if err != nil {
225 | 			return false, nil, fmt.Errorf("could not Decrypt cookie: %w", err)
226 | 		}
227 | 		value = string(decryptedValue)
228 | 	}
229 | 	cookie = &http.Cookie{
230 | 		Name:     name,
231 | 		Value:    value,
232 | 		Path:     path,
233 | 		Domain:   host_key,
234 | 		Secure:   is_secure == 1,
235 | 		HttpOnly: true,
236 | 		MaxAge:   expires_utc - int(time.Now().Unix()),
237 | 	}
238 | 	return
239 | }
240 | 


--------------------------------------------------------------------------------
/main/cookie/firefox/firefox.go:
--------------------------------------------------------------------------------
 1 | package firefox
 2 | 
 3 | import (
 4 | 	"os"
 5 | 	"path/filepath"
 6 | 	"runtime"
 7 | )
 8 | 
 9 | func BrowserDir() string {
10 | 	if runtime.GOOS == "windows" {
11 | 		appDataRoaming := os.Getenv("APPDATA")
12 | 		return filepath.FromSlash(filepath.Join(appDataRoaming, "Mozilla", "Firefox", "Profiles"))
13 | 	} else if runtime.GOOS == "darwin" {
14 | 		panic("TODO: implement darwin")
15 | 	}
16 | 	return filepath.FromSlash(os.ExpandEnv("~/.mozilla/firefox"))
17 | }
18 | 


--------------------------------------------------------------------------------
/main/cookie/utils/helper.go:
--------------------------------------------------------------------------------
 1 | package utils
 2 | 
 3 | import (
 4 | 	"log"
 5 | 	"os"
 6 | 	"path/filepath"
 7 | )
 8 | 
 9 | func FindMostRecentlyUsedFile(root, filename string) string {
10 | 	i := 0
11 | 	paths := make([]string, 0)
12 | 
13 | 	filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
14 | 		i++
15 | 		if info != nil && info.Name() == filename {
16 | 			paths = append(paths, path)
17 | 		}
18 | 		return nil
19 | 	})
20 | 
21 | 	log.Println("searched", i, "files")
22 | 
23 | 	if len(paths) == 0 {
24 | 		return ""
25 | 	}
26 | 
27 | 	maxPath := paths[0]
28 | 	info, _ := os.Lstat(maxPath)
29 | 	maxModTime := info.ModTime()
30 | 	for _, path := range paths[1:] {
31 | 		info, _ = os.Lstat(maxPath)
32 | 		modTime := info.ModTime()
33 | 		if modTime.After(maxModTime) {
34 | 			maxPath = path
35 | 			maxModTime = modTime
36 | 		}
37 | 	}
38 | 	return maxPath
39 | }
40 | 
41 | func ConfigHome() string {
42 | 	if xdgConfigHome := os.Getenv("XDG_CONFIG_HOME"); xdgConfigHome != "" {
43 | 		return xdgConfigHome
44 | 	}
45 | 	return filepath.Join(os.Getenv("HOME"), ".config")
46 | }
47 | 


--------------------------------------------------------------------------------
/main/cookie/utils/linux/env.go:
--------------------------------------------------------------------------------
  1 | package linux
  2 | 
  3 | import (
  4 | 	"errors"
  5 | 	secret_service "github.com/zalando/go-keyring/secret_service"
  6 | 	"log"
  7 | 	"os"
  8 | 	"os/exec"
  9 | 	"strings"
 10 | )
 11 | 
 12 | type LinuxDesktopEnvironment int
 13 | 
 14 | const (
 15 | 	Other LinuxDesktopEnvironment = iota
 16 | 	Unity
 17 | 	Gnome
 18 | 	Cinnamon
 19 | 	Kde
 20 | 	Pantheon
 21 | 	Xfce
 22 | )
 23 | 
 24 | func GetLinuxDesktopEnvironment() LinuxDesktopEnvironment {
 25 | 	xdgCurrentDesktop := os.Getenv("XDG_CURRENT_DESKTOP")
 26 | 	desktopSession := os.Getenv("DESKTOP_SESSION")
 27 | 
 28 | 	if xdgCurrentDesktop != "" {
 29 | 		splits := strings.Split(xdgCurrentDesktop, ":")
 30 | 		xdgCurrentDesktop = splits[0]
 31 | 		xdgCurrentDesktop = strings.TrimSpace(xdgCurrentDesktop)
 32 | 
 33 | 		if xdgCurrentDesktop == "Unity" {
 34 | 			if desktopSession != "" && strings.Contains(desktopSession, "gnome-fallback") {
 35 | 				return Gnome
 36 | 			}
 37 | 			return Unity
 38 | 		} else if xdgCurrentDesktop == "GNOME" {
 39 | 			return Gnome
 40 | 		} else if xdgCurrentDesktop == "X-Cinnamon" {
 41 | 			return Cinnamon
 42 | 		} else if xdgCurrentDesktop == "KDE" {
 43 | 			return Kde
 44 | 		} else if xdgCurrentDesktop == "Pantheon" {
 45 | 			return Pantheon
 46 | 		} else if xdgCurrentDesktop == "XFCE" {
 47 | 			return Xfce
 48 | 		}
 49 | 	} else if desktopSession != "" {
 50 | 		if desktopSession == "mate" || desktopSession == "gnome" {
 51 | 			return Gnome
 52 | 		} else if strings.Contains(desktopSession, "kde") {
 53 | 			return Kde
 54 | 		} else if strings.Contains(desktopSession, "xfce") {
 55 | 			return Xfce
 56 | 		}
 57 | 	} else {
 58 | 		if os.Getenv("GNOME_DESKTOP_SESSION_ID") != "" {
 59 | 			return Gnome
 60 | 		} else if os.Getenv("KDE_FULL_SESSION") != "" {
 61 | 			return Kde
 62 | 		}
 63 | 	}
 64 | 
 65 | 	return Other
 66 | }
 67 | 
 68 | type LinuxKeyring int
 69 | 
 70 | const (
 71 | 	// LinuxChromeCookieDecryptor is a decryptor for Linux Chrome cookies
 72 | 	KWALLET LinuxKeyring = iota
 73 | 	GNOMEKEYRING
 74 | 	BASICTEXT
 75 | )
 76 | 
 77 | var LinuxKeyringNames = map[LinuxKeyring]bool{
 78 | 	KWALLET:      true,
 79 | 	GNOMEKEYRING: true,
 80 | 	BASICTEXT:    true,
 81 | }
 82 | 
 83 | func ChooseLinuxKeyring() LinuxKeyring {
 84 | 	var keyring LinuxKeyring
 85 | 	env := GetLinuxDesktopEnvironment()
 86 | 	if env == Kde {
 87 | 		keyring = KWALLET
 88 | 	} else if env == Other {
 89 | 		keyring = BASICTEXT
 90 | 	} else {
 91 | 		keyring = GNOMEKEYRING
 92 | 	}
 93 | 	return keyring
 94 | }
 95 | 
 96 | func getKwalletNetworkWallet() string {
 97 | 	// https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/kwallet_dbus.cc
 98 | 	// KWalletDBus::NetworkWallet
 99 | 	// https://api.kde.org/frameworks/kwallet/html/classKWallet_1_1Wallet.html
100 | 	// Wallet::NetworkWallet
101 | 	defaultWallet := "kdewallet"
102 | 	out, err := exec.Command("dbus-send", "--session", "--print-reply=literal",
103 | 		"--dest=org.kde.kwalletd5",
104 | 		"/modules/kwalletd5",
105 | 		"org.kde.KWallet.networkWallet").Output()
106 | 	if err != nil {
107 | 		log.Println("failed to read NetworkWallet")
108 | 		return defaultWallet
109 | 	}
110 | 	log.Println("NetworkWallet =", string(out))
111 | 	return strings.TrimSpace(string(out))
112 | }
113 | 
114 | func GetKWalletPassword(browserKeyringName string) ([]byte, error) {
115 | 	log.Printf("GetKWalletPassword(%s)", browserKeyringName)
116 | 
117 | 	networkWallet := getKwalletNetworkWallet()
118 | 	out, err := exec.Command("kwallet-query", "--read-password", browserKeyringName+" Safe Storage", "--folder", browserKeyringName+" Keys", networkWallet).Output()
119 | 	if err != nil {
120 | 		return nil, errors.New("failed to read password")
121 | 	}
122 | 	log.Println("password =", string(out))
123 | 	return []byte(strings.TrimSpace(string(out))), nil
124 | }
125 | 
126 | func GetGnomeKeyringPassword(browserKeyringName string) ([]byte, error) {
127 | 	svc, err := secret_service.NewSecretService()
128 | 	if err != nil {
129 | 		return nil, errors.New("failed to create secret service")
130 | 	}
131 | 	collection := svc.GetLoginCollection()
132 | 	if err = svc.Unlock(collection.Path()); err != nil {
133 | 		return nil, errors.New("failed to unlock login collection")
134 | 	}
135 | 
136 | 	items, err := svc.SearchItems(collection, map[string]string{
137 | 		"application": browserKeyringName,
138 | 	})
139 | 	if err != nil {
140 | 		return nil, errors.New("failed to search items")
141 | 	}
142 | 	if len(items) == 0 {
143 | 		return nil, errors.New("no items found")
144 | 	}
145 | 	item := items[0]
146 | 
147 | 	session, err := svc.OpenSession()
148 | 	if err != nil {
149 | 		return nil, err
150 | 	}
151 | 	defer svc.Close(session)
152 | 
153 | 	secret, err := svc.GetSecret(item, session.Path())
154 | 	if err != nil {
155 | 		return nil, err
156 | 	}
157 | 	return secret.Value, nil
158 | }
159 | 


--------------------------------------------------------------------------------
/main/cookie/utils/map.go:
--------------------------------------------------------------------------------
 1 | package utils
 2 | 
 3 | import "runtime"
 4 | 
 5 | var ChromiumBasedBrowsers = map[string]bool{
 6 | 	"brave":    true,
 7 | 	"chrome":   true,
 8 | 	"chromium": true,
 9 | 	"edge":     true,
10 | 	"opera":    true,
11 | 	"vivaldi":  true,
12 | }
13 | 
14 | var SupportedBrowsers = map[string]bool{
15 | 	"brave":    true,
16 | 	"chrome":   true,
17 | 	"chromium": true,
18 | 	"edge":     true,
19 | 	"opera":    true,
20 | 	"vivaldi":  true,
21 | 	"firefox":  true,
22 | 	"safari":   true,
23 | }
24 | 
25 | var WindowsBrowserDir = map[string]string{
26 | 	"brave":    "BraveSoftware\\Brave-Browser\\User Data",
27 | 	"chrome":   "Google\\Chrome\\User Data",
28 | 	"chromium": "Chromium\\User Data",
29 | 	"edge":     "Microsoft\\Edge\\User Data",
30 | 	"opera":    "Opera Software\\Opera Stable",
31 | 	"vivaldi":  "Vivaldi\\User Data",
32 | }
33 | 
34 | var DarwinBrowserDir = map[string]string{
35 | 	"brave":    "BraveSoftware\\Brave-Browser",
36 | 	"chrome":   "Google\\Chrome",
37 | 	"chromium": "Chromium",
38 | 	"edge":     "Microsoft Edge",
39 | 	"opera":    "com.operasoftware.Opera",
40 | 	"vivaldi":  "Vivaldi",
41 | }
42 | 
43 | var LinuxBrowserDir = map[string]string{
44 | 	"brave":    "BraveSoftware\\Brave-Browser",
45 | 	"chrome":   "google-chrome",
46 | 	"chromium": "chromium",
47 | 	"edge":     "microsoft-edge",
48 | 	"opera":    "opera",
49 | 	"vivaldi":  "vivaldi",
50 | }
51 | 
52 | func KeyingName(browserName string) string {
53 | 	switch browserName {
54 | 	case "brave":
55 | 		return "Brave"
56 | 	case "chrome":
57 | 		return "Chrome"
58 | 	case "chromium":
59 | 		return "Chromium"
60 | 	case "edge":
61 | 		if runtime.GOOS == "darwin" {
62 | 			return "Chromium"
63 | 		} else {
64 | 			return "Microsoft Edge"
65 | 		}
66 | 	case "opera":
67 | 		if runtime.GOOS == "darwin" {
68 | 			return "Chromium"
69 | 		} else {
70 | 			return "Opera"
71 | 		}
72 | 	case "vivaldi":
73 | 		if runtime.GOOS == "darwin" {
74 | 			return "Chrome"
75 | 		} else {
76 | 			return "Chromium"
77 | 		}
78 | 	default:
79 | 		panic("KeyingName called with unsupported browser " + browserName)
80 | 	}
81 | }
82 | 


--------------------------------------------------------------------------------
/main/cookie/utils/windows_dpapi.go:
--------------------------------------------------------------------------------
 1 | //go:build windows
 2 | 
 3 | package utils
 4 | 
 5 | import (
 6 | 	"fmt"
 7 | 	"golang.org/x/sys/windows"
 8 | 	"unsafe"
 9 | )
10 | 
11 | func DecryptWindowsDpapi(ciphertext []byte) ([]byte, error) {
12 | 	var blobIn, blobOut windows.DataBlob
13 | 	blobIn.Size = uint32(len(ciphertext))
14 | 	blobIn.Data = &ciphertext[0]
15 | 
16 | 	err := windows.CryptUnprotectData(&blobIn, nil, nil, 0, nil, 0, &blobOut)
17 | 	if err != nil {
18 | 		return nil, fmt.Errorf("CryptUnprotectData failed: %v", err)
19 | 	}
20 | 
21 | 	d := make([]byte, blobOut.Size)
22 | 	copy(d, (*[1 << 30]byte)(unsafe.Pointer(blobOut.Data))[:])
23 | 
24 | 	_, err = windows.LocalFree(windows.Handle(unsafe.Pointer(blobOut.Data)))
25 | 	if err != nil {
26 | 		return nil, fmt.Errorf("LocalFree failed: %v", err)
27 | 	}
28 | 
29 | 	return d, nil
30 | }
31 | 


--------------------------------------------------------------------------------
/main/cookies_unix.go:
--------------------------------------------------------------------------------
 1 | //go:build linux
 2 | 
 3 | package main
 4 | 
 5 | import (
 6 | 	"log"
 7 | 	"net/http"
 8 | 	"os"
 9 | 	"runtime"
10 | )
11 | 
12 | func getCookies(s string) []*http.Cookie {
13 | 	if cookieFile != "" {
14 | 		_, err := os.Stat(cookieFile)
15 | 		if err == nil {
16 | 			log.Printf("load cookie from %s", cookieFile)
17 | 			return parasCookieFile(cookieFile)
18 | 		} else {
19 | 			log.Printf("cookie file %s not found", cookieFile)
20 | 		}
21 | 	}
22 | 	log.Fatalf("Cookies detected, but not supported on %s", runtime.GOOS)
23 | 	return nil
24 | }
25 | 


--------------------------------------------------------------------------------
/main/cookies_windows.go:
--------------------------------------------------------------------------------
 1 | //go:build windows && !no_cookies_detection
 2 | 
 3 | package main
 4 | 
 5 | import (
 6 | 	"fmt"
 7 | 	"log"
 8 | 	"net/http"
 9 | 	"os"
10 | 	"runtime"
11 | 
12 | 	"github.com/elvis972602/kemono-scraper/main/cookie"
13 | )
14 | 
15 | func getCookies(s string) []*http.Cookie {
16 | 	if cookieFile != "" {
17 | 		_, err := os.Stat(cookieFile)
18 | 		if err == nil {
19 | 			log.Printf("load cookie from %s", cookieFile)
20 | 			return parasCookieFile(cookieFile)
21 | 		} else {
22 | 			log.Printf("cookie file %s not found", cookieFile)
23 | 		}
24 | 	}
25 | 	if runtime.GOOS != "windows" {
26 | 		log.Fatalf("Cookies detected, but not supported on %s", runtime.GOOS)
27 | 	}
28 | 
29 | 	var cookies []*http.Cookie
30 | 	c := cookie.NewCookies()
31 | 	err := c.ReadCookies(cookieBrowser, "", 0)
32 | 	if err != nil {
33 | 		log.Fatalf("Error reading cookies: %s", err)
34 | 	}
35 | 	cs := c.GetCookies()
36 | 	for _, v := range cs {
37 | 		if v.Domain == fmt.Sprintf("%s.su", s) || v.Domain == fmt.Sprintf(".%s.su", s) {
38 | 			cookies = append(cookies, v)
39 | 		}
40 | 	}
41 | 	return cookies
42 | }
43 | 


--------------------------------------------------------------------------------
/main/cookies_windows_no_cookies_detected.go:
--------------------------------------------------------------------------------
 1 | //go:build windows && no_cookies_detection
 2 | 
 3 | package main
 4 | 
 5 | import (
 6 | 	"log"
 7 | 	"net/http"
 8 | 	"os"
 9 | )
10 | 
11 | func getCookies(s string) []*http.Cookie {
12 | 	if cookieFile != "" {
13 | 		_, err := os.Stat(cookieFile)
14 | 		if err == nil {
15 | 			log.Printf("load cookie from %s", cookieFile)
16 | 			return parasCookieFile(cookieFile)
17 | 		} else {
18 | 			log.Printf("cookie file %s not found", cookieFile)
19 | 		}
20 | 	}
21 | 	return []*http.Cookie{}
22 | }
23 | 


--------------------------------------------------------------------------------
/main/main.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"bufio"
  5 | 	"context"
  6 | 	"crypto/tls"
  7 | 	"encoding/json"
  8 | 	"flag"
  9 | 	"fmt"
 10 | 	"log"
 11 | 	"net/http"
 12 | 	"net/url"
 13 | 	"os"
 14 | 	"path/filepath"
 15 | 	"regexp"
 16 | 	"strconv"
 17 | 	"strings"
 18 | 	tmpl "text/template"
 19 | 	"time"
 20 | 
 21 | 	"github.com/elvis972602/kemono-scraper/downloader"
 22 | 	"github.com/elvis972602/kemono-scraper/kemono"
 23 | 	"github.com/elvis972602/kemono-scraper/term"
 24 | 	"github.com/elvis972602/kemono-scraper/utils"
 25 | 	"github.com/mattn/go-colorable"
 26 | )
 27 | 
 28 | const (
 29 | 	Kemono = "kemono"
 30 | 	Coomer = "coomer"
 31 | )
 32 | 
 33 | var (
 34 | 	creators               []string
 35 | 	links                  []string
 36 | 	options                map[string][]kemono.Option
 37 | 	sharedOptions          []kemono.Option
 38 | 	downloaderOptions      []downloader.DownloadOption
 39 | 	hasLink                bool
 40 | 	s, srv, userId, postId string
 41 | 	// map[<Creator>][]<postId>
 42 | 	idFilter map[string]map[kemono.Creator][]string
 43 | )
 44 | 
 45 | func init() {
 46 | 	idFilter = make(map[string]map[kemono.Creator][]string)
 47 | 	idFilter[Kemono] = make(map[kemono.Creator][]string)
 48 | 	idFilter[Coomer] = make(map[kemono.Creator][]string)
 49 | 	options = make(map[string][]kemono.Option)
 50 | }
 51 | 
 52 | func main() {
 53 | 	flag.Parse()
 54 | 	setPassedFlags()
 55 | 
 56 | 	if help {
 57 | 		PrintDefaults()
 58 | 		return
 59 | 	}
 60 | 
 61 | 	setFlag()
 62 | 
 63 | 	if creator != "" {
 64 | 		creatorComponents := strings.Split(creator, ",")
 65 | 		for _, c := range creatorComponents {
 66 | 			c = strings.TrimSpace(c)
 67 | 			if c == "" {
 68 | 				continue
 69 | 			}
 70 | 			creators = append(creators, c)
 71 | 		}
 72 | 	}
 73 | 
 74 | 	if link != "" {
 75 | 		linkComponents := strings.Split(link, ",")
 76 | 		for _, l := range linkComponents {
 77 | 			l = strings.TrimSpace(l)
 78 | 			if l == "" {
 79 | 				continue
 80 | 			}
 81 | 			links = append(links, l)
 82 | 		}
 83 | 	}
 84 | 	downloaderOptions = append(downloaderOptions, downloader.Async(async))
 85 | 
 86 | 	if len(links) > 0 {
 87 | 		hasLink = true
 88 | 		users := make(map[string][]string)
 89 | 		for _, l := range links {
 90 | 			s, srv, userId, postId = parasLink(l)
 91 | 
 92 | 			cs := kemono.NewCreator(srv, userId)
 93 | 			users[s] = append(users[s], srv, userId)
 94 | 			if postId != "" {
 95 | 				idFilter[s][cs] = append(idFilter[s][cs], postId)
 96 | 			}
 97 | 			options[s] = append(options[s],
 98 | 				kemono.WithUsersPair(users[s]...),
 99 | 			)
100 | 		}
101 | 	}
102 | 
103 | 	// check creator
104 | 	if len(creators) > 0 {
105 | 		for _, c := range creators {
106 | 			creatorComponents := strings.Split(c, ":")
107 | 			if len(creatorComponents) != 2 {
108 | 				log.Fatalf("invalid creator %s", c)
109 | 			}
110 | 			s, ok := kemono.SiteMap[creatorComponents[0]]
111 | 			if !ok {
112 | 				log.Fatalf("invalid creator %s", c)
113 | 			}
114 | 			options[s] = append(options[s], kemono.WithUsersPair(creatorComponents[0], creatorComponents[1]))
115 | 
116 | 		}
117 | 	} else if !hasLink && !favoriteCreator && !favoritePost {
118 | 		log.Fatal("creator is empty")
119 | 	}
120 | 
121 | 	if favoriteCreator || favoritePost {
122 | 		if site == "" {
123 | 			log.Fatal("fav-site is empty")
124 | 		}
125 | 		siteComponents := strings.Split(site, ",")
126 | 		for _, siteComponent := range siteComponents {
127 | 			siteComponent = strings.TrimSpace(siteComponent)
128 | 			if siteComponent == "" {
129 | 				continue
130 | 			}
131 | 			cs := getCookies(siteComponent)
132 | 			if len(cs) == 0 {
133 | 				log.Fatal("cookie is empty")
134 | 			}
135 | 			if favoriteCreator {
136 | 				for _, c := range fetchFavoriteCreators(siteComponent, cs) {
137 | 					options[siteComponent] = append(options[siteComponent], kemono.WithUsersPair(c.Service, c.Id))
138 | 				}
139 | 			}
140 | 			if favoritePost {
141 | 				for _, c := range fetchFavoritePosts(siteComponent, cs) {
142 | 					u := kemono.NewCreator(c.Service, c.User)
143 | 					options[siteComponent] = append(options[siteComponent], kemono.WithUsers(u))
144 | 					idFilter[siteComponent][u] = append(idFilter[siteComponent][u], c.Id)
145 | 				}
146 | 			}
147 | 		}
148 | 	}
149 | 
150 | 	for k, v := range idFilter {
151 | 		for u, ids := range v {
152 | 			if len(ids) > 0 {
153 | 				options[k] = append(options[k], kemono.WithUserPostFilter(u,
154 | 					kemono.IdFilter(ids...),
155 | 				))
156 | 			}
157 | 		}
158 | 	}
159 | 
160 | 	// banner
161 | 	sharedOptions = append(sharedOptions, kemono.WithBanner(banner))
162 | 
163 | 	// overwrite
164 | 	downloaderOptions = append(downloaderOptions, downloader.OverWrite(overwrite))
165 | 
166 | 	// check first
167 | 	if first != 0 {
168 | 		sharedOptions = append(sharedOptions, kemono.WithPostFilter(
169 | 			kemono.NumbFilter(func(i int) bool {
170 | 				if i <= first {
171 | 					return true
172 | 				}
173 | 				return false
174 | 			}),
175 | 		))
176 | 	}
177 | 
178 | 	// check last
179 | 	if last != 0 {
180 | 		sharedOptions = append(sharedOptions, kemono.WithPostFilter(
181 | 			kemono.NumbFilter(func(i int) bool {
182 | 				if i >= last {
183 | 					return true
184 | 				}
185 | 				return false
186 | 			}),
187 | 		))
188 | 	}
189 | 
190 | 	if date != 0 {
191 | 		t := parasData(strconv.Itoa(date))
192 | 		sharedOptions = append(sharedOptions, kemono.WithPostFilter(
193 | 			kemono.ReleaseDateFilter(t.Add(-1), t.Add(24*time.Hour-1)),
194 | 		))
195 | 	}
196 | 
197 | 	if dateBefore != 0 {
198 | 		t := parasData(strconv.Itoa(dateBefore))
199 | 		sharedOptions = append(sharedOptions, kemono.WithPostFilter(
200 | 			kemono.ReleaseDateBeforeFilter(t),
201 | 		))
202 | 	}
203 | 
204 | 	if dateAfter != 0 {
205 | 		t := parasData(strconv.Itoa(dateAfter))
206 | 		sharedOptions = append(sharedOptions, kemono.WithPostFilter(
207 | 			kemono.ReleaseDateAfterFilter(t),
208 | 		))
209 | 	}
210 | 
211 | 	if update != 0 {
212 | 		t := parasData(strconv.Itoa(update))
213 | 		sharedOptions = append(sharedOptions, kemono.WithPostFilter(
214 | 			kemono.EditDateFilter(t.Add(-1), t.Add(24*time.Hour-1)),
215 | 		))
216 | 	}
217 | 
218 | 	if updateBefore != 0 {
219 | 		t := parasData(strconv.Itoa(updateBefore))
220 | 		sharedOptions = append(sharedOptions, kemono.WithPostFilter(
221 | 			kemono.EditDateBeforeFilter(t),
222 | 		))
223 | 	}
224 | 
225 | 	if updateAfter != 0 {
226 | 		t := parasData(strconv.Itoa(updateAfter))
227 | 		sharedOptions = append(sharedOptions, kemono.WithPostFilter(
228 | 			kemono.EditDateAfterFilter(t),
229 | 		))
230 | 	}
231 | 
232 | 	// check extensionOnly
233 | 	if extensionOnly != "" {
234 | 		extensionComponents := strings.Split(extensionOnly, ",")
235 | 		// check extension has dot
236 | 		for i, extension := range extensionComponents {
237 | 			extension = strings.TrimSpace(extension)
238 | 			if !strings.HasPrefix(extension, ".") {
239 | 				extensionComponents[i] = "." + extension
240 | 			}
241 | 		}
242 | 		sharedOptions = append(sharedOptions, kemono.WithAttachmentFilter(
243 | 			kemono.ExtensionFilter(extensionComponents...),
244 | 		))
245 | 	}
246 | 
247 | 	// check extensionExclude
248 | 	if extensionExclude != "" {
249 | 		extensionComponents := strings.Split(extensionExclude, ",")
250 | 		// check extension has dot
251 | 		for i, extension := range extensionComponents {
252 | 			extension = strings.TrimSpace(extension)
253 | 			if !strings.HasPrefix(extension, ".") {
254 | 				extensionComponents[i] = "." + extension
255 | 			}
256 | 		}
257 | 		sharedOptions = append(sharedOptions, kemono.WithAttachmentFilter(
258 | 			kemono.ExtensionExcludeFilter(extensionComponents...),
259 | 		))
260 | 	}
261 | 
262 | 	if output == "" {
263 | 		output = "./download"
264 | 	}
265 | 
266 | 	if template == "" {
267 | 		if imageTemplate != "" || videoTemplate != "" || audioTemplate != "" || archiveTemplate != "" {
268 | 			log.Printf("to use image/video/audio/archive template, you must set template first")
269 | 		}
270 | 		var t *tmpl.Template
271 | 		defaultTemp, err := LoadPathTmpl(TmplDefault, output)
272 | 		if err != nil {
273 | 			log.Fatalf("load template failed: %s", err)
274 | 		}
275 | 		if nameRuleOnlyIndex {
276 | 			t, err = LoadPathTmpl(TmplIndexNumber, output)
277 | 			if err != nil {
278 | 				log.Fatalf("load template failed: %s", err)
279 | 			}
280 | 		} else if withPrefixNumber {
281 | 			t, err = LoadPathTmpl(TmplWithPrefixNumber, output)
282 | 			if err != nil {
283 | 				log.Fatalf("load template failed: %s", err)
284 | 			}
285 | 		} else {
286 | 			t = defaultTemp
287 | 		}
288 | 		downloaderOptions = append(downloaderOptions, downloader.SavePath(func(creator kemono.Creator, post kemono.Post, i int, attachment kemono.File) string {
289 | 			ext := filepath.Ext(attachment.Name)
290 | 			filehash := filepath.Base(attachment.Path)[0 : len(filepath.Base(attachment.Path))-len(filepath.Ext(attachment.Path))]
291 | 			filename := attachment.Name[0 : len(attachment.Name)-len(ext)]
292 | 			// use Path extension if Name extension is empty
293 | 			if ext == "" {
294 | 				ext = filepath.Ext(attachment.Path)
295 | 			}
296 | 			pathConfig := &PathConfig{
297 | 				Service:   creator.Service,
298 | 				Creator:   utils.ValidDirectoryName(creator.Name),
299 | 				Post:      utils.ValidDirectoryName(DirectoryName(post)),
300 | 				Index:     i,
301 | 				Filename:  utils.ValidDirectoryName(filename),
302 | 				Filehash:  utils.ValidDirectoryName(filehash),
303 | 				Extension: ext,
304 | 			}
305 | 			if ext == ".zip" || ext == ".rar" || ext == ".7z" {
306 | 				return ExecutePathTmpl(defaultTemp, pathConfig)
307 | 			} else {
308 | 				return ExecutePathTmpl(t, pathConfig)
309 | 			}
310 | 		}))
311 | 	} else {
312 | 		tmplCache := NewTmplCache()
313 | 		tmplCache.init()
314 | 
315 | 		downloaderOptions = append(downloaderOptions, downloader.SavePath(func(creator kemono.Creator, post kemono.Post, i int, attachment kemono.File) string {
316 | 			ext := filepath.Ext(attachment.Name)
317 | 			filehash := filepath.Base(attachment.Path)[0 : len(filepath.Base(attachment.Path))-len(filepath.Ext(attachment.Path))]
318 | 			filename := attachment.Name[0 : len(attachment.Name)-len(ext)]
319 | 			// use Path extension if Name extension is empty
320 | 			if ext == "" {
321 | 				ext = filepath.Ext(attachment.Path)
322 | 			}
323 | 			pathConfig := &PathConfig{
324 | 				Service:   creator.Service,
325 | 				Creator:   utils.ValidDirectoryName(creator.Name),
326 | 				Post:      utils.ValidDirectoryName(DirectoryName(post)),
327 | 				Index:     i,
328 | 				Filename:  utils.ValidDirectoryName(filename),
329 | 				Filehash:  utils.ValidDirectoryName(filehash),
330 | 				Extension: ext,
331 | 			}
332 | 			return tmplCache.Execute(getTyp(ext), pathConfig)
333 | 		}))
334 | 	}
335 | 
336 | 	downloaderOptions = append(downloaderOptions, downloader.WithContent(content))
337 | 
338 | 	if maxSize != "" {
339 | 		size := utils.ParseSize(maxSize)
340 | 		downloaderOptions = append(downloaderOptions, downloader.MaxSize(size))
341 | 	}
342 | 
343 | 	if minSize != "" {
344 | 		size := utils.ParseSize(minSize)
345 | 		downloaderOptions = append(downloaderOptions, downloader.MinSize(size))
346 | 	}
347 | 
348 | 	if downloadTimeout <= 0 {
349 | 		log.Fatalf("invalid download timeout %d", downloadTimeout)
350 | 	} else {
351 | 		downloaderOptions = append(downloaderOptions, downloader.Timeout(time.Duration(downloadTimeout)*time.Second))
352 | 	}
353 | 
354 | 	if retry < 0 {
355 | 		log.Fatalf("retry must be greater than 0")
356 | 	} else {
357 | 		downloaderOptions = append(downloaderOptions, downloader.Retry(retry))
358 | 		sharedOptions = append(sharedOptions, kemono.SetRetry(retry))
359 | 	}
360 | 
361 | 	if retryInterval < 0 {
362 | 		log.Fatalf("retry interval must be greater than 0")
363 | 	} else {
364 | 		downloaderOptions = append(downloaderOptions, downloader.RetryInterval(time.Duration(retryInterval)*time.Second))
365 | 		sharedOptions = append(sharedOptions, kemono.SetRetryInterval(time.Duration(retryInterval)*time.Second))
366 | 	}
367 | 
368 | 	// check maxDownloadGoroutine
369 | 	if maxDownloadParallel <= 0 {
370 | 		log.Fatalf("maxDownloadParallel must be greater than 0")
371 | 	} else {
372 | 		downloaderOptions = append(downloaderOptions, downloader.MaxConcurrent(maxDownloadParallel))
373 | 	}
374 | 
375 | 	if rateLimit <= 0 {
376 | 		log.Fatalf("rate limit must be greater than 0")
377 | 	} else {
378 | 		downloaderOptions = append(downloaderOptions, downloader.RateLimit(rateLimit))
379 | 	}
380 | 
381 | 	if proxy != "" {
382 | 		downloaderOptions = append(downloaderOptions, downloader.WithProxy(proxy))
383 | 	}
384 | 
385 | 	ctx := context.Background()
386 | 	defer ctx.Done()
387 | 
388 | 	terminal := term.NewTerminal(colorable.NewColorableStdout(), colorable.NewColorableStderr(), false)
389 | 	go terminal.Run(ctx)
390 | 
391 | 	downloaderOptions = append(downloaderOptions, downloader.SetLog(terminal))
392 | 	sharedOptions = append(sharedOptions, kemono.SetLog(terminal))
393 | 
394 | 	var (
395 | 		KKemono          *kemono.Kemono
396 | 		KCoomer          *kemono.Kemono
397 | 		KemonoDownloader kemono.Downloader
398 | 		CoomerDownloader kemono.Downloader
399 | 		k, c             bool
400 | 	)
401 | 
402 | 	// Kemono
403 | 	if len(options[Kemono]) > 0 {
404 | 		k = true
405 | 		options[Kemono] = append(options[Kemono], sharedOptions...)
406 | 		options[Kemono] = append(options[Kemono], kemono.WithDomain("kemono"))
407 | 		downloaderOptions = append(downloaderOptions, downloader.BaseURL("https://kemono.su"))
408 | 		token, err := utils.GenerateToken(16)
409 | 		if err != nil {
410 | 			log.Fatalf("generate token failed: %s", err)
411 | 		}
412 | 		downloaderOptions = append(downloaderOptions, downloader.WithCookie([]*http.Cookie{
413 | 			{
414 | 				Name:   "__ddg2",
415 | 				Value:  token,
416 | 				Path:   "/",
417 | 				Domain: ".kemono.su",
418 | 				Secure: false,
419 | 			},
420 | 		}))
421 | 		downloaderOptions = append(downloaderOptions, downloader.WithHeader(downloader.Header{
422 | 			"Host":                      "kemono.su",
423 | 			"User-Agent":                downloader.UserAgent,
424 | 			"Referer":                   "https://kemono.su",
425 | 			"Accept":                    downloader.Accept,
426 | 			"Accept-Language":           downloader.AcceptLanguage,
427 | 			"Accept-Encoding":           downloader.AcceptEncoding,
428 | 			"Sec-Ch-Ua":                 downloader.SecChUA,
429 | 			"Sec-Ch-Ua-Mobile":          downloader.SecChUAMobile,
430 | 			"Sec-Fetch-Dest":            downloader.SecFetchDest,
431 | 			"Sec-Fetch-Mode":            downloader.SecFetchMode,
432 | 			"Sec-Fetch-Site":            downloader.SecFetchSite,
433 | 			"Sec-Fetch-User":            downloader.SecFetchUser,
434 | 			"Upgrade-Insecure-Requests": downloader.UpgradeInsecureRequests,
435 | 			"Connection":                "keep-alive",
436 | 		}))
437 | 		KemonoDownloader = downloader.NewDownloader(downloaderOptions...)
438 | 		options[Kemono] = append(options[Kemono], kemono.SetDownloader(KemonoDownloader))
439 | 		KKemono = kemono.NewKemono(options[Kemono]...)
440 | 	}
441 | 	if len(options[Coomer]) > 0 {
442 | 		c = true
443 | 		options[Coomer] = append(options[Coomer], sharedOptions...)
444 | 		options[Coomer] = append(options[Coomer], kemono.WithDomain("coomer"))
445 | 		downloaderOptions = append(downloaderOptions, downloader.BaseURL("https://coomer.su"))
446 | 		token, err := utils.GenerateToken(16)
447 | 		if err != nil {
448 | 			log.Fatalf("generate token failed: %s", err)
449 | 		}
450 | 		downloaderOptions = append(downloaderOptions, downloader.WithCookie([]*http.Cookie{
451 | 			{
452 | 				Name:   "__ddg2",
453 | 				Value:  token,
454 | 				Path:   "/",
455 | 				Domain: ".coomer.su",
456 | 			},
457 | 		}))
458 | 		downloaderOptions = append(downloaderOptions, downloader.WithHeader(downloader.Header{
459 | 			"Host":                      "coomer.su",
460 | 			"User-Agent":                downloader.UserAgent,
461 | 			"Referer":                   "https://coomer.su/",
462 | 			"Accept":                    downloader.Accept,
463 | 			"Accept-Language":           downloader.AcceptLanguage,
464 | 			"Accept-Encoding":           downloader.AcceptEncoding,
465 | 			"Sec-Ch-Ua":                 downloader.SecChUA,
466 | 			"Sec-Ch-Ua-Mobile":          downloader.SecChUAMobile,
467 | 			"Sec-Fetch-Dest":            downloader.SecFetchDest,
468 | 			"Sec-Fetch-Mode":            downloader.SecFetchMode,
469 | 			"Sec-Fetch-Site":            downloader.SecFetchSite,
470 | 			"Sec-Fetch-User":            downloader.SecFetchUser,
471 | 			"Upgrade-Insecure-Requests": downloader.UpgradeInsecureRequests,
472 | 			"Connection":                "keep-alive",
473 | 		}))
474 | 		CoomerDownloader = downloader.NewDownloader(downloaderOptions...)
475 | 		options[Coomer] = append(options[Coomer], kemono.SetDownloader(CoomerDownloader))
476 | 		options[Coomer] = append(options[Coomer], kemono.WithBanner(true))
477 | 		KCoomer = kemono.NewKemono(options[Coomer]...)
478 | 	}
479 | 
480 | 	if k {
481 | 		terminal.Print("Downloading Kemono")
482 | 		err := KKemono.Start()
483 | 		if err != nil {
484 | 			log.Printf("kemono start failed: %s", err)
485 | 		}
486 | 	}
487 | 	if c {
488 | 		terminal.Print("Downloading Coomer")
489 | 		err := KCoomer.Start()
490 | 		if err != nil {
491 | 			log.Printf("coomer start failed: %s", err)
492 | 		}
493 | 	}
494 | }
495 | 
496 | func parasLink(link string) (s, service, userId, postId string) {
497 | 	u, err := url.Parse(link)
498 | 	if err != nil {
499 | 		log.Fatal("invalid url")
500 | 	}
501 | 
502 | 	pattern := `(?i)^(?:.*\.)?(kemono|coomer)\.su$`
503 | 	re := regexp.MustCompile(pattern)
504 | 
505 | 	matchedSubstrings := re.FindStringSubmatch(u.Host)
506 | 
507 | 	if matchedSubstrings == nil {
508 | 		log.Fatal("invalid host:", u.Host)
509 | 	}
510 | 
511 | 	s = matchedSubstrings[1]
512 | 
513 | 	pathComponents := strings.Split(u.Path, "/")
514 | 	if len(pathComponents) != 6 && len(pathComponents) != 4 {
515 | 		log.Fatal("Error splitting host component:", pathComponents, len(pathComponents))
516 | 		return
517 | 	}
518 | 	if len(pathComponents) == 6 {
519 | 		service = pathComponents[1]
520 | 		userId = pathComponents[3]
521 | 		postId = pathComponents[5]
522 | 	} else {
523 | 		service = pathComponents[1]
524 | 		userId = pathComponents[3]
525 | 	}
526 | 
527 | 	return
528 | }
529 | 
530 | func parasData(data string) time.Time {
531 | 	if len(data) != 8 {
532 | 		log.Fatalf("invalid date %s", data)
533 | 	}
534 | 	year, err := strconv.Atoi(data[:4])
535 | 	if err != nil {
536 | 		log.Fatalf("invalid date %s", data)
537 | 	}
538 | 	month, err := strconv.Atoi(data[4:6])
539 | 	if err != nil {
540 | 		log.Fatalf("invalid date %s", data)
541 | 	}
542 | 	day, err := strconv.Atoi(data[6:])
543 | 	if err != nil {
544 | 		log.Fatalf("invalid date %s", data)
545 | 	}
546 | 	return time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.UTC)
547 | }
548 | 
549 | func setFlag() {
550 | 	if !passedFlags["link"] && config["link"] != nil {
551 | 		link = config["link"].(string)
552 | 	}
553 | 	if !passedFlags["fav-site"] && config["fav-site"] != nil {
554 | 		site = config["fav-site"].(string)
555 | 	}
556 | 	if !passedFlags["creator"] && config["creator"] != nil {
557 | 		creator = config["creator"].(string)
558 | 	}
559 | 	if !passedFlags["banner"] && config["banner"] != nil {
560 | 		banner = config["banner"].(bool)
561 | 	}
562 | 	if !passedFlags["overwrite"] && config["overwrite"] != nil {
563 | 		overwrite = config["overwrite"].(bool)
564 | 	}
565 | 	if !passedFlags["first"] && config["first"] != nil {
566 | 		first = config["first"].(int)
567 | 	}
568 | 	if !passedFlags["last"] && config["last"] != nil {
569 | 		last = config["last"].(int)
570 | 	}
571 | 	if !passedFlags["date"] && config["date"] != nil {
572 | 		date = config["date"].(int)
573 | 	}
574 | 	if !passedFlags["date-before"] && config["date-before"] != nil {
575 | 		dateBefore = config["date-before"].(int)
576 | 	}
577 | 	if !passedFlags["date-after"] && config["date-after"] != nil {
578 | 		dateAfter = config["date-after"].(int)
579 | 	}
580 | 	if !passedFlags["update"] && config["update"] != nil {
581 | 		update = config["update"].(int)
582 | 	}
583 | 	if !passedFlags["update-before"] && config["update-before"] != nil {
584 | 		updateBefore = config["update-before"].(int)
585 | 	}
586 | 	if !passedFlags["update-after"] && config["update-after"] != nil {
587 | 		updateAfter = config["update-after"].(int)
588 | 	}
589 | 	if !passedFlags["extension-only"] && config["extension-only"] != nil {
590 | 		extensionOnly = config["extension-only"].(string)
591 | 	}
592 | 	if !passedFlags["extension-exclude"] && config["extension-exclude"] != nil {
593 | 		extensionExclude = config["extension-exclude"].(string)
594 | 	}
595 | 	if !passedFlags["output"] && config["output"] != nil {
596 | 		output = config["output"].(string)
597 | 	}
598 | 	if !passedFlags["template"] && config["template"] != nil {
599 | 		template = config["template"].(string)
600 | 	}
601 | 	if !passedFlags["image-template"] && config["image-template"] != nil {
602 | 		imageTemplate = config["image-template"].(string)
603 | 	}
604 | 	if !passedFlags["audio-template"] && config["audio-template"] != nil {
605 | 		audioTemplate = config["audio-template"].(string)
606 | 	}
607 | 	if !passedFlags["video-template"] && config["video-template"] != nil {
608 | 		videoTemplate = config["video-template"].(string)
609 | 	}
610 | 	if !passedFlags["archive-template"] && config["archive-template"] != nil {
611 | 		archiveTemplate = config["archive-template"].(string)
612 | 	}
613 | 	if !passedFlags["contrnt"] && config["content"] != nil {
614 | 		content = config["content"].(bool)
615 | 	}
616 | 	if !passedFlags["async"] && config["async"] != nil {
617 | 		async = config["async"].(bool)
618 | 	}
619 | 	if !passedFlags["max-size"] && config["max-size"] != nil {
620 | 		maxSize = config["max-size"].(string)
621 | 	}
622 | 	if !passedFlags["min-size"] && config["min-size"] != nil {
623 | 		minSize = config["min-size"].(string)
624 | 	}
625 | 	if !passedFlags["with-prefix-number"] && config["with-prefix-number"] != nil {
626 | 		withPrefixNumber = config["with-prefix-number"].(bool)
627 | 	}
628 | 	if !passedFlags["name-rule-only-index"] && config["name-rule-only-index"] != nil {
629 | 		nameRuleOnlyIndex = config["name-rule-only-index"].(bool)
630 | 	}
631 | 	if !passedFlags["download-timeout"] && config["download-timeout"] != nil {
632 | 		downloadTimeout = config["download-timeout"].(int)
633 | 	}
634 | 
635 | 	if !passedFlags["retry"] && config["retry"] != nil {
636 | 		retry = config["retry"].(int)
637 | 	}
638 | 	if !passedFlags["retry-interval"] && config["retry-interval"] != nil {
639 | 		// check if retry-interval is float64 or int
640 | 		_, ok := config["retry-interval"].(float64)
641 | 		if !ok {
642 | 			retryInterval = float64(config["retry-interval"].(int))
643 | 		} else {
644 | 			retryInterval = config["retry-interval"].(float64)
645 | 		}
646 | 	}
647 | 	if !passedFlags["max-download-parallel"] && config["max-download-parallel"] != nil {
648 | 		maxDownloadParallel = config["max-download-parallel"].(int)
649 | 	}
650 | 	if !passedFlags["rate-limit"] && config["rate-limit"] != nil {
651 | 		rateLimit = config["rate-limit"].(int)
652 | 	}
653 | 	if !passedFlags["proxy"] && config["proxy"] != nil {
654 | 		proxy = config["proxy"].(string)
655 | 	}
656 | 	if !passedFlags["fav-creator"] && config["fav-creator"] != nil {
657 | 		favoriteCreator = config["fav-creator"].(bool)
658 | 	}
659 | 	if !passedFlags["fav-post"] && config["fav-post"] != nil {
660 | 		favoritePost = config["fav-post"].(bool)
661 | 	}
662 | 	if !passedFlags["cookie-browser"] && config["cookie-browser"] != nil {
663 | 		cookieBrowser = config["cookie-browser"].(string)
664 | 	}
665 | 	if !passedFlags["cookie"] && config["cookie"] != nil {
666 | 		cookieFile = config["cookie"].(string)
667 | 	}
668 | }
669 | 
670 | func DirectoryName(p kemono.Post) string {
671 | 	return fmt.Sprintf("[%s] [%s] %s", p.Published.Format("20060102"), p.Id, p.Title)
672 | }
673 | 
674 | func fetchFavoriteCreators(s string, cookie []*http.Cookie) []kemono.FavoriteCreator {
675 | 	log.Printf("fetching favorite creators from %s.su", s)
676 | 	var client *http.Client
677 | 	client = http.DefaultClient
678 | 	if proxy != "" {
679 | 		client = &http.Client{
680 | 			Transport: &http.Transport{
681 | 				TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
682 | 				ResponseHeaderTimeout: 30 * time.Second,
683 | 			},
684 | 		}
685 | 		downloader.AddProxy(proxy, client.Transport.(*http.Transport))
686 | 	}
687 | 
688 | 	req, err := http.NewRequest("GET", fmt.Sprintf("https://%s.su/api/v1/account/favorites?type=user", s), nil)
689 | 	if err != nil {
690 | 		log.Fatalf("Error creating request: %s", err)
691 | 	}
692 | 	req.Header.Set("Host", fmt.Sprintf("%s.su", s))
693 | 	for _, v := range cookie {
694 | 		req.AddCookie(v)
695 | 	}
696 | 	resp, err := client.Do(req)
697 | 	if err != nil {
698 | 		log.Fatalf("Error getting favorites: %s", err)
699 | 	}
700 | 	defer resp.Body.Close()
701 | 	if resp.StatusCode != 200 {
702 | 		log.Fatalf("Error getting favorites: %d", resp.StatusCode)
703 | 	}
704 | 	var favoriteCreators []kemono.FavoriteCreator
705 | 	err = json.NewDecoder(resp.Body).Decode(&favoriteCreators)
706 | 	if err != nil {
707 | 		log.Fatalf("Error decoding favorites: %s", err)
708 | 	}
709 | 	return favoriteCreators
710 | }
711 | 
712 | func fetchFavoritePosts(s string, cookie []*http.Cookie) []kemono.PostRaw {
713 | 	log.Printf("fetching favorite posts from %s.su", s)
714 | 	var client *http.Client
715 | 	client = http.DefaultClient
716 | 	if proxy != "" {
717 | 		client = &http.Client{
718 | 			Transport: &http.Transport{
719 | 				TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
720 | 				ResponseHeaderTimeout: 30 * time.Second,
721 | 			},
722 | 		}
723 | 		downloader.AddProxy(proxy, client.Transport.(*http.Transport))
724 | 	}
725 | 	req, err := http.NewRequest("GET", fmt.Sprintf("https://%s.su/api/v1/account/favorites?type=post", s), nil)
726 | 	if err != nil {
727 | 		log.Fatalf("Error creating request: %s", err)
728 | 	}
729 | 	req.Header.Set("Host", fmt.Sprintf("%s.su", s))
730 | 	for _, v := range cookie {
731 | 		req.AddCookie(v)
732 | 	}
733 | 	resp, err := client.Do(req)
734 | 	if err != nil {
735 | 		log.Fatalf("Error getting posts: %s", err)
736 | 	}
737 | 	defer resp.Body.Close()
738 | 	if resp.StatusCode != 200 {
739 | 		log.Fatalf("Error getting posts: %d", resp.StatusCode)
740 | 	}
741 | 	var posts []kemono.PostRaw
742 | 	err = json.NewDecoder(resp.Body).Decode(&posts)
743 | 	if err != nil {
744 | 		log.Fatalf("Error decoding posts: %s", err)
745 | 	}
746 | 	return posts
747 | }
748 | 
749 | func parasCookieFile(cookieFile string) []*http.Cookie {
750 | 	var (
751 | 		cookies []*http.Cookie
752 | 		domain  string
753 | 	)
754 | 	f, err := os.Open(cookieFile)
755 | 	if err != nil {
756 | 		log.Fatalf("Error opening cookie file: %s", err)
757 | 	}
758 | 	defer f.Close()
759 | 	if site != "" {
760 | 		domain = site
761 | 	}
762 | 	scanner := bufio.NewScanner(f)
763 | 	for scanner.Scan() {
764 | 		line := scanner.Text()
765 | 		if strings.HasPrefix(line, "#") {
766 | 			continue
767 | 		}
768 | 		columns := strings.Fields(line)
769 | 		if len(columns) < 7 {
770 | 			continue
771 | 		}
772 | 		domainComponents := strings.Split(columns[0], ".")
773 | 		d := domainComponents[len(domainComponents)-2]
774 | 		if domain == "" {
775 | 			domain = d
776 | 		} else if domain != d {
777 | 			// other domain ignore
778 | 			continue
779 | 		}
780 | 		exp, err := strconv.ParseInt(columns[4], 10, 64)
781 | 		if err != nil {
782 | 			continue
783 | 		}
784 | 		c := &http.Cookie{
785 | 			Name:    columns[5],
786 | 			Value:   columns[6],
787 | 			Domain:  columns[0],
788 | 			Path:    columns[2],
789 | 			Secure:  columns[3] == "TRUE",
790 | 			Expires: time.Unix(exp, 0),
791 | 		}
792 | 		cookies = append(cookies, c)
793 | 	}
794 | 	if site == "" {
795 | 		site = domain
796 | 	}
797 | 	return cookies
798 | }
799 | 


--------------------------------------------------------------------------------
/main/path.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"path/filepath"
  5 | 	"strings"
  6 | 	tmpl "text/template"
  7 | )
  8 | 
  9 | const (
 10 | 	Service   = "<ks:service>"
 11 | 	Creator   = "<ks:creator>"
 12 | 	Post      = "<ks:post>"
 13 | 	Index     = "<ks:index>"
 14 | 	Filename  = "<ks:filename>"
 15 | 	Filehash  = "<ks:filehash>"
 16 | 	Extension = "<ks:extension>"
 17 | )
 18 | 
 19 | const (
 20 | 	TmplDefault          = "[" + Service + "]" + Creator + "/" + Post + "/" + Filename + Extension
 21 | 	TmplWithPrefixNumber = "[" + Service + "]" + Creator + "/" + Post + "/" + Index + "_" + Filename + Extension
 22 | 	TmplIndexNumber      = "[" + Service + "]" + Creator + "/" + Post + "/" + Index + Extension
 23 | )
 24 | 
 25 | type PathConfig struct {
 26 | 	Service   string
 27 | 	Creator   string
 28 | 	Post      string
 29 | 	Index     int
 30 | 	Filename  string
 31 | 	Filehash  string
 32 | 	Extension string
 33 | }
 34 | 
 35 | func LoadPathTmpl(templateStr string, output string) (*tmpl.Template, error) {
 36 | 	templateStr = filepath.Join(output, templateStr)
 37 | 	templateStr = strings.ReplaceAll(templateStr, Service, "{{.Service}}")
 38 | 	templateStr = strings.ReplaceAll(templateStr, Creator, "{{.Creator}}")
 39 | 	templateStr = strings.ReplaceAll(templateStr, Post, "{{.Post}}")
 40 | 	templateStr = strings.ReplaceAll(templateStr, Index, "{{.Index}}")
 41 | 	templateStr = strings.ReplaceAll(templateStr, Filename, "{{.Filename}}")
 42 | 	templateStr = strings.ReplaceAll(templateStr, Filehash, "{{.Filehash}}")
 43 | 	templateStr = strings.ReplaceAll(templateStr, Extension, "{{.Extension}}")
 44 | 	tmpl, err := tmpl.New("path").Parse(templateStr)
 45 | 	if err != nil {
 46 | 		return nil, err
 47 | 	}
 48 | 
 49 | 	return tmpl, nil
 50 | }
 51 | 
 52 | func ExecutePathTmpl(tmpl *tmpl.Template, config *PathConfig) string {
 53 | 	var path strings.Builder
 54 | 	err := tmpl.Execute(&path, config)
 55 | 	if err != nil {
 56 | 		panic(err)
 57 | 	}
 58 | 
 59 | 	return path.String()
 60 | }
 61 | 
 62 | type TmplCache struct {
 63 | 	tmpl map[string]*tmpl.Template
 64 | }
 65 | 
 66 | func NewTmplCache() *TmplCache {
 67 | 	return &TmplCache{
 68 | 		tmpl: make(map[string]*tmpl.Template),
 69 | 	}
 70 | }
 71 | 
 72 | func (t *TmplCache) init() {
 73 | 	// default template
 74 | 	tmpl, err := LoadPathTmpl(template, output)
 75 | 	if err != nil {
 76 | 		panic(err)
 77 | 	}
 78 | 	t.tmpl["default"] = tmpl
 79 | 	if imageTemplate != "" {
 80 | 		tmpl, err := LoadPathTmpl(imageTemplate, output)
 81 | 		if err != nil {
 82 | 			panic(err)
 83 | 		}
 84 | 		t.tmpl["image"] = tmpl
 85 | 	}
 86 | 	if videoTemplate != "" {
 87 | 		tmpl, err := LoadPathTmpl(videoTemplate, output)
 88 | 		if err != nil {
 89 | 			panic(err)
 90 | 		}
 91 | 		t.tmpl["video"] = tmpl
 92 | 	}
 93 | 	if audioTemplate != "" {
 94 | 		tmpl, err := LoadPathTmpl(audioTemplate, output)
 95 | 		if err != nil {
 96 | 			panic(err)
 97 | 		}
 98 | 		t.tmpl["audio"] = tmpl
 99 | 	}
100 | 	if archiveTemplate != "" {
101 | 		tmpl, err := LoadPathTmpl(archiveTemplate, output)
102 | 		if err != nil {
103 | 			panic(err)
104 | 		}
105 | 		t.tmpl["archive"] = tmpl
106 | 	}
107 | }
108 | 
109 | func (t *TmplCache) GetTmpl(typ string) *tmpl.Template {
110 | 	if tmpl, ok := t.tmpl[typ]; ok {
111 | 		return tmpl
112 | 	}
113 | 	return t.tmpl["default"]
114 | }
115 | 
116 | func (t *TmplCache) Execute(typ string, config *PathConfig) string {
117 | 	return ExecutePathTmpl(t.GetTmpl(typ), config)
118 | }
119 | 
120 | func getTyp(ext string) string {
121 | 	switch ext {
122 | 	case ".apng", ".avif", ".bmp", ".gif", ".ico", ".cur", ".jpg", ".jpeg", ".jfif", ".pjpeg", ".pjp", ".png", ".svg", ".tif", ".tiff", ".webp", ".jpe":
123 | 		return "image"
124 | 	case ".mp4", ".webm", ".mkv", ".avi", ".mov", ".wmv", ".flv", ".f4v", ".m4v", ".rmvb", ".rm", ".3gp", ".dat", ".ts", ".mts", ".vob":
125 | 		return "video"
126 | 	case ".mp3", ".wav", ".flac", ".ape", ".aac", ".ogg", ".wma", ".m4a", ".aiff", ".alac":
127 | 		return "audio"
128 | 	case ".zip", ".rar", ".7z", ".tar", ".gz", ".bz2", ".xz", ".zipmod":
129 | 		return "archive"
130 | 	default:
131 | 		return "default"
132 | 	}
133 | }
134 | 


--------------------------------------------------------------------------------
/term/README.md:
--------------------------------------------------------------------------------
1 | # Terminal
2 | 
3 | From https://github.com/restic/restic/tree/master/internal/ui/termstatus.


--------------------------------------------------------------------------------
/term/terminal.go:
--------------------------------------------------------------------------------
  1 | package term
  2 | 
  3 | import (
  4 | 	"bufio"
  5 | 	"bytes"
  6 | 	"context"
  7 | 	"fmt"
  8 | 	"golang.org/x/term"
  9 | 	"io"
 10 | 	"os"
 11 | 	"regexp"
 12 | 	"strings"
 13 | 	"unicode/utf8"
 14 | )
 15 | 
 16 | const asciiEscapeCode = "\x1b[0m"
 17 | 
 18 | // print directly to the terminal
 19 | type message struct {
 20 | 	line string
 21 | 	err  bool
 22 | }
 23 | 
 24 | // update lines in the terminal
 25 | type status struct {
 26 | 	lines []string
 27 | }
 28 | 
 29 | type fder interface {
 30 | 	Fd() uintptr
 31 | }
 32 | 
 33 | // no implementation for background mode
 34 | type Terminal struct {
 35 | 	wr        *bufio.Writer
 36 | 	fd        uintptr
 37 | 	errWriter io.Writer
 38 | 	buf       *bytes.Buffer
 39 | 	msg       chan message
 40 | 	status    chan status
 41 | 	// if it has the fd, can update the status lines
 42 | 	updateStatus bool
 43 | 
 44 | 	// will be closed when the goroutine which runs Run() terminates, so it'll
 45 | 	// yield a default value immediately
 46 | 	closed chan struct{}
 47 | 
 48 | 	clearCurrentLine func(io.Writer, uintptr)
 49 | 	moveCursorUp     func(io.Writer, uintptr, int)
 50 | 	asciiResetCode   string
 51 | }
 52 | 
 53 | func NewTerminal(w io.Writer, errWriter io.Writer, disableStatus bool) *Terminal {
 54 | 	t := &Terminal{
 55 | 		wr:        bufio.NewWriter(w),
 56 | 		errWriter: errWriter,
 57 | 		buf:       bytes.NewBuffer(nil),
 58 | 		msg:       make(chan message),
 59 | 		status:    make(chan status),
 60 | 		closed:    make(chan struct{}),
 61 | 	}
 62 | 	if disableStatus {
 63 | 		return t
 64 | 	}
 65 | 
 66 | 	// if it has the fd, can update the status lines
 67 | 	if f, ok := w.(fder); ok && CanUpdateStatus(f.Fd()) {
 68 | 		t.updateStatus = true
 69 | 		t.fd = f.Fd()
 70 | 		t.clearCurrentLine = clearCurrentLine(w, t.fd)
 71 | 		t.moveCursorUp = moveCursorUp(w, t.fd)
 72 | 	}
 73 | 
 74 | 	// check if the terminal supports ascii escape codes
 75 | 	if t.updateStatus && SupportsEscapeCodes(t.fd) {
 76 | 		t.asciiResetCode = asciiEscapeCode
 77 | 	} else {
 78 | 		t.asciiResetCode = ""
 79 | 	}
 80 | 
 81 | 	return t
 82 | }
 83 | 
 84 | func (t *Terminal) Run(ctx context.Context) {
 85 | 	defer close(t.closed)
 86 | 	if t.updateStatus {
 87 | 		t.run(ctx)
 88 | 		return
 89 | 	}
 90 | 
 91 | 	t.runWithoutStatus(ctx)
 92 | }
 93 | 
 94 | // run listens on the channels and updates the terminal screen.
 95 | func (t *Terminal) run(ctx context.Context) {
 96 | 	var status []string
 97 | 	var lastLineCount int
 98 | 	for {
 99 | 		select {
100 | 		case <-ctx.Done():
101 | 
102 | 			return
103 | 
104 | 		case msg := <-t.msg:
105 | 			for i := 0; i < lastLineCount-1; i++ {
106 | 				t.clearCurrentLine(t.wr, t.fd)
107 | 				t.moveCursorUp(t.wr, t.fd, 1)
108 | 			}
109 | 			t.clearCurrentLine(t.wr, t.fd)
110 | 			io.Writer(t.wr).Write([]byte("\r"))
111 | 
112 | 			var dst io.Writer
113 | 			if msg.err {
114 | 				dst = t.errWriter
115 | 
116 | 				// assume t.wr and t.errWriter are different, so we need to
117 | 				// flush clearing the current line
118 | 				err := t.wr.Flush()
119 | 				if err != nil {
120 | 					fmt.Fprintf(os.Stderr, "flush failed: %v\n", err)
121 | 				}
122 | 			} else {
123 | 				dst = t.wr
124 | 			}
125 | 
126 | 			if _, err := io.WriteString(dst, msg.line); err != nil {
127 | 				fmt.Fprintf(os.Stderr, "write failed: %v\n", err)
128 | 				continue
129 | 			}
130 | 
131 | 			t.writeStatus(status)
132 | 
133 | 			if err := t.wr.Flush(); err != nil {
134 | 				fmt.Fprintf(os.Stderr, "flush failed: %v\n", err)
135 | 			}
136 | 
137 | 		case stat := <-t.status:
138 | 			for i := 0; i < lastLineCount-1; i++ {
139 | 				t.clearCurrentLine(t.wr, t.fd)
140 | 				t.moveCursorUp(t.wr, t.fd, 1)
141 | 			}
142 | 			t.clearCurrentLine(t.wr, t.fd)
143 | 			io.Writer(t.wr).Write([]byte("\r"))
144 | 			lastLineCount = len(stat.lines)
145 | 			status = status[:0]
146 | 			status = append(status, stat.lines...)
147 | 			t.writeStatus(status)
148 | 		}
149 | 	}
150 | }
151 | 
152 | func (t *Terminal) writeStatus(status []string) {
153 | 	for _, line := range status {
154 | 		//t.clearCurrentLine(t.wr, t.fd)
155 | 
156 | 		_, err := t.wr.WriteString(line)
157 | 		if err != nil {
158 | 			fmt.Fprintf(os.Stderr, "write failed: %v\n", err)
159 | 		}
160 | 
161 | 		// flush is needed so that the current line is updated
162 | 		err = t.wr.Flush()
163 | 		if err != nil {
164 | 			fmt.Fprintf(os.Stderr, "flush failed: %v\n", err)
165 | 		}
166 | 	}
167 | 
168 | 	err := t.wr.Flush()
169 | 	if err != nil {
170 | 		fmt.Fprintf(os.Stderr, "flush failed: %v\n", err)
171 | 	}
172 | }
173 | 
174 | func (t *Terminal) runWithoutStatus(ctx context.Context) {
175 | 	for {
176 | 		select {
177 | 		case <-ctx.Done():
178 | 			return
179 | 		case msg := <-t.msg:
180 | 			var flush func() error
181 | 
182 | 			var dst io.Writer
183 | 			if msg.err {
184 | 				dst = t.errWriter
185 | 			} else {
186 | 				dst = t.wr
187 | 				flush = t.wr.Flush
188 | 			}
189 | 
190 | 			if _, err := io.WriteString(dst, msg.line); err != nil {
191 | 				fmt.Fprintf(os.Stderr, "write failed: %v\n", err)
192 | 			}
193 | 
194 | 			if flush == nil {
195 | 				continue
196 | 			}
197 | 
198 | 			if err := flush(); err != nil {
199 | 				fmt.Fprintf(os.Stderr, "flush failed: %v\n", err)
200 | 			}
201 | 
202 | 		case stat := <-t.status:
203 | 			for _, line := range stat.lines {
204 | 				// Ensure that each message ends with exactly one newline.
205 | 				fmt.Fprintln(t.wr, strings.TrimRight(line, "\n"))
206 | 			}
207 | 			if err := t.wr.Flush(); err != nil {
208 | 				fmt.Fprintf(os.Stderr, "flush failed: %v\n", err)
209 | 			}
210 | 		}
211 | 	}
212 | }
213 | 
214 | // clear the previously written status lines
215 | func (t *Terminal) undoStatus(lines int) {
216 | 	for i := 0; i < lines; i++ {
217 | 		t.clearCurrentLine(t.wr, t.fd)
218 | 
219 | 		_, err := t.wr.WriteRune('\n')
220 | 		if err != nil {
221 | 			fmt.Fprintf(os.Stderr, "write failed: %v\n", err)
222 | 		}
223 | 
224 | 		// flush is needed so that the current line is updated
225 | 		err = t.wr.Flush()
226 | 		if err != nil {
227 | 			fmt.Fprintf(os.Stderr, "flush failed: %v\n", err)
228 | 		}
229 | 	}
230 | 
231 | 	t.moveCursorUp(t.wr, t.fd, lines)
232 | 
233 | 	err := t.wr.Flush()
234 | 	if err != nil {
235 | 		fmt.Fprintf(os.Stderr, "flush failed: %v\n", err)
236 | 	}
237 | }
238 | 
239 | func (t *Terminal) print(line string, isErr bool) {
240 | 	// make sure the line ends with a line break
241 | 	if line[len(line)-1] != '\n' {
242 | 		line += "\n"
243 | 	}
244 | 
245 | 	select {
246 | 	case t.msg <- message{line: line, err: isErr}:
247 | 	case <-t.closed:
248 | 	}
249 | }
250 | 
251 | // Print writes a line to the terminal.
252 | func (t *Terminal) Print(line string) {
253 | 	t.print(line, false)
254 | }
255 | 
256 | // Printf uses fmt.Sprintf to write a line to the terminal.
257 | func (t *Terminal) Printf(msg string, args ...interface{}) {
258 | 	s := fmt.Sprintf(msg, args...)
259 | 	t.Print(s)
260 | }
261 | 
262 | // Error writes an error to the terminal.
263 | func (t *Terminal) Error(line string) {
264 | 	t.print(line, true)
265 | }
266 | 
267 | // Errorf uses fmt.Sprintf to write an error line to the terminal.
268 | func (t *Terminal) Errorf(msg string, args ...interface{}) {
269 | 	s := fmt.Sprintf(msg, args...)
270 | 	t.Error(s)
271 | }
272 | 
273 | var asciiColorPat = regexp.MustCompile(`(\x1b\[[0-9;]*m)?([\s\S]*?)(\x1b\[[0-9;]*m|$)`)
274 | 
275 | // Truncate truncates a string to a given width, taking into account ANSI color
276 | func Truncate(s string, w int) string {
277 | 	var (
278 | 		builder strings.Builder
279 | 	)
280 | 	for _, m := range asciiColorPat.FindAllStringSubmatchIndex(s, -1) {
281 | 		for i := 0; i < len(m); i++ {
282 | 			if m[i] == -1 {
283 | 				m[i] = 0
284 | 			}
285 | 		}
286 | 		// write ascii color
287 | 		builder.WriteString(s[m[2]:m[3]])
288 | 		// get text length
289 | 		textLen := utf8.RuneCountInString(s[m[4]:m[5]])
290 | 		if textLen > w {
291 | 			cutRune := truncateString(s[m[4]:m[5]], w)
292 | 			builder.Write([]byte(string(cutRune)))
293 | 			break
294 | 		} else {
295 | 			builder.WriteString(s[m[4]:m[5]])
296 | 			w -= textLen
297 | 		}
298 | 		if len(m) == 8 {
299 | 			builder.WriteString(s[m[6]:m[7]])
300 | 		}
301 | 	}
302 | 	return builder.String()
303 | }
304 | 
305 | func truncateString(runes string, w int) []rune {
306 | 	var cutRunes []rune
307 | 	for i := 0; i < len(runes); {
308 | 		_, size := utf8.DecodeRuneInString(runes[i:])
309 | 		if w < size {
310 | 			break
311 | 		}
312 | 		cutRunes = append(cutRunes, []rune(runes[i:i+size])...)
313 | 		w -= size
314 | 		i += size
315 | 	}
316 | 	return cutRunes
317 | }
318 | 
319 | // SetStatus updates the status lines.
320 | func (t *Terminal) SetStatus(lines []string) {
321 | 	if len(lines) == 0 {
322 | 		return
323 | 	}
324 | 
325 | 	// only truncate interactive status output
326 | 	var width int
327 | 	if t.updateStatus {
328 | 		var err error
329 | 		width, _, err = term.GetSize(int(t.fd))
330 | 		if err != nil || width <= 0 {
331 | 			// use 80 columns by default
332 | 			width = 80
333 | 		}
334 | 	}
335 | 
336 | 	// make sure that all lines have a line break and are not too long
337 | 	for i, line := range lines {
338 | 		line = strings.TrimRight(line, "\n")
339 | 		if width > 0 {
340 | 			line = Truncate(line, width-2)
341 | 		}
342 | 		lines[i] = line + t.asciiResetCode + "\n"
343 | 	}
344 | 
345 | 	// make sure the last line does not have a line break
346 | 	last := len(lines) - 1
347 | 	lines[last] = strings.TrimRight(lines[last], "\n")
348 | 
349 | 	select {
350 | 	case t.status <- status{lines: lines}:
351 | 	case <-t.closed:
352 | 	}
353 | }
354 | 


--------------------------------------------------------------------------------
/term/terminal_posix.go:
--------------------------------------------------------------------------------
 1 | package term
 2 | 
 3 | import (
 4 | 	"bytes"
 5 | 	"fmt"
 6 | 	"io"
 7 | 	"os"
 8 | )
 9 | 
10 | const (
11 | 	posixControlMoveCursorHome = "\r"
12 | 	posixControlMoveCursorUp   = "\x1b[1A"
13 | 	posixControlClearLine      = "\x1b[2K"
14 | )
15 | 
16 | func posixClearCurrentLine(wr io.Writer, fd uintptr) {
17 | 	// clear current line
18 | 	_, err := wr.Write([]byte(posixControlMoveCursorHome + posixControlClearLine))
19 | 	if err != nil {
20 | 		fmt.Fprintf(os.Stderr, "write failed: %v\n", err)
21 | 		return
22 | 	}
23 | }
24 | 
25 | func posixMoveCursorUp(wr io.Writer, fd uintptr, n int) {
26 | 	data := []byte(posixControlMoveCursorHome)
27 | 	data = append(data, bytes.Repeat([]byte(posixControlMoveCursorUp), n)...)
28 | 	_, err := wr.Write(data)
29 | 	if err != nil {
30 | 		fmt.Fprintf(os.Stderr, "write failed: %v\n", err)
31 | 		return
32 | 	}
33 | }
34 | 


--------------------------------------------------------------------------------
/term/terminal_unix.go:
--------------------------------------------------------------------------------
 1 | //go:build !windows
 2 | 
 3 | package term
 4 | 
 5 | import (
 6 | 	"io"
 7 | 	"os"
 8 | 
 9 | 	"golang.org/x/term"
10 | )
11 | 
12 | func clearCurrentLine(wr io.Writer, fd uintptr) func(io.Writer, uintptr) {
13 | 	return posixClearCurrentLine
14 | }
15 | 
16 | func moveCursorUp(wr io.Writer, fd uintptr) func(io.Writer, uintptr, int) {
17 | 	return posixMoveCursorUp
18 | }
19 | 
20 | func CanUpdateStatus(fd uintptr) bool {
21 | 	if !term.IsTerminal(int(fd)) {
22 | 		return false
23 | 	}
24 | 	term := os.Getenv("TERM")
25 | 	if term == "" {
26 | 		return false
27 | 	}
28 | 	// TODO actually read termcap db and detect if terminal supports what we need
29 | 	return term != "dumb"
30 | }
31 | 
32 | func SupportsEscapeCodes(fd uintptr) bool {
33 | 	return true
34 | }
35 | 


--------------------------------------------------------------------------------
/term/terminal_windows.go:
--------------------------------------------------------------------------------
  1 | //go:build windows
  2 | 
  3 | package term
  4 | 
  5 | import (
  6 | 	"golang.org/x/crypto/ssh/terminal"
  7 | 	"golang.org/x/sys/windows"
  8 | 	"io"
  9 | 	"strings"
 10 | 	"syscall"
 11 | 	"unsafe"
 12 | )
 13 | 
 14 | func clearCurrentLine(wr io.Writer, fd uintptr) func(io.Writer, uintptr) {
 15 | 	// easy case, the terminal is cmd or psh, without redirection
 16 | 	if isWindowsTerminal(fd) {
 17 | 		return windowsClearCurrentLine
 18 | 	}
 19 | 
 20 | 	// assume we're running in mintty/cygwin
 21 | 	return posixClearCurrentLine
 22 | }
 23 | 
 24 | // moveCursorUp moves the cursor to the line n lines above the current one.
 25 | func moveCursorUp(wr io.Writer, fd uintptr) func(io.Writer, uintptr, int) {
 26 | 	// easy case, the terminal is cmd or psh, without redirection
 27 | 	if isWindowsTerminal(fd) {
 28 | 		return windowsMoveCursorUp
 29 | 	}
 30 | 
 31 | 	// assume we're running in mintty/cygwin
 32 | 	return posixMoveCursorUp
 33 | }
 34 | 
 35 | var kernel32 = syscall.NewLazyDLL("kernel32.dll")
 36 | 
 37 | var (
 38 | 	procFillConsoleOutputCharacter = kernel32.NewProc("FillConsoleOutputCharacterW")
 39 | 	procFillConsoleOutputAttribute = kernel32.NewProc("FillConsoleOutputAttribute")
 40 | )
 41 | 
 42 | func windowsClearCurrentLine(wr io.Writer, fd uintptr) {
 43 | 	var info windows.ConsoleScreenBufferInfo
 44 | 	err := windows.GetConsoleScreenBufferInfo(windows.Handle(fd), &info)
 45 | 	if err != nil {
 46 | 		panic(err)
 47 | 	}
 48 | 
 49 | 	// clear the line
 50 | 	cursor := windows.Coord{
 51 | 		X: info.Window.Left,
 52 | 		Y: info.CursorPosition.Y,
 53 | 	}
 54 | 	var count, w uint32
 55 | 	count = uint32(info.Size.X)
 56 | 	procFillConsoleOutputAttribute.Call(fd, uintptr(info.Attributes), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w)))
 57 | 	procFillConsoleOutputCharacter.Call(fd, uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w)))
 58 | }
 59 | 
 60 | func windowsMoveCursorUp(wr io.Writer, fd uintptr, n int) {
 61 | 	var info windows.ConsoleScreenBufferInfo
 62 | 	windows.GetConsoleScreenBufferInfo(windows.Handle(fd), &info)
 63 | 
 64 | 	// move cursor up by n lines and to the first column
 65 | 	windows.SetConsoleCursorPosition(windows.Handle(fd), windows.Coord{
 66 | 		X: 0,
 67 | 		Y: info.CursorPosition.Y - int16(n),
 68 | 	})
 69 | 
 70 | }
 71 | 
 72 | func isWindowsTerminal(fd uintptr) bool {
 73 | 	return terminal.IsTerminal(int(fd))
 74 | }
 75 | 
 76 | func isPipe(fd uintptr) bool {
 77 | 	typ, err := windows.GetFileType(windows.Handle(fd))
 78 | 	return err == nil && typ == windows.FILE_TYPE_PIPE
 79 | }
 80 | 
 81 | func getFileNameByHandle(fd uintptr) (string, error) {
 82 | 	type FILE_NAME_INFO struct {
 83 | 		FileNameLength int32
 84 | 		FileName       [windows.MAX_LONG_PATH]uint16
 85 | 	}
 86 | 
 87 | 	var fi FILE_NAME_INFO
 88 | 	err := windows.GetFileInformationByHandleEx(windows.Handle(fd), windows.FileNameInfo, (*byte)(unsafe.Pointer(&fi)), uint32(unsafe.Sizeof(fi)))
 89 | 	if err != nil {
 90 | 		return "", err
 91 | 	}
 92 | 
 93 | 	filename := syscall.UTF16ToString(fi.FileName[:])
 94 | 	return filename, nil
 95 | }
 96 | 
 97 | func CanUpdateStatus(fd uintptr) bool {
 98 | 	// easy case, the terminal is cmd or psh, without redirection
 99 | 	if isWindowsTerminal(fd) {
100 | 		return true
101 | 	}
102 | 
103 | 	// pipes require special handling
104 | 	if !isPipe(fd) {
105 | 		return false
106 | 	}
107 | 
108 | 	fn, err := getFileNameByHandle(fd)
109 | 	if err != nil {
110 | 		return false
111 | 	}
112 | 
113 | 	// inspired by https://github.com/RyanGlScott/mintty/blob/master/src/System/Console/MinTTY/Win32.hsc
114 | 	// terminal: \msys-dd50a72ab4668b33-pty0-to-master
115 | 	// pipe to cat: \msys-dd50a72ab4668b33-13244-pipe-0x16
116 | 	if (strings.HasPrefix(fn, "\\cygwin-") || strings.HasPrefix(fn, "\\msys-")) &&
117 | 		strings.Contains(fn, "-pty") && strings.HasSuffix(fn, "-master") {
118 | 		return true
119 | 	}
120 | 
121 | 	return false
122 | }
123 | 
124 | func SupportsEscapeCodes(fd uintptr) bool {
125 | 	if isWindowsTerminal(fd) {
126 | 		h := syscall.Handle(fd)
127 | 		var mode uint32
128 | 		err := syscall.GetConsoleMode(h, &mode)
129 | 		if err != nil {
130 | 			return false
131 | 		}
132 | 		return mode&windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING != 0
133 | 	}
134 | 	return true
135 | }
136 | 


--------------------------------------------------------------------------------
/utils/format.go:
--------------------------------------------------------------------------------
 1 | package utils
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"strconv"
 6 | 	"strings"
 7 | )
 8 | 
 9 | const (
10 | 	B = 1 << (10 * iota)
11 | 	KB
12 | 	MB
13 | 	GB
14 | 	TB
15 | )
16 | 
17 | const (
18 | 	Nanosecond  = 1
19 | 	Microsecond = 1000 * Nanosecond
20 | 	Millisecond = 1000 * Microsecond
21 | 	Second      = 1000 * Millisecond
22 | 	Minute      = 60 * Second
23 | 	Hour        = 60 * Minute
24 | )
25 | 
26 | func FormatSize(size int64) string {
27 | 
28 | 	switch {
29 | 	case size >= TB:
30 | 		return fmt.Sprintf("%.2fTB", float64(size)/TB)
31 | 	case size >= GB:
32 | 		return fmt.Sprintf("%.2f GB", float64(size)/GB)
33 | 	case size >= MB:
34 | 		return fmt.Sprintf("%.2f MB", float64(size)/MB)
35 | 	case size >= KB:
36 | 		return fmt.Sprintf("%.2f KB", float64(size)/KB)
37 | 	default:
38 | 		return fmt.Sprintf("%d B", size)
39 | 	}
40 | }
41 | 
42 | func ParseSize(size string) int64 {
43 | 	size = strings.ToUpper(size)
44 | 	var (
45 | 		fvalue float64
46 | 		unit   string
47 | 		value  string
48 | 	)
49 | 	// 1MB or 1 MB
50 | 	parts := strings.Fields(size)
51 | 	if len(parts) != 2 {
52 | 		for _, u := range []string{"TB", "GB", "MB", "KB", "B"} {
53 | 			if strings.HasSuffix(size, u) {
54 | 				value = strings.TrimSuffix(size, u)
55 | 				unit = u
56 | 				break
57 | 			}
58 | 		}
59 | 		fvalue, _ = strconv.ParseFloat(value, 64)
60 | 	} else {
61 | 		fvalue, _ = strconv.ParseFloat(parts[0], 64)
62 | 		unit = parts[1]
63 | 	}
64 | 	switch unit {
65 | 	case "TB":
66 | 		return int64(fvalue * TB)
67 | 	case "GB":
68 | 		return int64(fvalue * GB)
69 | 	case "MB":
70 | 		return int64(fvalue * MB)
71 | 	case "KB":
72 | 		return int64(fvalue * KB)
73 | 	default:
74 | 		return int64(fvalue)
75 | 	}
76 | }
77 | 
78 | func FormatDuration(duration int64) string {
79 | 	switch {
80 | 	case duration >= Hour:
81 | 		return fmt.Sprintf("%.2dh%.2fm", duration/Hour, float64(duration%Hour)/Minute)
82 | 	case duration >= Minute:
83 | 		return fmt.Sprintf("%.2dm%.2fs", duration/Minute, float64(duration%Minute)/Second)
84 | 	case duration >= Second:
85 | 		return fmt.Sprintf("%.2fs", float64(duration)/Second)
86 | 	default:
87 | 		return fmt.Sprintf("%.2fms", float64(duration)/Millisecond)
88 | 	}
89 | }
90 | 


--------------------------------------------------------------------------------
/utils/helper.go:
--------------------------------------------------------------------------------
  1 | package utils
  2 | 
  3 | import (
  4 | 	"crypto/rand"
  5 | 	"crypto/sha256"
  6 | 	"encoding/hex"
  7 | 	"encoding/json"
  8 | 	"fmt"
  9 | 	"io"
 10 | 	"path/filepath"
 11 | 	"runtime"
 12 | 	"strings"
 13 | 	"time"
 14 | )
 15 | 
 16 | func SplitHash(str string) (string, error) {
 17 | 	parts := strings.Split(str, "/")
 18 | 	if len(parts) < 4 {
 19 | 		return "", nil
 20 | 	}
 21 | 	ext := filepath.Ext(parts[3])
 22 | 	name := parts[3][:len(parts[3])-len(ext)]
 23 | 	return name, nil
 24 | }
 25 | 
 26 | // Stringify beautify interface to string
 27 | func Stringify(v interface{}) string {
 28 | 	b, _ := json.MarshalIndent(v, "", "  ")
 29 | 	return string(b)
 30 | }
 31 | 
 32 | func Hash(w io.Reader) ([]byte, error) {
 33 | 	ha := sha256.New()
 34 | 	_, err := io.Copy(ha, w)
 35 | 	if err != nil {
 36 | 		return nil, err
 37 | 	}
 38 | 	return ha.Sum(nil), nil
 39 | }
 40 | 
 41 | func ValidDirectoryName(name string) string {
 42 | 	if name == "" {
 43 | 		return ""
 44 | 	}
 45 | 	if runtime.GOOS == "windows" {
 46 | 		invalidRune := "\x00-\x1f/\\:*?\"<>|\n\r\t"
 47 | 		validate := func(r rune) rune {
 48 | 			if strings.ContainsRune(invalidRune, r) {
 49 | 				return '_'
 50 | 			}
 51 | 			return r
 52 | 		}
 53 | 		s := strings.TrimSpace(strings.Map(validate, name))
 54 | 		if len(s) > 200 {
 55 | 			s = s[:200]
 56 | 		}
 57 | 		if len(s) > 0 && s[len(s)-1] == '.' {
 58 | 			s = s[:len(s)-1]
 59 | 			return fmt.Sprintf("%s_", s)
 60 | 		}
 61 | 		return s
 62 | 	}
 63 | 	invalidRune := "/\\\n\r\t"
 64 | 	validate := func(r rune) rune {
 65 | 		if strings.ContainsRune(invalidRune, r) {
 66 | 			return '_'
 67 | 		}
 68 | 		return r
 69 | 	}
 70 | 	s := strings.TrimSpace(strings.Map(validate, name))
 71 | 	if len(s) > 200 {
 72 | 		s = s[:200]
 73 | 	}
 74 | 	if len(s) > 0 && s[0] == '.' {
 75 | 		s = s[1:]
 76 | 		return fmt.Sprintf("_%s", s)
 77 | 	}
 78 | 	return s
 79 | }
 80 | 
 81 | type semaphore chan struct{}
 82 | 
 83 | func newSemaphore(n int) semaphore {
 84 | 	return make(semaphore, n)
 85 | }
 86 | 
 87 | func (s semaphore) acquire() {
 88 | 	s <- struct{}{}
 89 | }
 90 | 
 91 | func (s semaphore) release() {
 92 | 	<-s
 93 | }
 94 | 
 95 | type RateLimiter struct {
 96 | 	limit     int
 97 | 	semaphore semaphore
 98 | }
 99 | 
100 | func NewRateLimiter(tokenPreSecond int) *RateLimiter {
101 | 	r := &RateLimiter{
102 | 		limit:     tokenPreSecond,
103 | 		semaphore: newSemaphore(tokenPreSecond),
104 | 	}
105 | 	r.Timing()
106 | 	return r
107 | }
108 | 
109 | // Timing add token into semaphore
110 | func (r *RateLimiter) Timing() {
111 | 	t := time.NewTicker(time.Second)
112 | 	// full semaphore
113 | 	for i := 0; i < r.limit; i++ {
114 | 		r.semaphore.acquire()
115 | 	}
116 | 	go func() {
117 | 		for {
118 | 			select {
119 | 			case <-t.C:
120 | 				for i := 0; i < r.limit; i++ {
121 | 					r.semaphore.acquire()
122 | 				}
123 | 			}
124 | 		}
125 | 	}()
126 | }
127 | 
128 | func (r *RateLimiter) Token() {
129 | 	r.semaphore.release()
130 | }
131 | 
132 | func GenerateToken(size int) (string, error) {
133 | 	data := make([]byte, size)
134 | 	_, err := rand.Read(data)
135 | 	if err != nil {
136 | 		return "", err
137 | 	}
138 | 
139 | 	// Convert to hexadecimal
140 | 	hexStr := hex.EncodeToString(data)
141 | 
142 | 	return hexStr, nil
143 | 
144 | }
145 | 


--------------------------------------------------------------------------------