├── utils ├── expires │ ├── expires.s │ ├── dataexpires.go │ ├── cachemap │ │ ├── cachemap.go │ │ ├── utils.go │ │ ├── cacheunit.go │ │ └── cachemap_test.go │ └── expires.go ├── cachepool │ ├── malloc.s │ ├── syncpool.go │ ├── malloc_test.go │ ├── malloc.go │ ├── cachepool.go │ └── idcachepool.go ├── checkaccess │ ├── checkaccess.go │ ├── check_others.go │ ├── check_windows.go │ └── check_unix.go ├── prealloc │ ├── prealloc_plan9.go │ ├── errors.go │ ├── prealloc.go │ └── prealloc_windows.go ├── getip │ ├── errors.go │ ├── getip_test.go │ ├── getip.go │ ├── getip_techain.go │ └── getip_netease.go ├── regexp_pre.go ├── delay │ └── delay.go ├── escaper │ ├── escaper_test.go │ └── escaper.go ├── checksum │ ├── errors.go │ ├── checksum_test.go │ ├── file.go │ ├── checksum_write.go │ └── checksum.go ├── jwted25519 │ ├── jwted25519_test.go │ └── jwted25519.go ├── converter │ ├── short_display_test.go │ ├── converter_test.go │ ├── size_test.go │ ├── size.go │ └── converter.go ├── file_test.go ├── waitgroup │ ├── wait_group_test.go │ └── wait_group.go ├── bdcrypto │ ├── bdcrypto.go │ ├── reverse.go │ ├── base64.go │ ├── hmac.go │ ├── aes_test.go │ ├── bdcrypto_test.go │ ├── 3des.go │ ├── archive.go │ ├── ecb │ │ └── ecb.go │ ├── rsa.go │ └── des_test.go ├── error.go ├── jsonhelper │ └── jsonhelper.go ├── log_colorable_prefix.go ├── taskframework │ ├── taskinfo.go │ ├── task_unit.go │ ├── taskframework_test.go │ └── executor.go ├── addr.go ├── csttime │ └── time.go ├── utils.go ├── file.go └── crypto.go ├── pcsverbose ├── pcsdebug │ ├── pprof.go │ └── cpu.go ├── utils.go └── pcsverbose.go ├── requester ├── uploader │ ├── error.go │ ├── readed.go │ ├── example.go │ ├── block_test.go │ ├── instance_state.go │ ├── status.go │ ├── uploader.go │ ├── multiworker.go │ ├── block.go │ └── multiuploader.go ├── rio │ ├── part.go │ ├── multi_test.go │ ├── speeds │ │ ├── ratelimit_test.go │ │ ├── speeds.go │ │ └── ratelimit.go │ ├── buf.go │ ├── rio.go │ ├── multi.go │ └── file.go ├── downloader │ ├── sort.go │ ├── writer.go │ ├── resetcontroler.go │ ├── example.go │ ├── config.go │ ├── loadbalance.go │ ├── download_firstinfo.go │ ├── utils.go │ ├── download_test.go │ ├── status.go │ ├── instance_state.go │ └── monitor.go ├── transfer │ ├── transfer.proto │ ├── download_instanceinfo.go │ ├── download_status.go │ ├── rangelist.go │ └── transfer.pb.go ├── util.go ├── requester.go ├── http_client.go ├── fetch.go ├── multipartreader │ └── multipartreader.go └── dial.go ├── .gitignore ├── go.mod ├── cmd └── AndroidNDKBuild │ └── main.go ├── go.sum └── LICENSE /utils/expires/expires.s: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/cachepool/malloc.s: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/checkaccess/checkaccess.go: -------------------------------------------------------------------------------- 1 | package checkaccess 2 | -------------------------------------------------------------------------------- /utils/checkaccess/check_others.go: -------------------------------------------------------------------------------- 1 | //+build aix plan9 2 | 3 | package checkaccess 4 | 5 | func AccessRDWR(path string) bool { 6 | return true 7 | } 8 | -------------------------------------------------------------------------------- /utils/prealloc/prealloc_plan9.go: -------------------------------------------------------------------------------- 1 | //+build plan9 2 | 3 | package prealloc 4 | 5 | func PreAlloc(fd uintptr, length int64) error { 6 | return nil 7 | } 8 | -------------------------------------------------------------------------------- /utils/checkaccess/check_windows.go: -------------------------------------------------------------------------------- 1 | package checkaccess 2 | 3 | // TODO: check writable 4 | 5 | func AccessRDWR(path string) bool { 6 | return true 7 | } 8 | -------------------------------------------------------------------------------- /utils/getip/errors.go: -------------------------------------------------------------------------------- 1 | package getip 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | var ( 8 | // ErrParseIP 解析ip地址错误 9 | ErrParseIP = errors.New("parse ip error") 10 | ) 11 | -------------------------------------------------------------------------------- /pcsverbose/pcsdebug/pprof.go: -------------------------------------------------------------------------------- 1 | package pcsdebug 2 | 3 | import ( 4 | "net/http" 5 | _ "net/http/pprof" 6 | ) 7 | 8 | func StartPprofListen() { 9 | http.ListenAndServe("0.0.0.0:6060", nil) 10 | } 11 | -------------------------------------------------------------------------------- /utils/checkaccess/check_unix.go: -------------------------------------------------------------------------------- 1 | // +build !windows,!plan9,!aix 2 | 3 | package checkaccess 4 | 5 | import ( 6 | "syscall" 7 | ) 8 | 9 | func AccessRDWR(path string) bool { 10 | return syscall.Access(path, syscall.O_RDWR) == nil 11 | } 12 | -------------------------------------------------------------------------------- /requester/uploader/error.go: -------------------------------------------------------------------------------- 1 | package uploader 2 | 3 | type ( 4 | // MultiError 多线程上传的错误 5 | MultiError struct { 6 | Err error 7 | // IsRetry 是否重试, 8 | Terminated bool 9 | } 10 | ) 11 | 12 | func (me *MultiError) Error() string { 13 | return me.Err.Error() 14 | } 15 | -------------------------------------------------------------------------------- /utils/regexp_pre.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | var ( 8 | // HTTPSRE https regexp 9 | HTTPSRE = regexp.MustCompile("^https") 10 | // ChinaPhoneRE https regexp 11 | ChinaPhoneRE = regexp.MustCompile(`^(\+86)?1[3-9][0-9]\d{8}$`) 12 | ) 13 | -------------------------------------------------------------------------------- /utils/delay/delay.go: -------------------------------------------------------------------------------- 1 | package delay 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // NewDelayChan 发送延时信号 8 | func NewDelayChan(t time.Duration) <-chan struct{} { 9 | c := make(chan struct{}) 10 | go func() { 11 | time.Sleep(t) 12 | close(c) 13 | }() 14 | return c 15 | } 16 | -------------------------------------------------------------------------------- /utils/escaper/escaper_test.go: -------------------------------------------------------------------------------- 1 | package escaper_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/iikira/iikira-go-utils/utils/escaper" 6 | "testing" 7 | ) 8 | 9 | func TestEscape(t *testing.T) { 10 | fmt.Println(escaper.Escape(`asdf'asdfasd[]a[\[][sdf\[d]`, []rune{'[', '\''})) 11 | } 12 | -------------------------------------------------------------------------------- /utils/checksum/errors.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | var ( 8 | ErrFileIsNil = errors.New("file is nil") 9 | ErrChecksumWriteStop = errors.New("checksum write stop") 10 | ErrChecksumWriteAllStop = errors.New("checksum write all stop") 11 | ) 12 | -------------------------------------------------------------------------------- /utils/jwted25519/jwted25519_test.go: -------------------------------------------------------------------------------- 1 | package jwted25519_test 2 | 3 | import ( 4 | "crypto/rand" 5 | "fmt" 6 | "golang.org/x/crypto/ed25519" 7 | "testing" 8 | ) 9 | 10 | func TestGenEd25519Key(t *testing.T) { 11 | publicKey, privateKey, _ := ed25519.GenerateKey(rand.Reader) 12 | fmt.Println(publicKey, privateKey) 13 | } 14 | -------------------------------------------------------------------------------- /utils/converter/short_display_test.go: -------------------------------------------------------------------------------- 1 | package converter_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/iikira/iikira-go-utils/utils/converter" 6 | "testing" 7 | ) 8 | 9 | func TestShortDisplay(t *testing.T) { 10 | for i := 0; i < 20; i++ { 11 | fmt.Println([]byte(converter.ShortDisplay("\u0000我我\u0000我我我我", i))) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /requester/rio/part.go: -------------------------------------------------------------------------------- 1 | package rio 2 | 3 | import "io" 4 | 5 | type ( 6 | PartReaderLen64 struct { 7 | Part io.Reader 8 | Size int64 9 | } 10 | ) 11 | 12 | func (p2 *PartReaderLen64) Read(p []byte) (n int, err error) { 13 | return p2.Part.Read(p) 14 | } 15 | 16 | func (p2 *PartReaderLen64) Len() int64 { 17 | return p2.Size 18 | } 19 | -------------------------------------------------------------------------------- /utils/file_test.go: -------------------------------------------------------------------------------- 1 | package utils_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/iikira/iikira-go-utils/utils" 6 | "testing" 7 | ) 8 | 9 | func TestWalkDir(t *testing.T) { 10 | files, err := utils.WalkDir("/Users/syy/tmp", "") 11 | if err != nil { 12 | t.Fatal(err) 13 | } 14 | for _, file := range files { 15 | fmt.Println(file) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /utils/prealloc/errors.go: -------------------------------------------------------------------------------- 1 | package prealloc 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type ( 8 | PreAllocError struct { 9 | ProcName string 10 | Err error 11 | } 12 | ) 13 | 14 | func (pe *PreAllocError) Error() string { 15 | if pe.Err == nil { 16 | return "" 17 | } 18 | return fmt.Sprintf("%s error: %s\n", pe.ProcName, pe.Err) 19 | } 20 | -------------------------------------------------------------------------------- /requester/rio/multi_test.go: -------------------------------------------------------------------------------- 1 | package rio 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | func TestMultiReaderLen(t *testing.T) { 12 | rd1, rd2 := strings.NewReader("asdkfljalf"), strings.NewReader("---asva sdf") 13 | multi := MultiReaderLen(rd1, rd2) 14 | fmt.Println(rd1.Len()) 15 | io.Copy(os.Stdout, multi) 16 | } 17 | -------------------------------------------------------------------------------- /utils/waitgroup/wait_group_test.go: -------------------------------------------------------------------------------- 1 | package waitgroup 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestWg(t *testing.T) { 10 | wg := NewWaitGroup(2) 11 | for i := 0; i < 60; i++ { 12 | wg.AddDelta() 13 | go func(i int) { 14 | fmt.Println(i, wg.Parallel()) 15 | time.Sleep(1e9) 16 | wg.Done() 17 | }(i) 18 | } 19 | wg.Wait() 20 | } 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | # Others 18 | .DS_Store 19 | .idea/ 20 | -------------------------------------------------------------------------------- /utils/converter/converter_test.go: -------------------------------------------------------------------------------- 1 | package converter_test 2 | 3 | import ( 4 | "github.com/iikira/iikira-go-utils/utils/converter" 5 | "strings" 6 | "testing" 7 | ) 8 | 9 | func TestTrimPathInvalidChars(t *testing.T) { 10 | trimed := converter.TrimPathInvalidChars("ksjadfi*/?adf") 11 | if strings.Compare(trimed, "ksjadfiadf") != 0 { 12 | t.Fatalf("trimed: %s\n", trimed) 13 | } 14 | return 15 | } 16 | -------------------------------------------------------------------------------- /utils/prealloc/prealloc.go: -------------------------------------------------------------------------------- 1 | //+build !windows,!plan9 2 | 3 | // Package prealloc 初始化分配文件包 4 | package prealloc 5 | 6 | import ( 7 | "syscall" 8 | ) 9 | 10 | // PreAlloc 预分配文件空间 11 | func PreAlloc(fd uintptr, length int64) error { 12 | err := syscall.Ftruncate(int(fd), length) 13 | if err != nil { 14 | return &PreAllocError{ 15 | ProcName: "Ftruncate", 16 | Err: err, 17 | } 18 | } 19 | return nil 20 | } 21 | -------------------------------------------------------------------------------- /utils/converter/size_test.go: -------------------------------------------------------------------------------- 1 | package converter_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/iikira/iikira-go-utils/utils/converter" 6 | "testing" 7 | ) 8 | 9 | func TestParseFileSizeStr(t *testing.T) { 10 | for _, v := range []string{"1k", "3.86mb", "4.001Gb", "32"} { 11 | size, err := converter.ParseFileSizeStr(v) 12 | if err != nil { 13 | t.Fatalf("%s\n", err) 14 | } 15 | fmt.Println(v, size) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /utils/getip/getip_test.go: -------------------------------------------------------------------------------- 1 | package getip 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestGetIP(t *testing.T) { 8 | ipAddr, err := IPInfo(false) 9 | if err != nil { 10 | t.Errorf("err: %s\n", err) 11 | return 12 | } 13 | 14 | t.Logf("from ipify: %s\n", ipAddr) 15 | 16 | ipAddr, err = IPInfoFromNetease() 17 | if err != nil { 18 | t.Errorf("err: %s\n", err) 19 | return 20 | } 21 | 22 | t.Logf("from netease: %s\n", ipAddr) 23 | } 24 | -------------------------------------------------------------------------------- /utils/bdcrypto/bdcrypto.go: -------------------------------------------------------------------------------- 1 | package bdcrypto 2 | 3 | import ( 4 | "encoding/hex" 5 | ) 6 | 7 | // RSAEncryptOfWapBaidu 针对 WAP 登录百度的 RSA 加密 8 | func RSAEncryptOfWapBaidu(rsaPublicKeyModulus string, origData []byte) (string, error) { 9 | ciphertext, err := RSAEncryptNoPadding(rsaPublicKeyModulus, DefaultRSAPublicKeyExponent, BytesReverse(origData)) 10 | if err != nil { 11 | return "", err 12 | } 13 | 14 | return hex.EncodeToString(ciphertext), nil 15 | } 16 | -------------------------------------------------------------------------------- /utils/error.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "log" 5 | "os" 6 | ) 7 | 8 | // PrintErrIfExist 简易错误处理, 如果 err 存在, 就只向屏幕输出 err 。 9 | func PrintErrIfExist(err error) { 10 | if err != nil { 11 | log.Println(err) 12 | } 13 | } 14 | 15 | // PrintErrAndExit 简易错误处理, 如果 err 存在, 向屏幕输出 err 并退出, annotate 是加在 err 之前的注释信息。 16 | func PrintErrAndExit(annotate string, err error) { 17 | if err != nil { 18 | log.Println(annotate, err) 19 | os.Exit(1) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /utils/jsonhelper/jsonhelper.go: -------------------------------------------------------------------------------- 1 | package jsonhelper 2 | 3 | import ( 4 | "github.com/json-iterator/go" 5 | "io" 6 | ) 7 | 8 | // UnmarshalData 将 r 中的 json 格式的数据, 解析到 data 9 | func UnmarshalData(r io.Reader, data interface{}) error { 10 | d := jsoniter.NewDecoder(r) 11 | return d.Decode(data) 12 | } 13 | 14 | // MarshalData 将 data, 生成 json 格式的数据, 写入 w 中 15 | func MarshalData(w io.Writer, data interface{}) error { 16 | e := jsoniter.NewEncoder(w) 17 | return e.Encode(data) 18 | } 19 | -------------------------------------------------------------------------------- /requester/downloader/sort.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | type ( 4 | // ByLeftDesc 根据剩余下载量倒序排序 5 | ByLeftDesc struct { 6 | WorkerList 7 | } 8 | ) 9 | 10 | // Len 返回长度 11 | func (wl WorkerList) Len() int { 12 | return len(wl) 13 | } 14 | 15 | // Swap 交换 16 | func (wl WorkerList) Swap(i, j int) { 17 | wl[i], wl[j] = wl[j], wl[i] 18 | } 19 | 20 | // Less 实现倒序 21 | func (wl ByLeftDesc) Less(i, j int) bool { 22 | return wl.WorkerList[i].wrange.Len() > wl.WorkerList[j].wrange.Len() 23 | } 24 | -------------------------------------------------------------------------------- /utils/expires/dataexpires.go: -------------------------------------------------------------------------------- 1 | package expires 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type ( 8 | DataExpires interface { 9 | Data() interface{} 10 | Expires 11 | } 12 | 13 | dataExpires struct { 14 | data interface{} 15 | Expires 16 | } 17 | ) 18 | 19 | func NewDataExpires(data interface{}, dur time.Duration) DataExpires { 20 | return &dataExpires{ 21 | data: data, 22 | Expires: NewExpires(dur), 23 | } 24 | } 25 | 26 | func (de *dataExpires) Data() interface{} { 27 | return de.data 28 | } 29 | -------------------------------------------------------------------------------- /utils/bdcrypto/reverse.go: -------------------------------------------------------------------------------- 1 | package bdcrypto 2 | 3 | import ( 4 | "unsafe" 5 | ) 6 | 7 | // BytesReverse 反转字节数组, 此操作会修改原值 8 | func BytesReverse(b []byte) []byte { 9 | length := len(b) 10 | for i := 0; i < length/2; i++ { 11 | b[i], b[length-i-1] = b[length-i-1], b[i] 12 | } 13 | return b 14 | } 15 | 16 | // StringReverse 反转字符串, 此操作不会修改原值 17 | func StringReverse(s string) string { 18 | newBytes := make([]byte, len(s)) 19 | copy(newBytes, s) 20 | b := BytesReverse(newBytes) 21 | return *(*string)(unsafe.Pointer(&b)) 22 | } 23 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/iikira/iikira-go-utils 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/GeertJohan/go.incremental v1.0.0 7 | github.com/dgrijalva/jwt-go v3.2.0+incompatible 8 | github.com/fatih/color v1.10.0 9 | github.com/golang/protobuf v1.4.3 10 | github.com/json-iterator/go v1.1.10 11 | github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 12 | github.com/mattn/go-runewidth v0.0.10 13 | github.com/oleiade/lane v1.0.1 14 | golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad 15 | golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78 16 | ) 17 | -------------------------------------------------------------------------------- /utils/cachepool/syncpool.go: -------------------------------------------------------------------------------- 1 | package cachepool 2 | 3 | import ( 4 | "github.com/iikira/iikira-go-utils/utils/converter" 5 | "runtime" 6 | "sync" 7 | ) 8 | 9 | var ( 10 | syncPoolSize = int(64 * converter.KB) 11 | syncPoolFirstNew = false 12 | SyncPool = sync.Pool{ 13 | New: func() interface{} { 14 | syncPoolFirstNew = true 15 | return RawMallocByteSlice(syncPoolSize) 16 | }, 17 | } 18 | ) 19 | 20 | func SetSyncPoolSize(size int) { 21 | if syncPoolFirstNew && size != syncPoolSize { 22 | runtime.GC() 23 | } 24 | syncPoolSize = size 25 | } 26 | -------------------------------------------------------------------------------- /requester/downloader/writer.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "io" 5 | "os" 6 | ) 7 | 8 | type ( 9 | // Fder 获取fd接口 10 | Fder interface { 11 | Fd() uintptr 12 | } 13 | 14 | // Writer 下载器数据输出接口 15 | Writer interface { 16 | io.WriterAt 17 | } 18 | ) 19 | 20 | // NewDownloaderWriterByFilename 创建下载器数据输出接口, 类似于os.OpenFile 21 | func NewDownloaderWriterByFilename(name string, flag int, perm os.FileMode) (writer Writer, file *os.File, err error) { 22 | file, err = os.OpenFile(name, flag, perm) 23 | if err != nil { 24 | return 25 | } 26 | 27 | writer = file 28 | return 29 | } 30 | -------------------------------------------------------------------------------- /pcsverbose/utils.go: -------------------------------------------------------------------------------- 1 | package pcsverbose 2 | 3 | import ( 4 | "fmt" 5 | "github.com/iikira/iikira-go-utils/utils/csttime" 6 | "io" 7 | "io/ioutil" 8 | ) 9 | 10 | //PrintReader 输出Reader 11 | func PrintReader(r io.Reader) { 12 | b, _ := ioutil.ReadAll(r) 13 | fmt.Printf("%s\n", b) 14 | } 15 | 16 | // PrintArgs 输出字符串数组 17 | func PrintArgs(w io.Writer, args ...string) { 18 | for k, arg := range args { 19 | io.WriteString(w, fmt.Sprintf("args[%d] = `%s`, ", k, arg)) 20 | } 21 | w.Write([]byte{'\n'}) 22 | } 23 | 24 | func TimePrefix() string { 25 | return "[" + csttime.BeijingTimeOption("Refer") + "]" 26 | } 27 | -------------------------------------------------------------------------------- /utils/bdcrypto/base64.go: -------------------------------------------------------------------------------- 1 | package bdcrypto 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "io/ioutil" 7 | ) 8 | 9 | // Base64Encode base64加密 10 | func Base64Encode(raw []byte) []byte { 11 | var encoded bytes.Buffer 12 | encoder := base64.NewEncoder(base64.StdEncoding, &encoded) 13 | encoder.Write(raw) 14 | encoder.Close() 15 | return encoded.Bytes() 16 | } 17 | 18 | // Base64Decode base64解密 19 | func Base64Decode(raw []byte) []byte { 20 | buf := bytes.NewReader(raw) 21 | decoder := base64.NewDecoder(base64.StdEncoding, buf) 22 | decoded, _ := ioutil.ReadAll(decoder) 23 | return decoded 24 | } 25 | -------------------------------------------------------------------------------- /utils/log_colorable_prefix.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "github.com/fatih/color" 6 | "github.com/iikira/iikira-go-utils/utils/csttime" 7 | "log" 8 | ) 9 | 10 | var ( 11 | // ErrorColor 设置输出错误的颜色 12 | ErrorColor = color.New(color.FgRed).SprintFunc() 13 | ) 14 | 15 | // 自定义log writer 16 | type logWriter struct{} 17 | 18 | func (logWriter) Write(bytes []byte) (int, error) { 19 | return fmt.Fprint(color.Output, "["+csttime.BeijingTimeOption("Refer")+"] "+string(bytes)) 20 | } 21 | 22 | // SetLogPrefix 设置日志输出的时间前缀 23 | func SetLogPrefix() { 24 | log.SetFlags(0) 25 | log.SetOutput(new(logWriter)) 26 | } 27 | -------------------------------------------------------------------------------- /requester/transfer/transfer.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package transfer; 4 | 5 | // RangeGenMode 线程分配方式 6 | enum RangeGenMode { 7 | // RangeGenMode_Default 根据parallel平均生成 8 | Default = 0; 9 | // RangeGenMode_BlockSize 根据blockSize生成 10 | BlockSize = 1; 11 | } 12 | 13 | //Range 请求范围 14 | message Range { 15 | int64 begin = 1; 16 | int64 end = 2; 17 | } 18 | 19 | // DownloadInstanceInfoExport 断点续传 20 | message DownloadInstanceInfoExport { 21 | RangeGenMode range_gen_mode = 1; 22 | int64 total_size = 2; // 总大小 23 | int64 gen_begin = 3; 24 | int64 block_size = 4; 25 | repeated Range ranges = 5; 26 | } 27 | -------------------------------------------------------------------------------- /utils/taskframework/taskinfo.go: -------------------------------------------------------------------------------- 1 | package taskframework 2 | 3 | type ( 4 | TaskInfo struct { 5 | id string 6 | maxRetry int 7 | retry int 8 | } 9 | 10 | TaskInfoItem struct { 11 | Info *TaskInfo 12 | Unit TaskUnit 13 | } 14 | ) 15 | 16 | // IsExceedRetry 重试次数达到限制 17 | func (t *TaskInfo) IsExceedRetry() bool { 18 | return t.retry >= t.maxRetry 19 | } 20 | 21 | func (t *TaskInfo) Id() string { 22 | return t.id 23 | } 24 | 25 | func (t *TaskInfo) MaxRetry() int { 26 | return t.maxRetry 27 | } 28 | 29 | func (t *TaskInfo) SetMaxRetry(maxRetry int) { 30 | t.maxRetry = maxRetry 31 | } 32 | 33 | func (t *TaskInfo) Retry() int { 34 | return t.retry 35 | } 36 | -------------------------------------------------------------------------------- /utils/cachepool/malloc_test.go: -------------------------------------------------------------------------------- 1 | package cachepool_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/iikira/iikira-go-utils/utils/cachepool" 6 | "reflect" 7 | "runtime" 8 | "testing" 9 | "unsafe" 10 | ) 11 | 12 | func TestMalloc(t *testing.T) { 13 | b := cachepool.RawMallocByteSlice(128) 14 | for k := range b { 15 | b[k] = byte(k) 16 | } 17 | fmt.Println(b) 18 | runtime.GC() 19 | 20 | b = cachepool.RawMallocByteSlice(128) 21 | fmt.Printf("---%s---\n", b) 22 | runtime.GC() 23 | 24 | b = cachepool.RawByteSlice(128) 25 | fmt.Println(b) 26 | runtime.GC() 27 | 28 | b = cachepool.RawByteSlice(127) 29 | bH := (*reflect.SliceHeader)(unsafe.Pointer(&b)) 30 | fmt.Printf("%#v\n", bH) 31 | } 32 | -------------------------------------------------------------------------------- /requester/util.go: -------------------------------------------------------------------------------- 1 | package requester 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strings" 7 | ) 8 | 9 | // ParseCookieStr 解析 Cookie 字符串 10 | func ParseCookieStr(cookieStr string) []*http.Cookie { 11 | rawCookies := strings.SplitN(cookieStr, ";", -1) 12 | cookies := make([]*http.Cookie, 0, len(rawCookies)) 13 | 14 | for _, rawCookie := range rawCookies { 15 | s2 := strings.SplitN(rawCookie, "=", 2) 16 | if len(s2) < 2 { 17 | fmt.Println(s2) 18 | continue 19 | } 20 | 21 | s2[0] = strings.TrimSpace(s2[0]) 22 | s2[1] = strings.TrimSpace(s2[1]) 23 | 24 | cookies = append(cookies, &http.Cookie{ 25 | Name: s2[0], 26 | Value: s2[1], 27 | }) 28 | } 29 | return cookies 30 | } 31 | -------------------------------------------------------------------------------- /pcsverbose/pcsdebug/cpu.go: -------------------------------------------------------------------------------- 1 | // Package pcsdebug 调试包 2 | package pcsdebug 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "os" 8 | "runtime/pprof" 9 | ) 10 | 11 | //StartCPUProfile 收集cpu信息 12 | func StartCPUProfile(ctx context.Context, cpuProfile string) { 13 | if cpuProfile != "" { 14 | f, err := os.Create(cpuProfile) 15 | if err != nil { 16 | fmt.Fprintf(os.Stderr, "Can not create cpu profile output file: %s", err) 17 | return 18 | } 19 | if err := pprof.StartCPUProfile(f); err != nil { 20 | fmt.Fprintf(os.Stderr, "Can not start cpu profile: %s", err) 21 | f.Close() 22 | return 23 | } 24 | defer pprof.StopCPUProfile() 25 | } 26 | select { 27 | case <-ctx.Done(): 28 | return 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /requester/rio/speeds/ratelimit_test.go: -------------------------------------------------------------------------------- 1 | package speeds_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/iikira/iikira-go-utils/requester/rio/speeds" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestRateLimit(t *testing.T) { 11 | r := speeds.NewRateLimit(100) 12 | fmt.Println("adding 101...") 13 | r.Add(101) 14 | fmt.Println("adding 10...") 15 | r.Add(10) 16 | fmt.Println("adding 11...") 17 | r.Add(11) 18 | fmt.Println("adding 12...") 19 | r.Add(12) 20 | fmt.Println("adding 13...") 21 | r.Add(13) 22 | fmt.Println("adding 22...") 23 | r.Add(22) 24 | fmt.Println("adding 35...") 25 | r.Add(35) 26 | fmt.Println("adding 25...") 27 | r.Add(25) 28 | fmt.Println("adding 11...") 29 | r.Add(11) 30 | 31 | r.Stop() 32 | time.Sleep(10e9) 33 | } 34 | -------------------------------------------------------------------------------- /utils/addr.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "net" 5 | ) 6 | 7 | // ListAddresses 列出本地可用的 IP 地址 8 | func ListAddresses() (addresses []string) { 9 | iFaces, _ := net.Interfaces() 10 | addresses = make([]string, 0, len(iFaces)) 11 | for k := range iFaces { 12 | iFaceAddrs, _ := iFaces[k].Addrs() 13 | for l := range iFaceAddrs { 14 | switch v := iFaceAddrs[l].(type) { 15 | case *net.IPNet: 16 | addresses = append(addresses, v.IP.String()) 17 | case *net.IPAddr: 18 | addresses = append(addresses, v.IP.String()) 19 | } 20 | } 21 | } 22 | return 23 | } 24 | 25 | // ParseHost 解析地址中的host 26 | func ParseHost(address string) string { 27 | h, _, err := net.SplitHostPort(address) 28 | if err != nil { 29 | return address 30 | } 31 | return h 32 | } 33 | -------------------------------------------------------------------------------- /utils/checksum/checksum_test.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | var ( 8 | flagList = []int{ 9 | CHECKSUM_MD5 | 000000000000000000000000000 | 00000000000000000000000, 10 | 000000000000000000000 | CHECKSUM_SLICE_MD5 | 00000000000000000000000, 11 | 000000000000000000000 | 000000000000000000000000000 | CHECKSUM_CRC32, 12 | CHECKSUM_MD5 | CHECKSUM_SLICE_MD5 | 00000000000000000000000, 13 | 000000000000000000000 | CHECKSUM_SLICE_MD5 | CHECKSUM_CRC32, 14 | CHECKSUM_MD5 | 000000000000000000000000000 | CHECKSUM_CRC32, 15 | CHECKSUM_MD5 | CHECKSUM_SLICE_MD5 | CHECKSUM_CRC32, 16 | } 17 | ) 18 | 19 | func printFileMeta(meta *LocalFileMeta) { 20 | fmt.Printf("slicemd5: %x, md5: %x, crc32: %x %d\n", meta.SliceMD5, meta.MD5, meta.CRC32, meta.CRC32) 21 | } 22 | -------------------------------------------------------------------------------- /requester/requester.go: -------------------------------------------------------------------------------- 1 | // Package requester 提供网络请求简便操作 2 | package requester 3 | 4 | const ( 5 | // DefaultUserAgent 默认浏览器标识 6 | DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36" 7 | ) 8 | 9 | var ( 10 | // UserAgent 浏览器标识 11 | UserAgent = DefaultUserAgent 12 | // DefaultClient 默认 http 客户端 13 | DefaultClient = NewHTTPClient() 14 | ) 15 | 16 | type ( 17 | // ContentTyper Content-Type 接口 18 | ContentTyper interface { 19 | ContentType() string 20 | } 21 | 22 | // ContentLengther Content-Length 接口 23 | ContentLengther interface { 24 | ContentLength() int64 25 | } 26 | 27 | // Event 下载/上传任务运行时事件 28 | Event func() 29 | 30 | // EventOnError 任务出错运行时事件 31 | EventOnError func(err error) 32 | ) 33 | -------------------------------------------------------------------------------- /requester/uploader/readed.go: -------------------------------------------------------------------------------- 1 | package uploader 2 | 3 | import ( 4 | "github.com/iikira/iikira-go-utils/requester/rio" 5 | "sync/atomic" 6 | ) 7 | 8 | type ( 9 | // Readed64 增加获取已读取数据量, 用于统计速度 10 | Readed64 interface { 11 | rio.ReaderLen64 12 | Readed() int64 13 | } 14 | 15 | readed64 struct { 16 | readed int64 17 | rio.ReaderLen64 18 | } 19 | ) 20 | 21 | // NewReaded64 实现Readed64接口 22 | func NewReaded64(rl rio.ReaderLen64) Readed64 { 23 | return &readed64{ 24 | readed: 0, 25 | ReaderLen64: rl, 26 | } 27 | } 28 | 29 | func (r64 *readed64) Read(p []byte) (n int, err error) { 30 | n, err = r64.ReaderLen64.Read(p) 31 | atomic.AddInt64(&r64.readed, int64(n)) 32 | return n, err 33 | } 34 | 35 | func (r64 *readed64) Readed() int64 { 36 | return atomic.LoadInt64(&r64.readed) 37 | } 38 | -------------------------------------------------------------------------------- /requester/rio/buf.go: -------------------------------------------------------------------------------- 1 | package rio 2 | 3 | import ( 4 | "unsafe" 5 | ) 6 | 7 | // Buffer 为固定长度的 Buf, 实现 io.WriterAt 接口 8 | type Buffer struct { 9 | Buf []byte 10 | } 11 | 12 | // NewBuffer 初始化 Buffer 13 | func NewBuffer(buf []byte) *Buffer { 14 | return &Buffer{ 15 | Buf: buf, 16 | } 17 | } 18 | 19 | // ReadAt 实现 io.ReadAt 接口 20 | // 不进行越界检查 21 | func (b *Buffer) ReadAt(p []byte, off int64) (n int, err error) { 22 | n = copy(p, b.Buf[off:]) 23 | return n, nil 24 | } 25 | 26 | // WriteAt 实现 io.WriterAt 接口 27 | // 不进行越界检查 28 | func (b *Buffer) WriteAt(p []byte, off int64) (n int, err error) { 29 | n = copy(b.Buf[off:], p) 30 | return n, nil 31 | } 32 | 33 | // Bytes 返回 buf 34 | func (b *Buffer) Bytes() []byte { 35 | return b.Buf 36 | } 37 | 38 | func (b *Buffer) String() string { 39 | return *(*string)(unsafe.Pointer(&b.Buf)) 40 | } 41 | -------------------------------------------------------------------------------- /utils/getip/getip.go: -------------------------------------------------------------------------------- 1 | // Package getip 获取 ip 信息包 2 | package getip 3 | 4 | import ( 5 | "github.com/iikira/iikira-go-utils/requester" 6 | "net" 7 | "net/http" 8 | "unsafe" 9 | ) 10 | 11 | // IPInfoByClient 给定client获取ip地址 12 | func IPInfoByClient(c *requester.HTTPClient) (ipAddr string, err error) { 13 | if c == nil { 14 | c = requester.NewHTTPClient() 15 | } 16 | 17 | body, err := c.Fetch(http.MethodGet, "https://api.ipify.org", nil, nil) 18 | if err != nil { 19 | return 20 | } 21 | 22 | ipAddr = *(*string)(unsafe.Pointer(&body)) 23 | ip := net.ParseIP(ipAddr) 24 | if ip == nil { 25 | return "", ErrParseIP 26 | } 27 | return 28 | } 29 | 30 | //IPInfo 从ipify获取IP地址 31 | func IPInfo(https bool) (ipAddr string, err error) { 32 | c := requester.NewHTTPClient() 33 | c.SetHTTPSecure(https) 34 | return IPInfoByClient(c) 35 | } 36 | -------------------------------------------------------------------------------- /utils/bdcrypto/hmac.go: -------------------------------------------------------------------------------- 1 | package bdcrypto 2 | 3 | import ( 4 | "crypto/hmac" 5 | "crypto/md5" 6 | "crypto/sha1" 7 | "crypto/sha256" 8 | "crypto/sha512" 9 | ) 10 | 11 | // HmacSHA1 HMAC-SHA-1签名认证 12 | func HmacSHA1(key, origData []byte) (sum []byte) { 13 | mac := hmac.New(sha1.New, key) 14 | mac.Write(origData) 15 | return mac.Sum(nil) 16 | } 17 | 18 | // HmacSHA256 HMAC-SHA-256签名认证 19 | func HmacSHA256(key, origData []byte) (sum []byte) { 20 | mac := hmac.New(sha256.New, key) 21 | mac.Write(origData) 22 | return mac.Sum(nil) 23 | } 24 | 25 | // HmacSHA512 HMAC-SHA-512签名认证 26 | func HmacSHA512(key, origData []byte) (sum []byte) { 27 | mac := hmac.New(sha512.New, key) 28 | mac.Write(origData) 29 | return mac.Sum(nil) 30 | } 31 | 32 | // HmacMD5 HMAC-SHA512-签名认证 33 | func HmacMD5(key, origData []byte) (sum []byte) { 34 | mac := hmac.New(md5.New, key) 35 | mac.Write(origData) 36 | return mac.Sum(nil) 37 | } 38 | -------------------------------------------------------------------------------- /utils/cachepool/malloc.go: -------------------------------------------------------------------------------- 1 | package cachepool 2 | 3 | import ( 4 | "reflect" 5 | "unsafe" 6 | ) 7 | 8 | //go:linkname mallocgc runtime.mallocgc 9 | func mallocgc(size uintptr, typ uintptr, needzero bool) unsafe.Pointer 10 | 11 | //go:linkname rawbyteslice runtime.rawbyteslice 12 | func rawbyteslice(size int) (b []byte) 13 | 14 | // RawByteSlice point to runtime.rawbyteslice 15 | func RawByteSlice(size int) (b []byte) { 16 | return rawbyteslice(size) 17 | } 18 | 19 | // RawMalloc allocates a new slice. The slice is not zeroed. 20 | func RawMalloc(size int) unsafe.Pointer { 21 | return mallocgc(uintptr(size), 0, false) 22 | } 23 | 24 | // RawMallocByteSlice allocates a new byte slice. The slice is not zeroed. 25 | func RawMallocByteSlice(size int) []byte { 26 | p := mallocgc(uintptr(size), 0, false) 27 | b := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ 28 | Data: uintptr(p), 29 | Len: size, 30 | Cap: size, 31 | })) 32 | return b 33 | } 34 | -------------------------------------------------------------------------------- /requester/uploader/example.go: -------------------------------------------------------------------------------- 1 | package uploader 2 | 3 | import ( 4 | "fmt" 5 | "github.com/iikira/iikira-go-utils/requester/rio" 6 | "github.com/iikira/iikira-go-utils/utils/converter" 7 | ) 8 | 9 | // DoUpload 执行上传 10 | func DoUpload(uploadURL string, readerlen64 rio.ReaderLen64, checkFunc CheckFunc) { 11 | u := NewUploader(uploadURL, readerlen64) 12 | u.SetCheckFunc(checkFunc) 13 | 14 | exitChan := make(chan struct{}) 15 | 16 | u.OnExecute(func() { 17 | statusChan := u.GetStatusChan() 18 | for { 19 | select { 20 | case <-exitChan: 21 | return 22 | case v, ok := <-statusChan: 23 | if !ok { 24 | return 25 | } 26 | 27 | fmt.Printf("\r ↑ %s/%s %s/s in %s ............", 28 | converter.ConvertFileSize(v.Uploaded(), 2), 29 | converter.ConvertFileSize(v.TotalSize(), 2), 30 | converter.ConvertFileSize(v.SpeedsPerSecond(), 2), 31 | v.TimeElapsed(), 32 | ) 33 | } 34 | } 35 | }) 36 | 37 | u.Execute() 38 | close(exitChan) 39 | 40 | return 41 | } 42 | -------------------------------------------------------------------------------- /utils/expires/cachemap/cachemap.go: -------------------------------------------------------------------------------- 1 | package cachemap 2 | 3 | import ( 4 | "github.com/iikira/iikira-go-utils/utils/expires" 5 | "sync" 6 | ) 7 | 8 | var ( 9 | GlobalCacheOpMap = CacheOpMap{} 10 | ) 11 | 12 | type ( 13 | CacheOpMap struct { 14 | cachePool sync.Map 15 | } 16 | ) 17 | 18 | func (cm *CacheOpMap) LazyInitCachePoolOp(op string) CacheUnit { 19 | cacheItf, _ := cm.cachePool.LoadOrStore(op, &cacheUnit{}) 20 | return cacheItf.(CacheUnit) 21 | } 22 | 23 | func (cm *CacheOpMap) RemoveCachePoolOp(op string) { 24 | cm.cachePool.Delete(op) 25 | } 26 | 27 | // ClearInvalidate 清除已过期的数据(一般用不到) 28 | func (cm *CacheOpMap) ClearInvalidate() { 29 | cm.cachePool.Range(func(_, cacheItf interface{}) bool { 30 | cache := cacheItf.(CacheUnit) 31 | cache.Range(func(key interface{}, exp expires.DataExpires) bool { 32 | if exp.IsExpires() { 33 | cache.Delete(key) 34 | } 35 | return true 36 | }) 37 | return true 38 | }) 39 | } 40 | 41 | // PrintAll 输出所有缓冲项目 42 | func (cm *CacheOpMap) PrintAll() { 43 | } 44 | -------------------------------------------------------------------------------- /requester/rio/rio.go: -------------------------------------------------------------------------------- 1 | // Package rio rquester io 工具包 2 | package rio 3 | 4 | import ( 5 | "io" 6 | ) 7 | 8 | type ( 9 | // Lener 返回32-bit长度接口 10 | Lener interface { 11 | Len() int 12 | } 13 | 14 | // Lener64 返回64-bit长度接口 15 | Lener64 interface { 16 | Len() int64 17 | } 18 | 19 | // ReaderLen 实现io.Reader和32-bit长度接口 20 | ReaderLen interface { 21 | io.Reader 22 | Lener 23 | } 24 | 25 | // ReaderLen64 实现io.Reader和64-bit长度接口 26 | ReaderLen64 interface { 27 | io.Reader 28 | Lener64 29 | } 30 | 31 | // ReaderAtLen64 实现io.ReaderAt和64-bit长度接口 32 | ReaderAtLen64 interface { 33 | io.ReaderAt 34 | Lener64 35 | } 36 | 37 | // WriterLen64 实现io.Writer和64-bit长度接口 38 | WriterLen64 interface { 39 | io.Writer 40 | Lener64 41 | } 42 | 43 | // WriteCloserAt 实现io.WriteCloser和io.WriterAt接口 44 | WriteCloserAt interface { 45 | io.WriteCloser 46 | io.WriterAt 47 | } 48 | 49 | // WriteCloserLen64At 实现rio.WriteCloserAt和64-bit长度接口 50 | WriteCloserLen64At interface { 51 | WriteCloserAt 52 | Lener64 53 | } 54 | ) 55 | -------------------------------------------------------------------------------- /utils/taskframework/task_unit.go: -------------------------------------------------------------------------------- 1 | package taskframework 2 | 3 | import "time" 4 | 5 | type ( 6 | TaskUnit interface { 7 | SetTaskInfo(info *TaskInfo) 8 | // 执行任务 9 | Run() (result *TaskUnitRunResult) 10 | // 重试任务执行的方法 11 | // 当达到最大重试次数, 执行失败 12 | OnRetry(lastRunResult *TaskUnitRunResult) 13 | // 每次执行成功执行的方法 14 | OnSuccess(lastRunResult *TaskUnitRunResult) 15 | // 每次执行失败执行的方法 16 | OnFailed(lastRunResult *TaskUnitRunResult) 17 | // 每次执行结束执行的方法, 不管成功失败 18 | OnComplete(lastRunResult *TaskUnitRunResult) 19 | // 重试等待的时间 20 | RetryWait() time.Duration 21 | } 22 | 23 | // 任务单元执行结果 24 | TaskUnitRunResult struct { 25 | Succeed bool // 是否执行成功 26 | NeedRetry bool // 是否需要重试 27 | 28 | // 以下是额外的信息 29 | Err error // 错误信息 30 | ResultCode int // 结果代码 31 | ResultMessage string // 结果描述 32 | Extra interface{} // 额外的信息 33 | } 34 | ) 35 | 36 | var ( 37 | // TaskUnitRunResultSuccess 任务执行成功 38 | TaskUnitRunResultSuccess = &TaskUnitRunResult{} 39 | ) 40 | -------------------------------------------------------------------------------- /utils/bdcrypto/aes_test.go: -------------------------------------------------------------------------------- 1 | package bdcrypto 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | var ( 11 | bkey = []byte("asfasfawefwfgagasfasfawefwfgag") 12 | plaintext = "1111" 13 | ) 14 | 15 | func TestAesStream(t *testing.T) { 16 | var key [32]byte 17 | copy(key[:], bkey) 18 | 19 | rd, err := Aes256CFBEncrypt(key, strings.NewReader(plaintext)) 20 | if err != nil { 21 | t.Error(err) 22 | } 23 | 24 | by, _ := ioutil.ReadAll(rd) 25 | 26 | t.Log("encd:", by) 27 | 28 | rd, err = Aes256CFBDecrypt(key, bytes.NewReader(by)) 29 | by, _ = ioutil.ReadAll(rd) 30 | t.Log("decd:", by) 31 | 32 | } 33 | 34 | func TestAesBlock(t *testing.T) { 35 | var key [16]byte 36 | copy(key[:], bkey) 37 | 38 | ciphertext, err := Aes128ECBEncrypt(key, []byte(plaintext)) 39 | if err != nil { 40 | t.Log(err) 41 | } 42 | 43 | t.Log("encd:", ciphertext) 44 | 45 | plain, err := Aes128ECBDecrypt(key, ciphertext) 46 | if err != nil { 47 | t.Log(err) 48 | } 49 | 50 | t.Log("decd:", plain) 51 | } 52 | -------------------------------------------------------------------------------- /utils/getip/getip_techain.go: -------------------------------------------------------------------------------- 1 | package getip 2 | 3 | import ( 4 | "bytes" 5 | "github.com/iikira/iikira-go-utils/requester" 6 | "github.com/iikira/iikira-go-utils/utils/converter" 7 | "net/http" 8 | ) 9 | 10 | func IPInfoFromTechainBaiduByClient(c *requester.HTTPClient) (ipAddr string, err error) { 11 | body, err := c.Fetch(http.MethodGet, "https://techain.baidu.com/srcmon", nil, map[string]string{ 12 | "User-Agent": "x18/600000101/10.0.63/4.1.3", 13 | "Pragma": "no-cache", 14 | "Accept": "*/*", 15 | "Content-Type": "application/x-www-form-urlencoded", 16 | "x-auth-ver": "1", 17 | "Accept-Language": "zh-CN", 18 | "x-device-id": "00000000000000000000000000000000", 19 | }) 20 | if err != nil { 21 | return 22 | } 23 | return converter.ToString(bytes.TrimSpace(body)), nil 24 | } 25 | 26 | // IPInfoFromTechainBaidu 从 techain.baidu.com 获取ip 27 | func IPInfoFromTechainBaidu() (ipAddr string, err error) { 28 | c := requester.NewHTTPClient() 29 | return IPInfoFromTechainBaiduByClient(c) 30 | } 31 | -------------------------------------------------------------------------------- /utils/checksum/file.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import ( 4 | "bytes" 5 | "path/filepath" 6 | ) 7 | 8 | // EqualLengthMD5 检测md5和大小是否相同 9 | func (lfm *LocalFileMeta) EqualLengthMD5(m *LocalFileMeta) bool { 10 | if lfm.Length != m.Length { 11 | return false 12 | } 13 | if bytes.Compare(lfm.MD5, m.MD5) != 0 { 14 | return false 15 | } 16 | return true 17 | } 18 | 19 | // CompleteAbsPath 补齐绝对路径 20 | func (lfm *LocalFileMeta) CompleteAbsPath() { 21 | if filepath.IsAbs(lfm.Path) { 22 | return 23 | } 24 | 25 | absPath, err := filepath.Abs(lfm.Path) 26 | if err != nil { 27 | return 28 | } 29 | 30 | lfm.Path = absPath 31 | } 32 | 33 | // GetFileSum 获取文件的大小, md5, 前256KB切片的 md5, crc32 34 | func GetFileSum(localPath string, flag int) (lfc *LocalFileChecksum, err error) { 35 | lfc = NewLocalFileChecksum(localPath, 256*1024) 36 | defer lfc.Close() 37 | 38 | err = lfc.OpenPath() 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | err = lfc.Sum(flag) 44 | if err != nil { 45 | return nil, err 46 | } 47 | return lfc, nil 48 | } 49 | -------------------------------------------------------------------------------- /requester/uploader/block_test.go: -------------------------------------------------------------------------------- 1 | package uploader_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/iikira/iikira-go-utils/requester/rio" 6 | "github.com/iikira/iikira-go-utils/requester/transfer" 7 | "github.com/iikira/iikira-go-utils/requester/uploader" 8 | "github.com/iikira/iikira-go-utils/utils/cachepool" 9 | "io" 10 | "testing" 11 | ) 12 | 13 | var ( 14 | blockList = uploader.SplitBlock(10000, 999) 15 | ) 16 | 17 | func TestSplitBlock(t *testing.T) { 18 | for k, e := range blockList { 19 | fmt.Printf("%d %#v\n", k, e) 20 | } 21 | } 22 | 23 | func TestSplitUnitRead(t *testing.T) { 24 | var size int64 = 65536*2 + 3432 25 | buffer := rio.NewBuffer(cachepool.RawMallocByteSlice(int(size))) 26 | unit := uploader.NewBufioSplitUnit(buffer, transfer.Range{Begin: 2, End: size}, nil, nil) 27 | 28 | buf := cachepool.RawMallocByteSlice(1022) 29 | for { 30 | n, err := unit.Read(buf) 31 | if err != nil { 32 | if err == io.EOF { 33 | break 34 | } 35 | t.Fatalf("read error: %s\n", err) 36 | } 37 | fmt.Printf("n: %d, left: %d\n", n, unit.Left()) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /utils/getip/getip_netease.go: -------------------------------------------------------------------------------- 1 | package getip 2 | 3 | import ( 4 | "github.com/iikira/iikira-go-utils/requester" 5 | "github.com/iikira/iikira-go-utils/utils/jsonhelper" 6 | "net" 7 | "net/http" 8 | ) 9 | 10 | type ( 11 | // IPResNetease 网易服务器获取ip返回的结果 12 | IPResNetease struct { 13 | Result string `json:"result"` 14 | Code int `json:"code"` 15 | Message string `json:"message"` 16 | } 17 | ) 18 | 19 | func IPInfoFromNeteaseByClient(c *requester.HTTPClient) (ipAddr string, err error) { 20 | resp, err := c.Req(http.MethodGet, "http://mam.netease.com/api/config/getClientIp", nil, nil) 21 | if resp != nil { 22 | defer resp.Body.Close() 23 | } 24 | if err != nil { 25 | return 26 | } 27 | 28 | res := &IPResNetease{} 29 | err = jsonhelper.UnmarshalData(resp.Body, res) 30 | if err != nil { 31 | return 32 | } 33 | 34 | ip := net.ParseIP(res.Result) 35 | if ip == nil { 36 | err = ErrParseIP 37 | return 38 | } 39 | 40 | ipAddr = res.Result 41 | return 42 | } 43 | 44 | // IPInfoFromNetease 从网易服务器获取ip 45 | func IPInfoFromNetease() (ipAddr string, err error) { 46 | c := requester.NewHTTPClient() 47 | return IPInfoFromNeteaseByClient(c) 48 | } 49 | -------------------------------------------------------------------------------- /utils/waitgroup/wait_group.go: -------------------------------------------------------------------------------- 1 | // Package waitgroup sync.WaitGroup extension 2 | package waitgroup 3 | 4 | import "sync" 5 | 6 | // WaitGroup 在 sync.WaitGroup 的基础上, 新增线程控制功能 7 | type WaitGroup struct { 8 | wg sync.WaitGroup 9 | p chan struct{} 10 | 11 | sync.RWMutex 12 | } 13 | 14 | // NewWaitGroup returns a pointer to a new `WaitGroup` object. 15 | // parallel 为最大并发数, 0 代表无限制 16 | func NewWaitGroup(parallel int) (w *WaitGroup) { 17 | w = &WaitGroup{ 18 | wg: sync.WaitGroup{}, 19 | } 20 | 21 | if parallel <= 0 { 22 | return 23 | } 24 | 25 | w.p = make(chan struct{}, parallel) 26 | return 27 | } 28 | 29 | // AddDelta sync.WaitGroup.Add(1) 30 | func (w *WaitGroup) AddDelta() { 31 | if w.p != nil { 32 | w.p <- struct{}{} 33 | } 34 | 35 | w.wg.Add(1) 36 | } 37 | 38 | // Done sync.WaitGroup.Done() 39 | func (w *WaitGroup) Done() { 40 | w.wg.Done() 41 | 42 | if w.p != nil { 43 | <-w.p 44 | } 45 | } 46 | 47 | // Wait 参照 sync.WaitGroup 的 Wait 方法 48 | func (w *WaitGroup) Wait() { 49 | w.wg.Wait() 50 | if w.p != nil { 51 | close(w.p) 52 | } 53 | } 54 | 55 | // Parallel 返回当前正在进行的任务数量 56 | func (w *WaitGroup) Parallel() int { 57 | return len(w.p) 58 | } 59 | -------------------------------------------------------------------------------- /requester/downloader/resetcontroler.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "github.com/iikira/iikira-go-utils/utils/expires" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | // ResetController 网络连接控制器 10 | type ResetController struct { 11 | mu sync.Mutex 12 | currentTime time.Time 13 | maxResetNum int 14 | resetEntity map[expires.Expires]struct{} 15 | } 16 | 17 | // NewResetController 初始化*ResetController 18 | func NewResetController(maxResetNum int) *ResetController { 19 | return &ResetController{ 20 | currentTime: time.Now(), 21 | maxResetNum: maxResetNum, 22 | resetEntity: map[expires.Expires]struct{}{}, 23 | } 24 | } 25 | 26 | func (rc *ResetController) update() { 27 | for k := range rc.resetEntity { 28 | if k.IsExpires() { 29 | delete(rc.resetEntity, k) 30 | } 31 | } 32 | } 33 | 34 | // AddResetNum 增加连接 35 | func (rc *ResetController) AddResetNum() { 36 | rc.mu.Lock() 37 | defer rc.mu.Unlock() 38 | rc.update() 39 | rc.resetEntity[expires.NewExpires(9*time.Second)] = struct{}{} 40 | } 41 | 42 | // CanReset 是否可以建立连接 43 | func (rc *ResetController) CanReset() bool { 44 | rc.mu.Lock() 45 | defer rc.mu.Unlock() 46 | rc.update() 47 | return len(rc.resetEntity) < rc.maxResetNum 48 | } 49 | -------------------------------------------------------------------------------- /utils/bdcrypto/bdcrypto_test.go: -------------------------------------------------------------------------------- 1 | package bdcrypto 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestRSAEncryptOfWapBaidu(t *testing.T) { 9 | b, _ := RSAEncryptOfWapBaidu(DefaultRSAPublicKeyModulus, []byte("123")) 10 | fmt.Println(b) 11 | fmt.Println("-------------------") 12 | } 13 | 14 | func BenchmarkRSAEncryptOfWapBaidu(b *testing.B) { 15 | var by = []byte("Pythonphp123sdif8e83") 16 | for i := 0; i < b.N; i++ { 17 | RSAEncryptOfWapBaidu(DefaultRSAPublicKeyModulus, by) 18 | } 19 | } 20 | 21 | func TestBase64(t *testing.T) { 22 | var b = []byte("12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890") 23 | en := Base64Encode(b) 24 | fmt.Println(string(en)) 25 | de := Base64Decode(en) 26 | fmt.Println(string(de)) 27 | } 28 | 29 | func TestReverse(t *testing.T) { 30 | var b = []byte("1234567890") 31 | fmt.Println(string(BytesReverse(b))) 32 | } 33 | 34 | func BenchmarkReverse(b *testing.B) { 35 | var by = []byte("Pythonphp123sdif8e83") 36 | for i := 0; i < b.N; i++ { 37 | BytesReverse(by) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /utils/expires/expires.go: -------------------------------------------------------------------------------- 1 | package expires 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | _ "unsafe" // for go:linkname 7 | ) 8 | 9 | type ( 10 | Expires interface { 11 | IsExpires() bool 12 | GetExpires() time.Time 13 | SetExpires(e bool) 14 | fmt.Stringer 15 | } 16 | 17 | expires struct { 18 | expiresAt time.Time 19 | abort bool 20 | } 21 | ) 22 | 23 | //go:linkname stripMono time.(*Time).stripMono 24 | func stripMono(t *time.Time) 25 | 26 | // StripMono strip monotonic clocks 27 | func StripMono(t *time.Time) { 28 | stripMono(t) 29 | } 30 | 31 | func NewExpires(dur time.Duration) Expires { 32 | t := time.Now().Add(dur) 33 | StripMono(&t) 34 | return &expires{ 35 | expiresAt: t, 36 | } 37 | } 38 | 39 | func NewExpiresAt(at time.Time) Expires { 40 | StripMono(&at) 41 | return &expires{ 42 | expiresAt: at, 43 | } 44 | } 45 | 46 | func (ep *expires) GetExpires() time.Time { 47 | return ep.expiresAt 48 | } 49 | 50 | func (ep *expires) SetExpires(e bool) { 51 | ep.abort = e 52 | } 53 | 54 | func (ep *expires) IsExpires() bool { 55 | return ep.abort || time.Now().After(ep.expiresAt) 56 | } 57 | 58 | func (ep *expires) String() string { 59 | return fmt.Sprintf("expires at: %s, abort: %t", ep.expiresAt, ep.abort) 60 | } 61 | -------------------------------------------------------------------------------- /requester/rio/speeds/speeds.go: -------------------------------------------------------------------------------- 1 | // Package speeds 速度计算工具包 2 | package speeds 3 | 4 | import ( 5 | "sync" 6 | "sync/atomic" 7 | "time" 8 | ) 9 | 10 | type ( 11 | // Speeds 统计速度 12 | Speeds struct { 13 | count int64 14 | interval time.Duration // 刷新周期 15 | nowTime time.Time 16 | once sync.Once 17 | } 18 | ) 19 | 20 | func (sps *Speeds) initOnce() { 21 | sps.once.Do(func() { 22 | sps.nowTime = time.Now() 23 | if sps.interval <= 0 { 24 | sps.interval = 1 * time.Second 25 | } 26 | }) 27 | } 28 | 29 | // SetInterval 设置刷新周期 30 | func (sps *Speeds) SetInterval(interval time.Duration) { 31 | if interval <= 0 { 32 | return 33 | } 34 | sps.interval = interval 35 | } 36 | 37 | // Add 原子操作, 增加数据量 38 | func (sps *Speeds) Add(count int64) { 39 | // 初始化 40 | sps.initOnce() 41 | atomic.AddInt64(&sps.count, count) 42 | } 43 | 44 | // GetSpeeds 结束统计速度, 并返回速度 45 | func (sps *Speeds) GetSpeeds() (speeds int64) { 46 | sps.initOnce() 47 | 48 | since := time.Since(sps.nowTime) 49 | if since <= 0 { 50 | return 0 51 | } 52 | speeds = int64(float64(atomic.LoadInt64(&sps.count)) * sps.interval.Seconds() / since.Seconds()) 53 | 54 | // 更新下一轮 55 | if since >= sps.interval { 56 | atomic.StoreInt64(&sps.count, 0) 57 | sps.nowTime = time.Now() 58 | } 59 | return 60 | } 61 | -------------------------------------------------------------------------------- /utils/bdcrypto/3des.go: -------------------------------------------------------------------------------- 1 | package bdcrypto 2 | 3 | import ( 4 | "crypto/cipher" 5 | "crypto/des" 6 | "errors" 7 | ) 8 | 9 | // DESCBCEncrypt3 实现3DES加密, CBC模式 10 | func DESCBCEncrypt3(plaintext, key, iv []byte) (ciphertext []byte, err error) { 11 | block, err := des.NewTripleDESCipher(key) 12 | if err != nil { 13 | return nil, err 14 | } 15 | 16 | defer func() { 17 | if rerr := recover(); rerr != nil { 18 | err = errors.New(rerr.(string)) 19 | } 20 | }() 21 | 22 | bs := block.BlockSize() 23 | plaintext = PKCS5Padding(plaintext, bs) 24 | blockMode := cipher.NewCBCEncrypter(block, iv) 25 | 26 | ciphertext = make([]byte, len(plaintext)) 27 | blockMode.CryptBlocks(ciphertext, plaintext) 28 | return ciphertext, nil 29 | } 30 | 31 | // DESCBCDecrypt3 实现3DES解密, CBC模式 32 | func DESCBCDecrypt3(ciphertext, key, iv []byte) (plaintext []byte, err error) { 33 | block, err := des.NewTripleDESCipher(key) 34 | if err != nil { 35 | return nil, err 36 | } 37 | 38 | defer func() { 39 | if rerr := recover(); rerr != nil { 40 | err = errors.New(rerr.(string)) 41 | } 42 | }() 43 | 44 | blockMode := cipher.NewCBCDecrypter(block, iv) 45 | 46 | plaintext = make([]byte, len(ciphertext)) 47 | blockMode.CryptBlocks(plaintext, ciphertext) 48 | plaintext = PKCS5UnPadding(plaintext) 49 | return plaintext, nil 50 | } 51 | -------------------------------------------------------------------------------- /utils/expires/cachemap/utils.go: -------------------------------------------------------------------------------- 1 | package cachemap 2 | 3 | import ( 4 | "github.com/iikira/iikira-go-utils/utils/expires" 5 | ) 6 | 7 | type ( 8 | OpFunc func() expires.DataExpires 9 | OpFuncWithError func() (expires.DataExpires, error) 10 | ) 11 | 12 | func (cm *CacheOpMap) CacheOperation(op string, key interface{}, opFunc OpFunc) (data expires.DataExpires) { 13 | var ( 14 | cache = cm.LazyInitCachePoolOp(op) 15 | ok bool 16 | ) 17 | 18 | cache.LockKey(key) 19 | defer cache.UnlockKey(key) 20 | data, ok = cache.Load(key) 21 | if !ok { 22 | if opFunc == nil { 23 | return 24 | } 25 | data = opFunc() 26 | if data != nil { 27 | cache.Store(key, data) 28 | } 29 | return 30 | } 31 | 32 | return 33 | } 34 | 35 | func (cm *CacheOpMap) CacheOperationWithError(op string, key interface{}, opFunc OpFuncWithError) (data expires.DataExpires, err error) { 36 | var ( 37 | cache = cm.LazyInitCachePoolOp(op) 38 | ok bool 39 | ) 40 | 41 | cache.LockKey(key) 42 | defer cache.UnlockKey(key) 43 | data, ok = cache.Load(key) 44 | if !ok { 45 | if opFunc == nil { 46 | return 47 | } 48 | data, err = opFunc() 49 | if err != nil { 50 | return 51 | } 52 | if data == nil { 53 | // 数据为空时也不存 54 | return 55 | } 56 | cache.Store(key, data) 57 | } 58 | 59 | return 60 | } 61 | -------------------------------------------------------------------------------- /requester/uploader/instance_state.go: -------------------------------------------------------------------------------- 1 | package uploader 2 | 3 | import ( 4 | "github.com/iikira/iikira-go-utils/requester/transfer" 5 | ) 6 | 7 | type ( 8 | // BlockState 文件区块信息 9 | BlockState struct { 10 | ID int `json:"id"` 11 | Range transfer.Range `json:"range"` 12 | CheckSum string `json:"checksum"` 13 | } 14 | 15 | // InstanceState 上传断点续传信息 16 | InstanceState struct { 17 | BlockList []*BlockState `json:"block_list"` 18 | } 19 | ) 20 | 21 | func (muer *MultiUploader) getWorkerListByInstanceState(is *InstanceState) workerList { 22 | workers := make(workerList, 0, len(is.BlockList)) 23 | for _, blockState := range is.BlockList { 24 | if blockState.CheckSum == "" { 25 | workers = append(workers, &worker{ 26 | id: blockState.ID, 27 | partOffset: blockState.Range.Begin, 28 | splitUnit: NewBufioSplitUnit(muer.file, blockState.Range, muer.speedsStat, muer.rateLimit), 29 | checksum: blockState.CheckSum, 30 | }) 31 | } else { 32 | // 已经完成的, 也要加入 (可继续优化) 33 | workers = append(workers, &worker{ 34 | id: blockState.ID, 35 | partOffset: blockState.Range.Begin, 36 | splitUnit: &fileBlock{ 37 | readRange: blockState.Range, 38 | readed: blockState.Range.End - blockState.Range.Begin, 39 | readerAt: muer.file, 40 | }, 41 | checksum: blockState.CheckSum, 42 | }) 43 | } 44 | } 45 | return workers 46 | } 47 | -------------------------------------------------------------------------------- /utils/escaper/escaper.go: -------------------------------------------------------------------------------- 1 | package escaper 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | type ( 8 | // RuneFunc 判断指定rune 9 | RuneFunc func(r rune) bool 10 | ) 11 | 12 | // EscapeByRuneFunc 通过runeFunc转义, runeFunc返回真, 则转义 13 | func EscapeByRuneFunc(s string, runeFunc RuneFunc) string { 14 | if runeFunc == nil { 15 | return s 16 | } 17 | 18 | var ( 19 | builder = &strings.Builder{} 20 | rs = []rune(s) 21 | ) 22 | 23 | for k := range rs { 24 | if !runeFunc(rs[k]) { 25 | builder.WriteRune(rs[k]) 26 | continue 27 | } 28 | 29 | if k >= 1 && rs[k-1] == '\\' { 30 | builder.WriteRune(rs[k]) 31 | continue 32 | } 33 | builder.WriteString(`\`) 34 | builder.WriteRune(rs[k]) 35 | } 36 | return builder.String() 37 | } 38 | 39 | // Escape 转义指定的escapeRunes, 在escapeRunes的前面加上一个反斜杠 40 | func Escape(s string, escapeRunes []rune) string { 41 | return EscapeByRuneFunc(s, func(r rune) bool { 42 | for k := range escapeRunes { 43 | if escapeRunes[k] == r { 44 | return true 45 | } 46 | } 47 | return false 48 | }) 49 | } 50 | 51 | // EscapeStrings 转义字符串数组 52 | func EscapeStrings(ss []string, escapeRunes []rune) { 53 | for k := range ss { 54 | ss[k] = Escape(ss[k], escapeRunes) 55 | } 56 | } 57 | 58 | // EscapeStringsByRuneFunc 转义字符串数组, 通过runeFunc 59 | func EscapeStringsByRuneFunc(ss []string, runeFunc RuneFunc) { 60 | for k := range ss { 61 | ss[k] = EscapeByRuneFunc(ss[k], runeFunc) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /requester/downloader/example.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "fmt" 5 | "github.com/iikira/iikira-go-utils/requester/transfer" 6 | "github.com/iikira/iikira-go-utils/utils/converter" 7 | "io" 8 | "os" 9 | ) 10 | 11 | // DoDownload 执行下载 12 | func DoDownload(durl string, savePath string, cfg *Config) { 13 | var ( 14 | file *os.File 15 | writer io.WriterAt 16 | err error 17 | ) 18 | 19 | if savePath != "" { 20 | writer, file, err = NewDownloaderWriterByFilename(savePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0666) 21 | if err != nil { 22 | fmt.Println(err) 23 | return 24 | } 25 | defer file.Close() 26 | } 27 | 28 | download := NewDownloader(durl, writer, cfg) 29 | 30 | exitDownloadFunc := make(chan struct{}) 31 | 32 | download.OnDownloadStatusEvent(func(status transfer.DownloadStatuser, workersCallback func(RangeWorkerFunc)) { 33 | var ts string 34 | if status.TotalSize() <= 0 { 35 | ts = converter.ConvertFileSize(status.Downloaded(), 2) 36 | } else { 37 | ts = converter.ConvertFileSize(status.TotalSize(), 2) 38 | } 39 | 40 | fmt.Printf("\r ↓ %s/%s %s/s in %s ............", 41 | converter.ConvertFileSize(status.Downloaded(), 2), 42 | ts, 43 | converter.ConvertFileSize(status.SpeedsPerSecond(), 2), 44 | status.TimeElapsed(), 45 | ) 46 | }) 47 | 48 | err = download.Execute() 49 | close(exitDownloadFunc) 50 | if err != nil { 51 | fmt.Printf("err: %s\n", err) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /utils/csttime/time.go: -------------------------------------------------------------------------------- 1 | // Package csttime 时间工具包 2 | package csttime 3 | 4 | import ( 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | var ( 10 | // CSTLocation 东八区时区 11 | CSTLocation = time.FixedZone("CST", 8*3600) 12 | ) 13 | 14 | /* 15 | BeijingTimeOption 根据给定的 get 返回时间格式. 16 | 17 | get: 时间格式 18 | 19 | "Refer": 2017-7-21 12:02:32.000 20 | "printLog": 2017-7-21_12:02:32 21 | "day": 21 22 | "ymd": 2017-7-21 23 | "hour": 12 24 | 默认时间戳: 1500609752 25 | */ 26 | func BeijingTimeOption(get string) string { 27 | //获取北京（东八区）时间 28 | now := time.Now().In(CSTLocation) 29 | year, mon, day := now.Date() 30 | hour, min, sec := now.Clock() 31 | millisecond := now.Nanosecond() / 1e6 32 | switch get { 33 | case "Refer": 34 | return fmt.Sprintf("%d-%d-%d %02d:%02d:%02d.%03d", year, mon, day, hour, min, sec, millisecond) 35 | case "printLog": 36 | return fmt.Sprintf("%d-%d-%d_%02dh%02dm%02ds", year, mon, day, hour, min, sec) 37 | case "day": 38 | return fmt.Sprint(day) 39 | case "ymd": 40 | return fmt.Sprintf("%d-%d-%d", year, mon, day) 41 | case "hour": 42 | return fmt.Sprint(hour) 43 | default: 44 | return fmt.Sprint(time.Now().Unix()) 45 | } 46 | } 47 | 48 | // FormatTime 将 Unix 时间戳, 转换为字符串 49 | func FormatTime(t int64) string { 50 | tt := time.Unix(t, 0).In(CSTLocation) 51 | year, mon, day := tt.Date() 52 | hour, min, sec := tt.Clock() 53 | return fmt.Sprintf("%d-%02d-%02d %02d:%02d:%02d", year, mon, day, hour, min, sec) 54 | } 55 | -------------------------------------------------------------------------------- /requester/downloader/config.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "github.com/iikira/iikira-go-utils/requester/transfer" 5 | ) 6 | 7 | const ( 8 | //CacheSize 默认的下载缓存 9 | CacheSize = 8192 10 | ) 11 | 12 | var ( 13 | // MinParallelSize 单个线程最小的数据量 14 | MinParallelSize int64 = 128 * 1024 // 128kb 15 | ) 16 | 17 | //Config 下载配置 18 | type Config struct { 19 | Mode transfer.RangeGenMode // 下载Range分配模式 20 | MaxParallel int // 最大下载并发量 21 | CacheSize int // 下载缓冲 22 | BlockSize int64 // 每个Range区块的大小, RangeGenMode 为 RangeGenMode2 时才有效 23 | MaxRate int64 // 限制最大下载速度 24 | InstanceStateStorageFormat InstanceStateStorageFormat // 断点续传储存类型 25 | InstanceStatePath string // 断点续传信息路径 26 | IsTest bool // 是否测试下载 27 | TryHTTP bool // 是否尝试使用 http 连接 28 | } 29 | 30 | //NewConfig 返回默认配置 31 | func NewConfig() *Config { 32 | return &Config{ 33 | MaxParallel: 5, 34 | CacheSize: CacheSize, 35 | IsTest: false, 36 | } 37 | } 38 | 39 | //Fix 修复配置信息, 使其合法 40 | func (cfg *Config) Fix() { 41 | fixCacheSize(&cfg.CacheSize) 42 | if cfg.MaxParallel < 1 { 43 | cfg.MaxParallel = 1 44 | } 45 | } 46 | 47 | //Copy 拷贝新的配置 48 | func (cfg *Config) Copy() *Config { 49 | newCfg := *cfg 50 | return &newCfg 51 | } 52 | -------------------------------------------------------------------------------- /utils/taskframework/taskframework_test.go: -------------------------------------------------------------------------------- 1 | package taskframework 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | type ( 10 | TestUnit struct { 11 | retry bool 12 | taskInfo *TaskInfo 13 | } 14 | ) 15 | 16 | func (tu *TestUnit) SetTaskInfo(taskInfo *TaskInfo) { 17 | tu.taskInfo = taskInfo 18 | } 19 | 20 | func (tu *TestUnit) OnFailed(lastRunResult *TaskUnitRunResult) { 21 | fmt.Printf("[%s] error: %s, failed\n", tu.taskInfo.Id(), lastRunResult.Err) 22 | } 23 | 24 | func (tu *TestUnit) OnSuccess(lastRunResult *TaskUnitRunResult) { 25 | fmt.Printf("[%s] success\n", tu.taskInfo.Id()) 26 | } 27 | 28 | func (tu *TestUnit) OnComplete(lastRunResult *TaskUnitRunResult) { 29 | fmt.Printf("[%s] complete\n", tu.taskInfo.Id()) 30 | } 31 | 32 | func (tu *TestUnit) Run() (result *TaskUnitRunResult) { 33 | fmt.Printf("[%s] running...\n", tu.taskInfo.Id()) 34 | return &TaskUnitRunResult{ 35 | //Succeed: true, 36 | NeedRetry: true, 37 | } 38 | } 39 | 40 | func (tu *TestUnit) OnRetry(lastRunResult *TaskUnitRunResult) { 41 | fmt.Printf("[%s] prepare retry, times [%d/%d]...\n", tu.taskInfo.Id(), tu.taskInfo.Retry(), tu.taskInfo.MaxRetry()) 42 | } 43 | 44 | func (tu *TestUnit) RetryWait() time.Duration { 45 | return 1 * time.Second 46 | } 47 | 48 | func TestTaskExecutor(t *testing.T) { 49 | te := NewTaskExecutor() 50 | te.SetParallel(2) 51 | for i := 0; i < 3; i++ { 52 | tu := TestUnit{ 53 | retry: false, 54 | } 55 | te.Append(&tu, 2) 56 | } 57 | te.Execute() 58 | } 59 | -------------------------------------------------------------------------------- /utils/jwted25519/jwted25519.go: -------------------------------------------------------------------------------- 1 | package jwted25519 2 | 3 | import ( 4 | "github.com/dgrijalva/jwt-go" 5 | "golang.org/x/crypto/ed25519" 6 | "unsafe" 7 | ) 8 | 9 | type ( 10 | SigningMethodEd25519 struct{} 11 | ) 12 | 13 | var ( 14 | SigningMethodED25519 *SigningMethodEd25519 15 | ) 16 | 17 | func init() { 18 | // ED25519 19 | SigningMethodED25519 = &SigningMethodEd25519{} 20 | jwt.RegisterSigningMethod(SigningMethodED25519.Alg(), func() jwt.SigningMethod { 21 | return SigningMethodED25519 22 | }) 23 | } 24 | 25 | func (m *SigningMethodEd25519) Alg() string { 26 | return "ED25519" 27 | } 28 | 29 | func (m *SigningMethodEd25519) Sign(signingString string, key interface{}) (string, error) { 30 | if privkey, ok := key.(ed25519.PrivateKey); ok { 31 | if len(privkey) != ed25519.PrivateKeySize { 32 | return "", jwt.ErrInvalidKey 33 | } 34 | 35 | signature := ed25519.Sign(privkey, *(*[]byte)(unsafe.Pointer(&signingString))) 36 | return jwt.EncodeSegment(signature), nil 37 | } 38 | 39 | return "", jwt.ErrInvalidKeyType 40 | } 41 | 42 | func (m *SigningMethodEd25519) Verify(signingString, signature string, key interface{}) error { 43 | pubkey, ok := key.(ed25519.PublicKey) 44 | if !ok { 45 | return jwt.ErrInvalidKeyType 46 | } 47 | 48 | if len(pubkey) != ed25519.PublicKeySize { 49 | return jwt.ErrInvalidKey 50 | } 51 | 52 | message, err := jwt.DecodeSegment(signature) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | ok = ed25519.Verify(pubkey, *(*[]byte)(unsafe.Pointer(&signingString)), message) 58 | if !ok { 59 | return jwt.ErrSignatureInvalid 60 | } 61 | 62 | return nil 63 | } 64 | -------------------------------------------------------------------------------- /requester/rio/multi.go: -------------------------------------------------------------------------------- 1 | package rio 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | // MultiReaderLen 合并多个ReaderLen 8 | func MultiReaderLen(readerLens ...ReaderLen) ReaderLen { 9 | // TODO: 和copy对比 10 | r := make([]io.Reader, 0, len(readerLens)) 11 | for k := range readerLens { 12 | if readerLens[k] == nil { 13 | continue 14 | } 15 | r = append(r, readerLens[k]) 16 | } 17 | return &multiReaderLen{ 18 | mrls: readerLens, 19 | multiReader: io.MultiReader(r...), 20 | } 21 | } 22 | 23 | type multiReaderLen struct { 24 | mrls []ReaderLen 25 | multiReader io.Reader 26 | } 27 | 28 | func (mrl *multiReaderLen) Read(p []byte) (n int, err error) { 29 | return mrl.multiReader.Read(p) 30 | } 31 | 32 | func (mrl *multiReaderLen) Len() int { 33 | var i int 34 | for _, v := range mrl.mrls { 35 | i += v.Len() 36 | } 37 | return i 38 | } 39 | 40 | // MultiReaderLen64 合并多个ReaderLen64 41 | func MultiReaderLen64(readerLen64s ...ReaderLen64) ReaderLen64 { 42 | // TODO: 和copy对比 43 | r := make([]io.Reader, 0, len(readerLen64s)) 44 | for k := range readerLen64s { 45 | if readerLen64s[k] == nil { 46 | continue 47 | } 48 | r = append(r, readerLen64s[k]) 49 | } 50 | return &multiReaderLen64{ 51 | mrl64s: readerLen64s, 52 | multiReader: io.MultiReader(r...), 53 | } 54 | } 55 | 56 | type multiReaderLen64 struct { 57 | mrl64s []ReaderLen64 58 | multiReader io.Reader 59 | } 60 | 61 | func (mrl64 *multiReaderLen64) Read(p []byte) (n int, err error) { 62 | return mrl64.multiReader.Read(p) 63 | } 64 | 65 | func (mrl64 *multiReaderLen64) Len() int64 { 66 | var l int64 67 | for _, v := range mrl64.mrl64s { 68 | l += v.Len() 69 | } 70 | return l 71 | } 72 | -------------------------------------------------------------------------------- /requester/downloader/loadbalance.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "net/http" 5 | "sync/atomic" 6 | ) 7 | 8 | type ( 9 | // LoadBalancerResponse 负载均衡响应状态 10 | LoadBalancerResponse struct { 11 | URL string 12 | Referer string 13 | } 14 | 15 | // LoadBalancerResponseList 负载均衡列表 16 | LoadBalancerResponseList struct { 17 | lbr []*LoadBalancerResponse 18 | cursor int32 19 | } 20 | 21 | LoadBalancerCompareFunc func(info map[string]string, subResp *http.Response) bool 22 | ) 23 | 24 | // NewLoadBalancerResponseList 初始化负载均衡列表 25 | func NewLoadBalancerResponseList(lbr []*LoadBalancerResponse) *LoadBalancerResponseList { 26 | return &LoadBalancerResponseList{ 27 | lbr: lbr, 28 | } 29 | } 30 | 31 | // SequentialGet 顺序获取 32 | func (lbrl *LoadBalancerResponseList) SequentialGet() *LoadBalancerResponse { 33 | if len(lbrl.lbr) == 0 { 34 | return nil 35 | } 36 | 37 | if int(lbrl.cursor) >= len(lbrl.lbr) { 38 | lbrl.cursor = 0 39 | } 40 | 41 | lbr := lbrl.lbr[int(lbrl.cursor)] 42 | atomic.AddInt32(&lbrl.cursor, 1) 43 | return lbr 44 | } 45 | 46 | // RandomGet 随机获取 47 | func (lbrl *LoadBalancerResponseList) RandomGet() *LoadBalancerResponse { 48 | return lbrl.lbr[RandomNumber(0, len(lbrl.lbr))] 49 | } 50 | 51 | // AddLoadBalanceServer 增加负载均衡服务器 52 | func (der *Downloader) AddLoadBalanceServer(urls ...string) { 53 | der.loadBalansers = append(der.loadBalansers, urls...) 54 | } 55 | 56 | // DefaultLoadBalancerCompareFunc 检测负载均衡的服务器是否一致 57 | func DefaultLoadBalancerCompareFunc(info map[string]string, subResp *http.Response) bool { 58 | if info == nil || subResp == nil { 59 | return false 60 | } 61 | 62 | for headerKey, value := range info { 63 | if value != subResp.Header.Get(headerKey) { 64 | return false 65 | } 66 | } 67 | 68 | return true 69 | } 70 | -------------------------------------------------------------------------------- /utils/bdcrypto/archive.go: -------------------------------------------------------------------------------- 1 | package bdcrypto 2 | 3 | import ( 4 | "compress/gzip" 5 | "io" 6 | "os" 7 | ) 8 | 9 | // GZIPCompress GZIP 压缩 10 | func GZIPCompress(src io.Reader, writeTo io.Writer) (err error) { 11 | w := gzip.NewWriter(writeTo) 12 | _, err = io.Copy(w, src) 13 | if err != nil { 14 | return 15 | } 16 | 17 | w.Flush() 18 | return w.Close() 19 | } 20 | 21 | // GZIPUncompress GZIP 解压缩 22 | func GZIPUncompress(src io.Reader, writeTo io.Writer) (err error) { 23 | unReader, err := gzip.NewReader(src) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | _, err = io.Copy(writeTo, unReader) 29 | if err != nil { 30 | return 31 | } 32 | 33 | return unReader.Close() 34 | } 35 | 36 | // GZIPCompressFile GZIP 压缩文件 37 | func GZIPCompressFile(filePath string) (err error) { 38 | return gzipCompressFile("en", filePath) 39 | } 40 | 41 | // GZIPUnompressFile GZIP 解压缩文件 42 | func GZIPUnompressFile(filePath string) (err error) { 43 | return gzipCompressFile("de", filePath) 44 | } 45 | 46 | func gzipCompressFile(op, filePath string) (err error) { 47 | f, err := os.Open(filePath) 48 | if err != nil { 49 | return 50 | } 51 | 52 | defer f.Close() 53 | 54 | fInfo, err := f.Stat() 55 | if err != nil { 56 | return 57 | } 58 | 59 | tempFilePath := filePath + ".gzip.tmp" 60 | // 保留文件权限 61 | tempFile, err := os.OpenFile(tempFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fInfo.Mode()) 62 | if err != nil { 63 | return 64 | } 65 | 66 | defer tempFile.Close() 67 | 68 | switch op { 69 | case "en": 70 | err = GZIPCompress(f, tempFile) 71 | case "de": 72 | err = GZIPUncompress(f, tempFile) 73 | default: 74 | panic("unknown op" + op) 75 | } 76 | 77 | if err != nil { 78 | os.Remove(tempFilePath) 79 | return 80 | } 81 | 82 | return os.Rename(tempFilePath, filePath) 83 | } 84 | -------------------------------------------------------------------------------- /requester/downloader/download_firstinfo.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "reflect" 7 | ) 8 | 9 | type ( 10 | DownloadFirstInfo struct { 11 | ContentLength int64 12 | ContentMD5 string 13 | ContentCRC32 string 14 | AcceptRanges string 15 | Referer string 16 | } 17 | ) 18 | 19 | func NewDownloadFirstInfoByResp(contentLength int64, resp *http.Response) (dfi *DownloadFirstInfo) { 20 | dfi = &DownloadFirstInfo{} 21 | if resp == nil { 22 | dfi.ContentLength = contentLength 23 | return 24 | } 25 | if contentLength != resp.ContentLength { 26 | dfi.ContentLength = contentLength 27 | } 28 | dfi.AcceptRanges = resp.Header.Get("Accept-Ranges") 29 | dfi.Referer = resp.Header.Get("Referer") 30 | return 31 | } 32 | 33 | func (dfi *DownloadFirstInfo) Compare(n *DownloadFirstInfo) bool { 34 | if n == nil { 35 | return false 36 | } 37 | if dfi.ContentLength != n.ContentLength { 38 | return false 39 | } 40 | if dfi.AcceptRanges != n.AcceptRanges { 41 | return false 42 | } 43 | if dfi.Referer != n.Referer { 44 | return false 45 | } 46 | return true 47 | } 48 | 49 | // ToMap 转换为map 50 | func (dfi *DownloadFirstInfo) ToMap() map[string]string { 51 | m := map[string]string{ 52 | "Content-MD5": dfi.ContentMD5, 53 | "x-bs-meta-crc32": dfi.ContentCRC32, 54 | "Accept-Ranges": dfi.AcceptRanges, 55 | "Referer": dfi.Referer, 56 | } 57 | return m 58 | } 59 | 60 | // ToMapByReflect 用reflect转换为map 61 | func (dfi *DownloadFirstInfo) ToMapByReflect() map[string]string { 62 | te := reflect.TypeOf(dfi).Elem() 63 | ve := reflect.ValueOf(dfi).Elem() 64 | n := te.NumField() 65 | m := map[string]string{} 66 | for i := 0; i < n; i++ { 67 | f := te.Field(i) 68 | m[f.Name] = fmt.Sprint(ve.Field(i).Interface()) 69 | } 70 | return m 71 | } 72 | -------------------------------------------------------------------------------- /utils/cachepool/cachepool.go: -------------------------------------------------------------------------------- 1 | package cachepool 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | var ( 8 | //CachePool []byte 缓存池 2 9 | CachePool = cachePool2{} 10 | ) 11 | 12 | //Cache cache 13 | type Cache interface { 14 | Bytes() []byte 15 | Free() 16 | } 17 | 18 | type cache struct { 19 | isUsed bool 20 | b []byte 21 | } 22 | 23 | func (c *cache) Bytes() []byte { 24 | if !c.isUsed { 25 | return nil 26 | } 27 | return c.b 28 | } 29 | 30 | func (c *cache) Free() { 31 | c.isUsed = false 32 | } 33 | 34 | type cachePool2 struct { 35 | pool []*cache 36 | mu sync.Mutex 37 | } 38 | 39 | func (cp2 *cachePool2) Require(size int) Cache { 40 | cp2.mu.Lock() 41 | defer cp2.mu.Unlock() 42 | for k := range cp2.pool { 43 | if cp2.pool[k] == nil || cp2.pool[k].isUsed || len(cp2.pool[k].b) < size { 44 | continue 45 | } 46 | 47 | cp2.pool[k].isUsed = true 48 | return cp2.pool[k] 49 | } 50 | newCache := &cache{ 51 | isUsed: true, 52 | b: RawMallocByteSlice(size), 53 | } 54 | cp2.addCache(newCache) 55 | return newCache 56 | } 57 | 58 | func (cp2 *cachePool2) addCache(newCache *cache) { 59 | for k := range cp2.pool { 60 | if cp2.pool[k] == nil { 61 | cp2.pool[k] = newCache 62 | return 63 | } 64 | } 65 | cp2.pool = append(cp2.pool, newCache) 66 | } 67 | 68 | func (cp2 *cachePool2) DeleteNotUsed() { 69 | cp2.mu.Lock() 70 | defer cp2.mu.Unlock() 71 | for k := range cp2.pool { 72 | if cp2.pool[k] == nil { 73 | continue 74 | } 75 | 76 | if !cp2.pool[k].isUsed { 77 | cp2.pool[k] = nil 78 | } 79 | } 80 | } 81 | 82 | func (cp2 *cachePool2) DeleteAll() { 83 | cp2.mu.Lock() 84 | defer cp2.mu.Unlock() 85 | for k := range cp2.pool { 86 | cp2.pool[k] = nil 87 | } 88 | } 89 | 90 | //Require 申请Cache 91 | func Require(size int) Cache { 92 | return CachePool.Require(size) 93 | } 94 | -------------------------------------------------------------------------------- /requester/rio/speeds/ratelimit.go: -------------------------------------------------------------------------------- 1 | package speeds 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | "time" 7 | ) 8 | 9 | type ( 10 | RateLimit struct { 11 | MaxRate int64 12 | 13 | count int64 14 | interval time.Duration 15 | ticker *time.Ticker 16 | muChan chan struct{} 17 | closeChan chan struct{} 18 | backServiceOnce sync.Once 19 | } 20 | 21 | // AddCountFunc func() (count int64) 22 | ) 23 | 24 | func NewRateLimit(maxRate int64) *RateLimit { 25 | return &RateLimit{ 26 | MaxRate: maxRate, 27 | } 28 | } 29 | 30 | func (rl *RateLimit) SetInterval(i time.Duration) { 31 | if i <= 0 { 32 | i = 1 * time.Second 33 | } 34 | rl.interval = i 35 | if rl.ticker != nil { 36 | rl.ticker.Stop() 37 | rl.ticker = time.NewTicker(i) 38 | } 39 | } 40 | 41 | func (rl *RateLimit) Stop() { 42 | if rl.ticker != nil { 43 | rl.ticker.Stop() 44 | } 45 | if rl.closeChan != nil { 46 | close(rl.closeChan) 47 | } 48 | return 49 | } 50 | 51 | func (rl *RateLimit) resetChan() { 52 | if rl.muChan != nil { 53 | close(rl.muChan) 54 | } 55 | rl.muChan = make(chan struct{}) 56 | } 57 | 58 | func (rl *RateLimit) backService() { 59 | if rl.interval <= 0 { 60 | rl.interval = 1 * time.Second 61 | } 62 | rl.ticker = time.NewTicker(rl.interval) 63 | rl.closeChan = make(chan struct{}) 64 | rl.resetChan() 65 | go func() { 66 | for { 67 | select { 68 | case <-rl.ticker.C: 69 | rl.resetChan() 70 | atomic.StoreInt64(&rl.count, 0) 71 | case <-rl.closeChan: 72 | return 73 | } 74 | } 75 | }() 76 | } 77 | 78 | func (rl *RateLimit) Add(count int64) { 79 | rl.backServiceOnce.Do(rl.backService) 80 | for { 81 | if atomic.LoadInt64(&rl.count) >= rl.MaxRate { // 超出最大限额 82 | // 阻塞 83 | <-rl.muChan 84 | continue 85 | } 86 | atomic.AddInt64(&rl.count, count) 87 | break 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /requester/rio/file.go: -------------------------------------------------------------------------------- 1 | package rio 2 | 3 | import ( 4 | cryptorand "crypto/rand" 5 | "io" 6 | "os" 7 | "sync/atomic" 8 | ) 9 | 10 | type ( 11 | fileReadedlen64 struct { 12 | readed int64 13 | f *os.File 14 | } 15 | 16 | rdReadedlen64 struct { 17 | readed int64 18 | size int64 19 | rd io.Reader 20 | } 21 | ) 22 | 23 | // NewFileReaderLen64 *os.File 实现 ReadedLen64 接口 24 | func NewFileReaderLen64(f *os.File) ReaderLen64 { 25 | if f == nil { 26 | return nil 27 | } 28 | 29 | return &fileReadedlen64{ 30 | f: f, 31 | } 32 | } 33 | 34 | // NewFileReaderAtLen64 *os.File 实现 ReaderAtLen64 接口 35 | func NewFileReaderAtLen64(f *os.File) ReaderAtLen64 { 36 | if f == nil { 37 | return nil 38 | } 39 | 40 | return &fileReadedlen64{ 41 | f: f, 42 | } 43 | } 44 | 45 | func NewCryptoRandReaderAtLen64(size int64) ReaderAtLen64 { 46 | return &rdReadedlen64{ 47 | rd: cryptorand.Reader, 48 | size: size, 49 | } 50 | } 51 | 52 | // Read 读文件, 并记录已读取数据量 53 | func (fr *fileReadedlen64) Read(b []byte) (n int, err error) { 54 | n, err = fr.f.Read(b) 55 | atomic.AddInt64(&fr.readed, int64(n)) 56 | return n, err 57 | } 58 | 59 | // ReadAt 读文件, 不记录已读取数据量 60 | func (fr *fileReadedlen64) ReadAt(b []byte, off int64) (n int, err error) { 61 | n, err = fr.f.ReadAt(b, off) 62 | return n, err 63 | } 64 | 65 | // Len 返回文件的大小 66 | func (fr *fileReadedlen64) Len() int64 { 67 | info, err := fr.f.Stat() 68 | if err != nil { 69 | return 0 70 | } 71 | return info.Size() - fr.readed 72 | } 73 | 74 | func (rr *rdReadedlen64) Read(b []byte) (n int, err error) { 75 | n, err = rr.ReadAt(b, 0) 76 | atomic.AddInt64(&rr.readed, int64(n)) 77 | return n, err 78 | } 79 | 80 | func (rr *rdReadedlen64) ReadAt(b []byte, off int64) (n int, err error) { 81 | n, err = rr.rd.Read(b) 82 | return n, err 83 | } 84 | 85 | func (rr *rdReadedlen64) Len() int64 { 86 | return rr.size - rr.readed 87 | } 88 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | // Package pcsutil 工具包 2 | package utils 3 | 4 | import ( 5 | "compress/gzip" 6 | "flag" 7 | "io" 8 | "io/ioutil" 9 | "net/http/cookiejar" 10 | "net/url" 11 | "strings" 12 | ) 13 | 14 | // TrimPathPrefix 去除目录的前缀 15 | func TrimPathPrefix(path, prefixPath string) string { 16 | if prefixPath == "/" { 17 | return path 18 | } 19 | return strings.TrimPrefix(path, prefixPath) 20 | } 21 | 22 | // ContainsString 检测字符串是否在字符串数组里 23 | func ContainsString(ss []string, s string) bool { 24 | for k := range ss { 25 | if ss[k] == s { 26 | return true 27 | } 28 | } 29 | return false 30 | } 31 | 32 | // GetURLCookieString 返回cookie字串 33 | func GetURLCookieString(urlString string, jar *cookiejar.Jar) string { 34 | u, _ := url.Parse(urlString) 35 | cookies := jar.Cookies(u) 36 | cookieString := "" 37 | for _, v := range cookies { 38 | cookieString += v.String() + "; " 39 | } 40 | cookieString = strings.TrimRight(cookieString, "; ") 41 | return cookieString 42 | } 43 | 44 | // DecompressGZIP 对 io.Reader 数据, 进行 gzip 解压 45 | func DecompressGZIP(r io.Reader) ([]byte, error) { 46 | gzipReader, err := gzip.NewReader(r) 47 | if err != nil { 48 | return nil, err 49 | } 50 | gzipReader.Close() 51 | return ioutil.ReadAll(gzipReader) 52 | } 53 | 54 | // FlagProvided 检测命令行是否提供名为 name 的 flag, 支持多个name(names) 55 | func FlagProvided(names ...string) bool { 56 | if len(names) == 0 { 57 | return false 58 | } 59 | var targetFlag *flag.Flag 60 | for _, name := range names { 61 | targetFlag = flag.Lookup(name) 62 | if targetFlag == nil { 63 | return false 64 | } 65 | if targetFlag.DefValue == targetFlag.Value.String() { 66 | return false 67 | } 68 | } 69 | return true 70 | } 71 | 72 | // Trigger 用于触发事件 73 | func Trigger(f func()) { 74 | if f == nil { 75 | return 76 | } 77 | go f() 78 | } 79 | 80 | // TriggerOnSync 用于触发事件, 同步触发 81 | func TriggerOnSync(f func()) { 82 | if f == nil { 83 | return 84 | } 85 | f() 86 | } 87 | -------------------------------------------------------------------------------- /pcsverbose/pcsverbose.go: -------------------------------------------------------------------------------- 1 | // Package pcsverbose 调试包 2 | package pcsverbose 3 | 4 | import ( 5 | "fmt" 6 | "io" 7 | "os" 8 | ) 9 | 10 | const ( 11 | // EnvVerbose 启用调试环境变量 12 | EnvVerbose = "IIKIRA_GO_UTILS_VERBOSE" 13 | ) 14 | 15 | var ( 16 | // IsVerbose 是否调试 17 | IsVerbose = os.Getenv(EnvVerbose) == "1" 18 | 19 | // Outputs 输出 20 | Outputs = []io.Writer{os.Stderr} 21 | ) 22 | 23 | // PCSVerbose 调试 24 | type PCSVerbose struct { 25 | Module string 26 | } 27 | 28 | // New 根据module, 初始化PCSVerbose 29 | func New(module string) *PCSVerbose { 30 | return &PCSVerbose{ 31 | Module: module, 32 | } 33 | } 34 | 35 | // Info 提示 36 | func (pv *PCSVerbose) Info(l string) { 37 | Verbosef("DEBUG: %s INFO: %s\n", pv.Module, l) 38 | } 39 | 40 | // Infof 提示, 格式输出 41 | func (pv *PCSVerbose) Infof(format string, a ...interface{}) { 42 | Verbosef("DEBUG: %s INFO: %s", pv.Module, fmt.Sprintf(format, a...)) 43 | } 44 | 45 | // Warn 警告 46 | func (pv *PCSVerbose) Warn(l string) { 47 | Verbosef("DEBUG: %s WARN: %s\n", pv.Module, l) 48 | } 49 | 50 | // Warnf 警告, 格式输出 51 | func (pv *PCSVerbose) Warnf(format string, a ...interface{}) { 52 | Verbosef("DEBUG: %s WARN: %s", pv.Module, fmt.Sprintf(format, a...)) 53 | } 54 | 55 | // Verbosef 调试格式输出 56 | func Verbosef(format string, a ...interface{}) (n int, err error) { 57 | if IsVerbose { 58 | for _, Output := range Outputs { 59 | n1, err := fmt.Fprintf(Output, TimePrefix()+" "+format, a...) 60 | n += n1 61 | if err != nil { 62 | return n, err 63 | } 64 | } 65 | } 66 | return 67 | } 68 | 69 | // Verboseln 调试输出一行 70 | func Verboseln(a ...interface{}) (n int, err error) { 71 | if IsVerbose { 72 | for _, Output := range Outputs { 73 | n1, err := fmt.Fprint(Output, TimePrefix()+" ") 74 | n += n1 75 | if err != nil { 76 | return n, err 77 | } 78 | n2, err := fmt.Fprintln(Output, a...) 79 | n += n2 80 | if err != nil { 81 | return n, err 82 | } 83 | } 84 | } 85 | return 86 | } 87 | -------------------------------------------------------------------------------- /requester/downloader/utils.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "github.com/iikira/iikira-go-utils/pcsverbose" 5 | "github.com/iikira/iikira-go-utils/requester" 6 | mathrand "math/rand" 7 | "mime" 8 | "net/url" 9 | "path" 10 | "regexp" 11 | "strconv" 12 | "time" 13 | ) 14 | 15 | var ( 16 | // ContentRangeRE Content-Range 正则 17 | ContentRangeRE = regexp.MustCompile(`^.*? \d*?-\d*?/(\d*?)$`) 18 | 19 | // ranSource 随机数种子 20 | ranSource = mathrand.NewSource(time.Now().UnixNano()) 21 | 22 | // ran 一个随机数实例 23 | ran = mathrand.New(ranSource) 24 | ) 25 | 26 | // RandomNumber 生成指定区间随机数 27 | func RandomNumber(min, max int) int { 28 | if min > max { 29 | min, max = max, min 30 | } 31 | return ran.Intn(max-min) + min 32 | } 33 | 34 | // GetFileName 获取文件名 35 | func GetFileName(uri string, client *requester.HTTPClient) (filename string, err error) { 36 | if client == nil { 37 | client = requester.NewHTTPClient() 38 | } 39 | 40 | resp, err := client.Req("HEAD", uri, nil, nil) 41 | if resp != nil { 42 | defer resp.Body.Close() 43 | } 44 | if err != nil { 45 | return "", err 46 | } 47 | 48 | _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) 49 | if err != nil { 50 | pcsverbose.Verbosef("DEBUG: GetFileName ParseMediaType error: %s\n", err) 51 | return path.Base(uri), nil 52 | } 53 | 54 | filename, err = url.QueryUnescape(params["filename"]) 55 | if err != nil { 56 | return 57 | } 58 | 59 | if filename == "" { 60 | filename = path.Base(uri) 61 | } 62 | 63 | return 64 | } 65 | 66 | // ParseContentRange 解析Content-Range 67 | func ParseContentRange(contentRange string) (contentLength int64) { 68 | raw := ContentRangeRE.FindStringSubmatch(contentRange) 69 | if len(raw) < 2 { 70 | return -1 71 | } 72 | 73 | c, err := strconv.ParseInt(raw[1], 10, 64) 74 | if err != nil { 75 | return -1 76 | } 77 | return c 78 | } 79 | 80 | func fixCacheSize(size *int) { 81 | if *size < 1024 { 82 | *size = 1024 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /requester/transfer/download_instanceinfo.go: -------------------------------------------------------------------------------- 1 | package transfer 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type ( 8 | //DownloadInstanceInfo 状态详细信息, 用于导出状态文件 9 | DownloadInstanceInfo struct { 10 | DownloadStatus *DownloadStatus 11 | Ranges RangeList 12 | } 13 | 14 | // DownloadInstanceInfoExporter 断点续传类型接口 15 | DownloadInstanceInfoExporter interface { 16 | GetInstanceInfo() *DownloadInstanceInfo 17 | SetInstanceInfo(*DownloadInstanceInfo) 18 | } 19 | ) 20 | 21 | // GetInstanceInfo 从断点信息获取下载状态 22 | func (m *DownloadInstanceInfoExport) GetInstanceInfo() (eii *DownloadInstanceInfo) { 23 | eii = &DownloadInstanceInfo{ 24 | Ranges: m.Ranges, 25 | } 26 | 27 | var downloaded int64 28 | switch m.RangeGenMode { 29 | case RangeGenMode_BlockSize: 30 | downloaded = m.GenBegin - eii.Ranges.Len() 31 | default: 32 | downloaded = m.TotalSize - eii.Ranges.Len() 33 | } 34 | eii.DownloadStatus = &DownloadStatus{ 35 | startTime: time.Now(), 36 | totalSize: m.TotalSize, 37 | downloaded: downloaded, 38 | gen: NewRangeListGenBlockSize(m.TotalSize, m.GenBegin, m.BlockSize), 39 | } 40 | switch m.RangeGenMode { 41 | case RangeGenMode_BlockSize: 42 | eii.DownloadStatus.gen = NewRangeListGenBlockSize(m.TotalSize, m.GenBegin, m.BlockSize) 43 | default: 44 | eii.DownloadStatus.gen = NewRangeListGenDefault(m.TotalSize, m.TotalSize, len(m.Ranges), len(m.Ranges)) 45 | } 46 | return eii 47 | } 48 | 49 | // SetInstanceInfo 从下载状态导出断点信息 50 | func (m *DownloadInstanceInfoExport) SetInstanceInfo(eii *DownloadInstanceInfo) { 51 | if eii == nil { 52 | return 53 | } 54 | 55 | if eii.DownloadStatus != nil { 56 | m.TotalSize = eii.DownloadStatus.TotalSize() 57 | if eii.DownloadStatus.gen != nil { 58 | m.GenBegin = eii.DownloadStatus.gen.LoadBegin() 59 | m.BlockSize = eii.DownloadStatus.gen.LoadBlockSize() 60 | m.RangeGenMode = eii.DownloadStatus.gen.RangeGenMode() 61 | } else { 62 | m.RangeGenMode = RangeGenMode_Default 63 | } 64 | } 65 | m.Ranges = eii.Ranges 66 | } 67 | -------------------------------------------------------------------------------- /utils/bdcrypto/ecb/ecb.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Electronic Code Book (ECB) mode. 6 | 7 | // ECB provides confidentiality by assigning a fixed ciphertext block to each 8 | // plaintext block. 9 | 10 | // See NIST SP 800-38A, pp 08-09 11 | 12 | package ecb 13 | 14 | import ( 15 | "crypto/cipher" 16 | ) 17 | 18 | type ecb struct { 19 | b cipher.Block 20 | blockSize int 21 | } 22 | 23 | func newECB(b cipher.Block) *ecb { 24 | return &ecb{ 25 | b: b, 26 | blockSize: b.BlockSize(), 27 | } 28 | } 29 | 30 | type ecbEncrypter ecb 31 | 32 | // NewECBEncrypter returns a BlockMode which encrypts in electronic code book 33 | // mode, using the given Block. 34 | func NewECBEncrypter(b cipher.Block) cipher.BlockMode { 35 | return (*ecbEncrypter)(newECB(b)) 36 | } 37 | 38 | func (x *ecbEncrypter) BlockSize() int { return x.blockSize } 39 | 40 | func (x *ecbEncrypter) CryptBlocks(dst, src []byte) { 41 | if len(src)%x.blockSize != 0 { 42 | panic("crypto/cipher: input not full blocks") 43 | } 44 | if len(dst) < len(src) { 45 | panic("crypto/cipher: output smaller than input") 46 | } 47 | for len(src) > 0 { 48 | x.b.Encrypt(dst, src[:x.blockSize]) 49 | src = src[x.blockSize:] 50 | dst = dst[x.blockSize:] 51 | } 52 | } 53 | 54 | type ecbDecrypter ecb 55 | 56 | // NewECBDecrypter returns a BlockMode which decrypts in electronic code book 57 | // mode, using the given Block. 58 | func NewECBDecrypter(b cipher.Block) cipher.BlockMode { 59 | return (*ecbDecrypter)(newECB(b)) 60 | } 61 | 62 | func (x *ecbDecrypter) BlockSize() int { return x.blockSize } 63 | 64 | func (x *ecbDecrypter) CryptBlocks(dst, src []byte) { 65 | if len(src)%x.blockSize != 0 { 66 | panic("crypto/cipher: input not full blocks") 67 | } 68 | if len(dst) < len(src) { 69 | panic("crypto/cipher: output smaller than input") 70 | } 71 | for len(src) > 0 { 72 | x.b.Decrypt(dst, src[:x.blockSize]) 73 | src = src[x.blockSize:] 74 | dst = dst[x.blockSize:] 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /utils/expires/cachemap/cacheunit.go: -------------------------------------------------------------------------------- 1 | package cachemap 2 | 3 | import ( 4 | "github.com/iikira/iikira-go-utils/utils/expires" 5 | "sync" 6 | ) 7 | 8 | type ( 9 | CacheUnit interface { 10 | Delete(key interface{}) 11 | Load(key interface{}) (value expires.DataExpires, ok bool) 12 | LoadOrStore(key interface{}, value expires.DataExpires) (actual expires.DataExpires, loaded bool) 13 | Range(f func(key interface{}, value expires.DataExpires) bool) 14 | Store(key interface{}, value expires.DataExpires) 15 | LockKey(key interface{}) 16 | UnlockKey(key interface{}) 17 | } 18 | 19 | cacheUnit struct { 20 | unit sync.Map 21 | keyMap sync.Map 22 | } 23 | ) 24 | 25 | func (cu *cacheUnit) Delete(key interface{}) { 26 | cu.unit.Delete(key) 27 | cu.keyMap.Delete(key) 28 | } 29 | 30 | func (cu *cacheUnit) Load(key interface{}) (value expires.DataExpires, ok bool) { 31 | val, ok := cu.unit.Load(key) 32 | if !ok { 33 | return nil, ok 34 | } 35 | exp := val.(expires.DataExpires) 36 | if exp.IsExpires() { 37 | cu.unit.Delete(key) 38 | return nil, false 39 | } 40 | return exp, ok 41 | } 42 | 43 | func (cu *cacheUnit) Range(f func(key interface{}, value expires.DataExpires) bool) { 44 | cu.unit.Range(func(k, val interface{}) bool { 45 | exp := val.(expires.DataExpires) 46 | if exp.IsExpires() { 47 | cu.unit.Delete(k) 48 | return true 49 | } 50 | return f(k, val.(expires.DataExpires)) 51 | }) 52 | } 53 | 54 | func (cu *cacheUnit) LoadOrStore(key interface{}, value expires.DataExpires) (actual expires.DataExpires, loaded bool) { 55 | ac, loaded := cu.unit.LoadOrStore(key, value) 56 | exp := ac.(expires.DataExpires) 57 | if exp.IsExpires() { 58 | cu.unit.Delete(key) 59 | return nil, false 60 | } 61 | return exp, loaded 62 | } 63 | 64 | func (cu *cacheUnit) Store(key interface{}, value expires.DataExpires) { 65 | if value.IsExpires() { 66 | return 67 | } 68 | cu.unit.Store(key, value) 69 | } 70 | 71 | func (cu *cacheUnit) LockKey(key interface{}) { 72 | muItf, _ := cu.keyMap.LoadOrStore(key, &sync.Mutex{}) 73 | mu := muItf.(*sync.Mutex) 74 | mu.Lock() 75 | } 76 | 77 | func (cu *cacheUnit) UnlockKey(key interface{}) { 78 | muItf, _ := cu.keyMap.LoadOrStore(key, &sync.Mutex{}) 79 | mu := muItf.(*sync.Mutex) 80 | mu.Unlock() 81 | } 82 | -------------------------------------------------------------------------------- /utils/prealloc/prealloc_windows.go: -------------------------------------------------------------------------------- 1 | package prealloc 2 | 3 | import ( 4 | "golang.org/x/sys/windows" 5 | "log" 6 | "syscall" 7 | ) 8 | 9 | var ( 10 | kernel32 = syscall.NewLazyDLL("kernel32.dll") 11 | procSetFileValidData = kernel32.NewProc("SetFileValidData") 12 | ) 13 | 14 | func initPrivilege() error { 15 | current, err := windows.GetCurrentProcess() 16 | if err != nil { 17 | return &PreAllocError{ 18 | ProcName: "GetCurrentProcess", 19 | Err: err, 20 | } 21 | } 22 | 23 | var hToken windows.Token 24 | err = windows.OpenProcessToken(current, windows.TOKEN_ADJUST_PRIVILEGES|windows.TOKEN_QUERY, &hToken) 25 | if err != nil { 26 | return &PreAllocError{ 27 | ProcName: "OpenProcessToken", 28 | Err: err, 29 | } 30 | } 31 | 32 | var ( 33 | seManageVolumeName, _ = windows.UTF16PtrFromString("SeManageVolumePrivilege") 34 | tp = windows.Tokenprivileges{ 35 | PrivilegeCount: 1, 36 | Privileges: [1]windows.LUIDAndAttributes{ 37 | windows.LUIDAndAttributes{ 38 | Luid: windows.LUID{}, 39 | Attributes: windows.SE_PRIVILEGE_ENABLED, 40 | }, 41 | }, 42 | } 43 | ) 44 | err = windows.LookupPrivilegeValue(nil, seManageVolumeName, &tp.Privileges[0].Luid) 45 | if err != nil { 46 | return &PreAllocError{ 47 | ProcName: "LookupPrivilegeValue", 48 | Err: err, 49 | } 50 | } 51 | 52 | err = windows.AdjustTokenPrivileges(hToken, false, &tp, 0, nil, nil) 53 | if err != nil { 54 | return &PreAllocError{ 55 | ProcName: "AdjustTokenPrivileges", 56 | Err: err, 57 | } 58 | } 59 | 60 | return nil 61 | } 62 | 63 | // 初始化权限 64 | func init() { 65 | err := initPrivilege() 66 | if err != nil { 67 | log.Printf("prealloc: init privileges error: %s\n", err) // 打印警告 68 | } 69 | } 70 | 71 | // PreAlloc 预分配文件空间 72 | func PreAlloc(fd uintptr, length int64) error { 73 | err := syscall.Ftruncate(syscall.Handle(fd), length) 74 | if err != nil { 75 | return &PreAllocError{ 76 | ProcName: "Ftruncate", 77 | Err: err, 78 | } 79 | } 80 | 81 | r1, _, err := procSetFileValidData.Call(fd, uintptr(length)) 82 | if r1 == 0 { 83 | return &PreAllocError{ 84 | ProcName: "SetFileValidData", 85 | Err: err, 86 | } 87 | } 88 | 89 | return nil 90 | } 91 | -------------------------------------------------------------------------------- /utils/expires/cachemap/cachemap_test.go: -------------------------------------------------------------------------------- 1 | package cachemap_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/iikira/iikira-go-utils/utils/expires" 6 | "github.com/iikira/iikira-go-utils/utils/expires/cachemap" 7 | "sync" 8 | "sync/atomic" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func TestCacheMapDataExpires(t *testing.T) { 14 | cm :=cachemap. CacheOpMap{} 15 | cache := cm.LazyInitCachePoolOp("op") 16 | cache.Store("key_1", expires.NewDataExpires("value_1", 1*time.Second)) 17 | 18 | time.Sleep(2 * time.Second) 19 | data, ok := cache.Load("key_1") 20 | if ok { 21 | fmt.Printf("data: %s\n", data.Data()) 22 | // 超时仍能读取到数据, 失败 23 | t.FailNow() 24 | } 25 | } 26 | 27 | func TestCacheOperation(t *testing.T) { 28 | cm := cachemap.CacheOpMap{} 29 | data := cm.CacheOperation("op", "key_1", func() expires.DataExpires { 30 | return expires.NewDataExpires("value_1", 1*time.Second) 31 | }) 32 | fmt.Printf("data: %s\n", data.Data()) 33 | 34 | newData := cm.CacheOperation("op", "key_1", func() expires.DataExpires { 35 | return expires.NewDataExpires("value_3", 1*time.Second) 36 | }) 37 | if data != newData { 38 | t.FailNow() 39 | } 40 | fmt.Printf("data: %s\n", data.Data()) 41 | } 42 | 43 | func TestCacheOperation_LockKey(t *testing.T) { 44 | cm := cachemap.CacheOpMap{} 45 | wg := sync.WaitGroup{} 46 | wg.Add(5000) 47 | 48 | var ( 49 | execTimes1 int32 = 0 // 执行次数1 50 | execTimes2 int32 = 0 // 执行次数2 51 | ) 52 | 53 | for i := 0; i < 5000; i++ { 54 | go func(i int) { 55 | defer wg.Done() 56 | cm.CacheOperation("op", "key_1", func() expires.DataExpires { 57 | time.Sleep(50 * time.Microsecond) // 一些耗时的操作 58 | atomic.AddInt32(&execTimes1, 1) 59 | return expires.NewDataExpires(fmt.Sprintf("value_1: %d", i), 10*time.Second) 60 | }) 61 | 62 | cm.CacheOperation("op", "key_2", func() expires.DataExpires { 63 | time.Sleep(50 * time.Microsecond) // 一些耗时的操作 64 | atomic.AddInt32(&execTimes2, 1) 65 | return expires.NewDataExpires(fmt.Sprintf("value_2: %d", i), 10*time.Second) 66 | }) 67 | }(i) 68 | } 69 | wg.Wait() 70 | 71 | // 执行次数应为1 72 | if execTimes1 != 1 { 73 | fmt.Printf("execTimes1: %d\n", execTimes1) 74 | t.FailNow() 75 | } 76 | if execTimes2 != 1 { 77 | fmt.Printf("execTimes2: %d\n", execTimes2) 78 | t.FailNow() 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /utils/converter/size.go: -------------------------------------------------------------------------------- 1 | package converter 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | const ( 11 | // B byte 12 | B = (int64)(1 << (10 * iota)) 13 | // KB kilobyte 14 | KB 15 | // MB megabyte 16 | MB 17 | // GB gigabyte 18 | GB 19 | // TB terabyte 20 | TB 21 | // PB petabyte 22 | PB 23 | ) 24 | 25 | // ConvertFileSize 文件大小格式化输出 26 | func ConvertFileSize(size int64, precision ...int) string { 27 | pint := "6" 28 | if len(precision) == 1 { 29 | pint = fmt.Sprint(precision[0]) 30 | } 31 | if size < 0 { 32 | return "0B" 33 | } 34 | if size < KB { 35 | return fmt.Sprintf("%dB", size) 36 | } 37 | if size < MB { 38 | return fmt.Sprintf("%."+pint+"fKB", float64(size)/float64(KB)) 39 | } 40 | if size < GB { 41 | return fmt.Sprintf("%."+pint+"fMB", float64(size)/float64(MB)) 42 | } 43 | if size < TB { 44 | return fmt.Sprintf("%."+pint+"fGB", float64(size)/float64(GB)) 45 | } 46 | if size < PB { 47 | return fmt.Sprintf("%."+pint+"fTB", float64(size)/float64(TB)) 48 | } 49 | return fmt.Sprintf("%."+pint+"fPB", float64(size)/float64(PB)) 50 | } 51 | 52 | // ParseFileSizeStr 将文件大小字符串转换成字节数 53 | func ParseFileSizeStr(ss string) (size int64, err error) { 54 | if ss == "" { 55 | err = errors.New("converter: size is empty") 56 | return 57 | } 58 | if !(ss[0] == '.' || '0' <= ss[0] && ss[0] <= '9') { 59 | err = errors.New("converter: invalid size: " + ss) 60 | return 61 | } 62 | 63 | var i int 64 | for i = range ss[1:] { 65 | i++ 66 | if ss[i] == '.' || ('0' <= ss[i] && ss[i] <= '9') { 67 | // 属于数字 68 | continue 69 | } 70 | break 71 | } 72 | if ss[i] == '.' || ('0' <= ss[i] && ss[i] <= '9') { // 最后一个分隔符是否为数字 73 | i++ 74 | } 75 | 76 | var ( 77 | sizeStr = ss[:i] // 数字部分 78 | unitStr = ss[i:] // 单位部分 79 | sizeFloat, _ = strconv.ParseFloat(sizeStr, 10) 80 | ) 81 | switch strings.ToUpper(unitStr) { 82 | case "", "B": 83 | size = int64(sizeFloat) 84 | case "K", "KB": 85 | size = int64(sizeFloat * float64(KB)) 86 | case "M", "MB": 87 | size = int64(sizeFloat * float64(MB)) 88 | case "G", "GB": 89 | size = int64(sizeFloat * float64(GB)) 90 | case "T", "TB": 91 | size = int64(sizeFloat * float64(TB)) 92 | case "P", "PB": 93 | size = int64(sizeFloat * float64(PB)) 94 | default: 95 | err = errors.New("converter: invalid unit " + unitStr) 96 | } 97 | return 98 | } 99 | -------------------------------------------------------------------------------- /requester/downloader/download_test.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "fmt" 5 | "github.com/iikira/iikira-go-utils/pcsverbose" 6 | "github.com/iikira/iikira-go-utils/requester" 7 | "os" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | var ( 13 | url1 = "https://dldir1.qq.com/qqfile/qq/TIM2.1.8/23475/TIM2.1.8.exe" 14 | url2 = "https://git.oschina.net/lufenping/pixabay_img/raw/master/tiny-20170712/lizard-2427248_1920.jpg" 15 | ) 16 | 17 | func TestRandomNumber(t *testing.T) { 18 | for i := 0; i < 10; i++ { 19 | fmt.Println(RandomNumber(0, 5)) 20 | } 21 | } 22 | 23 | func TestExample(t *testing.T) { 24 | DoDownload(url2, "lizard-2427248_1920.jpg", nil) 25 | } 26 | 27 | func TestDownloadTIM(t *testing.T) { 28 | pcsverbose.IsVerbose = true 29 | 30 | file, _ := os.OpenFile("tim.exe", os.O_CREATE|os.O_WRONLY, 0777) 31 | d := NewDownloader(url1, file, &Config{ 32 | MaxParallel: 10, 33 | CacheSize: 8192, 34 | InstanceStatePath: "tmp.txt", 35 | }) 36 | 37 | client := requester.NewHTTPClient() 38 | client.SetTimeout(10 * time.Second) 39 | d.SetClient(client) 40 | 41 | go func() { 42 | for { 43 | if d.monitor != nil { 44 | fmt.Println(d.monitor) 45 | } 46 | time.Sleep(1e9) 47 | } 48 | }() 49 | go func() { 50 | time.Sleep(3e9) 51 | d.Pause() 52 | time.Sleep(5e9) 53 | d.Resume() 54 | time.Sleep(9e9) 55 | d.Pause() 56 | time.Sleep(5e9) 57 | d.Resume() 58 | time.Sleep(3e9) 59 | d.Cancel() 60 | fmt.Println("canceled") 61 | time.Sleep(3e9) 62 | }() 63 | err := d.Execute() 64 | if err != nil { 65 | fmt.Println(err) 66 | } 67 | } 68 | 69 | func newSlice() [][]byte { 70 | s := make([][]byte, 20) 71 | s[0] = []byte("kjashdfiuqwheirhwuq") 72 | s[9] = []byte("kjashdfiuqwheirhwuq") 73 | return s 74 | } 75 | 76 | func rangeSlice(f func(key int, by []byte) bool) { 77 | s := newSlice() 78 | for k := range s { 79 | if s[k] == nil { 80 | continue 81 | } 82 | if !f(k, s[k]) { 83 | break 84 | } 85 | } 86 | } 87 | 88 | func BenchmarkRange1(b *testing.B) { 89 | for i := 0; i < b.N; i++ { 90 | var a = 0 91 | rangeSlice(func(key int, s []byte) bool { 92 | a++ 93 | return true 94 | }) 95 | } 96 | } 97 | 98 | func BenchmarkRange2(b *testing.B) { 99 | for i := 0; i < b.N; i++ { 100 | s := newSlice() 101 | a := 0 102 | for k := range s { 103 | if s[k] == nil { 104 | continue 105 | } 106 | a++ 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /utils/cachepool/idcachepool.go: -------------------------------------------------------------------------------- 1 | // Package cachepool []byte缓存池 2 | package cachepool 3 | 4 | import ( 5 | "sync" 6 | "sync/atomic" 7 | ) 8 | 9 | var ( 10 | // IDCachePool []byte 缓存池 11 | IDCachePool = cachePool{ 12 | cachepool: sync.Map{}, 13 | } 14 | ) 15 | 16 | type cachePool struct { 17 | lastID int32 18 | cachepool sync.Map 19 | } 20 | 21 | func (cp *cachePool) Apply(size int) (id int32) { 22 | for { 23 | _, ok := cp.cachepool.Load(cp.lastID) 24 | atomic.AddInt32(&cp.lastID, 1) 25 | if ok { 26 | continue 27 | } 28 | break 29 | } 30 | 31 | cp.Set(cp.lastID, size) 32 | return cp.lastID 33 | } 34 | 35 | func (cp *cachePool) Existed(id int32) (existed bool) { 36 | _, existed = cp.cachepool.Load(id) 37 | return 38 | } 39 | 40 | func (cp *cachePool) Get(id int32) []byte { 41 | cache, ok := cp.cachepool.Load(id) 42 | if !ok { 43 | return nil 44 | } 45 | return cache.([]byte) 46 | } 47 | 48 | func (cp *cachePool) Set(id int32, size int) []byte { 49 | cache := RawMallocByteSlice(size) 50 | cp.cachepool.Store(id, cache) 51 | return cp.Get(id) 52 | } 53 | 54 | func (cp *cachePool) SetIfNotExist(id int32, size int) []byte { 55 | ok := cp.Existed(id) 56 | cache := cp.Get(id) 57 | if !ok || len(cache) < size { 58 | cache = nil 59 | cp.Delete(id) 60 | cp.Set(id, size) 61 | } 62 | return cp.Get(id) 63 | } 64 | 65 | func (cp *cachePool) Delete(id int32) { 66 | cp.cachepool.Store(id, nil) 67 | cp.cachepool.Delete(id) 68 | } 69 | 70 | func (cp *cachePool) DeleteAll() { 71 | cp.cachepool.Range(func(k interface{}, _ interface{}) bool { 72 | cp.Delete(k.(int32)) 73 | return true 74 | }) 75 | } 76 | 77 | // Apply 申请缓存, 返回缓存id 78 | func Apply(size int) (id int32) { 79 | return IDCachePool.Apply(size) 80 | } 81 | 82 | // Existed 通过缓存id检测是否存在缓存 83 | func Existed(id int32) bool { 84 | return IDCachePool.Existed(id) 85 | } 86 | 87 | // Get 通过缓存id获取缓存[]byte 88 | func Get(id int32) []byte { 89 | return IDCachePool.Get(id) 90 | } 91 | 92 | // Set 设置缓存, 通过给定的缓存id 93 | func Set(id int32, size int) []byte { 94 | return IDCachePool.Set(id, size) 95 | } 96 | 97 | // SetIfNotExist 如果缓存不存在, 则设置缓存池 98 | func SetIfNotExist(id int32, size int) []byte { 99 | return IDCachePool.SetIfNotExist(id, size) 100 | } 101 | 102 | // Delete 通过缓存id删除缓存 103 | func Delete(id int32) { 104 | IDCachePool.Delete(id) 105 | } 106 | 107 | // DeleteAll 清空缓存池 108 | func DeleteAll() { 109 | IDCachePool.DeleteAll() 110 | } 111 | -------------------------------------------------------------------------------- /utils/bdcrypto/rsa.go: -------------------------------------------------------------------------------- 1 | package bdcrypto 2 | 3 | import ( 4 | "crypto/x509" 5 | "encoding/pem" 6 | "errors" 7 | "math/big" 8 | ) 9 | 10 | const ( 11 | // DefaultRSAPublicKeyModulus 默认的公钥模数 12 | DefaultRSAPublicKeyModulus = "AE47B04D3A55A5FDABC612A426D84484BCB1C29C63BBAC33544A1BB94D930772E6E201CF2B39B5B6EDED1CCCBB5E4DCE713B87C6DD88C3DBBEE3A1FBE220723F01E2AA81ED9497C8FFB05FF54A3E982A76D682B0AABC60DBF9D1A8243FE2922E43DD5DF9C259442147BBF4717E5ED8D4C1BD5344DD1A8F35B631D80AB45A9BC7" 13 | 14 | // DefaultRSAPublicKeyExponent 默认的公钥指数 15 | DefaultRSAPublicKeyExponent = 0x10001 16 | 17 | // DefaultRSAPrivateKey 默认的私钥 18 | DefaultRSAPrivateKey = `-----BEGIN RSA PRIVATE KEY----- 19 | MIICXAIBAAKBgQCuR7BNOlWl/avGEqQm2ESEvLHCnGO7rDNUShu5TZMHcubiAc8r 20 | ObW27e0czLteTc5xO4fG3YjD277jofviIHI/AeKqge2Ul8j/sF/1Sj6YKnbWgrCq 21 | vGDb+dGoJD/iki5D3V35wllEIUe79HF+XtjUwb1TRN0ajzW2MdgKtFqbxwIDAQAB 22 | AoGAW0CoHFe9/tLq/SRHlRtKDSJsBRUz11Fb8vd2urjWkmDkaVQ/MEfgUK8Vpy2/ 23 | saoVvQ5JkqPud3b45WGsbINGrb8saugZ1h5huDbuxVXKDj1ZWyJPkmxHLUK2+7iL 24 | 5c7F7+v2C+n6polIgMV9SbLXD6YIXUJ+GengWQffhTRE7WECQQDj/g5x7Rj5vc7X 25 | o3i0SQmyN4RcxxOWfiLe5OUASKM2UPVBQKI3CugkmiTaXTi7auuG3I4GVPRHVHw9 26 | y/Ekz7J3AkEAw7B5+uI60MwcDMeGoXAMAEYe/s7LhyBICarY6cNwySb46B7OHEUz 27 | ooFV2qx31I6ivpMRwCqrRKXEvjPEAfPlMQJAGrXi/1nluSyRlRXjyEteRXDXov73 28 | voPclfx/D79yz6RAd3qZBpXSiKc+dg7B3MM0AMLKKNe/HrQ5Mgw4njVvFQJAPKvX 29 | ddh0Qc42mCO4cw8JOYCEFZ5J7fAtRYoJzJhCvKrvmxAJ+SvfcW/GDZFRab57aLiy 30 | VTElfpgiopHsIGrc0QJBAMliJywM9BNYn9Q4aqKN/dR22W/gctfa6bxU1m9SfJ5t 31 | 5G8MR8HsB/9Cafv8f+KnFzp2SncEu0zuFc/S8n5X5v0= 32 | -----END RSA PRIVATE KEY-----` 33 | ) 34 | 35 | // RSAEncryptNoPadding 无填充模式的 RSA 加密 36 | func RSAEncryptNoPadding(rsaPublicKeyModulus string, rsaPublicKeyExponent int64, origData []byte) (ciphertext []byte, err error) { 37 | var m = new(big.Int) 38 | _, ok := m.SetString(rsaPublicKeyModulus, 16) 39 | if !ok { 40 | return nil, errors.New("rsaPublicKeyModulus is invalid") 41 | } 42 | 43 | c := new(big.Int).SetBytes(origData) 44 | return c.Exp(c, big.NewInt(rsaPublicKeyExponent), m).Bytes(), nil 45 | } 46 | 47 | // RSADecryptNoPadding 无填充模式的 RSA 解密 48 | func RSADecryptNoPadding(rsaPrivateKey string, ciphertext []byte) ([]byte, error) { 49 | block, _ := pem.Decode([]byte(rsaPrivateKey)) 50 | if block == nil { 51 | return nil, errors.New("private key error") 52 | } 53 | 54 | priv, err := x509.ParsePKCS1PrivateKey(block.Bytes) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | c := new(big.Int).SetBytes(ciphertext) 60 | return c.Exp(c, priv.D, priv.N).Bytes(), nil 61 | } 62 | -------------------------------------------------------------------------------- /requester/uploader/status.go: -------------------------------------------------------------------------------- 1 | package uploader 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type ( 8 | // Status 上传状态接口 9 | Status interface { 10 | TotalSize() int64 // 总大小 11 | Uploaded() int64 // 已上传数据 12 | SpeedsPerSecond() int64 // 每秒的上传速度 13 | TimeElapsed() time.Duration // 上传时间 14 | } 15 | 16 | // UploadStatus 上传状态 17 | UploadStatus struct { 18 | totalSize int64 // 总大小 19 | uploaded int64 // 已上传数据 20 | speedsPerSecond int64 // 每秒的上传速度 21 | timeElapsed time.Duration // 上传时间 22 | } 23 | 24 | UploadStatusFunc func(status Status, updateChan <-chan struct{}) 25 | ) 26 | 27 | // TotalSize 返回总大小 28 | func (us *UploadStatus) TotalSize() int64 { 29 | return us.totalSize 30 | } 31 | 32 | // Uploaded 返回已上传数据 33 | func (us *UploadStatus) Uploaded() int64 { 34 | return us.uploaded 35 | } 36 | 37 | // SpeedsPerSecond 返回每秒的上传速度 38 | func (us *UploadStatus) SpeedsPerSecond() int64 { 39 | return us.speedsPerSecond 40 | } 41 | 42 | // TimeElapsed 返回上传时间 43 | func (us *UploadStatus) TimeElapsed() time.Duration { 44 | return us.timeElapsed 45 | } 46 | 47 | // GetStatusChan 获取上传状态 48 | func (u *Uploader) GetStatusChan() <-chan Status { 49 | c := make(chan Status) 50 | 51 | go func() { 52 | for { 53 | select { 54 | case <-u.finished: 55 | close(c) 56 | return 57 | default: 58 | if !u.executed { 59 | time.Sleep(1 * time.Second) 60 | continue 61 | } 62 | 63 | old := u.readed64.Readed() 64 | time.Sleep(1 * time.Second) // 每秒统计 65 | 66 | readed := u.readed64.Readed() 67 | c <- &UploadStatus{ 68 | totalSize: u.readed64.Len(), 69 | uploaded: readed, 70 | speedsPerSecond: readed - old, 71 | timeElapsed: time.Since(u.executeTime) / 1e7 * 1e7, 72 | } 73 | } 74 | } 75 | }() 76 | return c 77 | } 78 | 79 | func (muer *MultiUploader) uploadStatusEvent() { 80 | if muer.onUploadStatusEvent == nil { 81 | return 82 | } 83 | 84 | go func() { 85 | ticker := time.NewTicker(1 * time.Second) // 每秒统计 86 | defer ticker.Stop() 87 | for { 88 | select { 89 | case <-muer.finished: 90 | return 91 | case <-ticker.C: 92 | readed := muer.workers.Readed() 93 | muer.onUploadStatusEvent(&UploadStatus{ 94 | totalSize: muer.file.Len(), 95 | uploaded: readed, 96 | speedsPerSecond: muer.speedsStat.GetSpeeds(), 97 | timeElapsed: time.Since(muer.executeTime) / 1e8 * 1e8, 98 | }, muer.updateInstanceStateChan) 99 | } 100 | } 101 | }() 102 | } 103 | -------------------------------------------------------------------------------- /requester/downloader/status.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "github.com/iikira/iikira-go-utils/requester/transfer" 5 | ) 6 | 7 | type ( 8 | //WorkerStatuser 状态 9 | WorkerStatuser interface { 10 | StatusCode() StatusCode //状态码 11 | StatusText() string 12 | } 13 | 14 | //StatusCode 状态码 15 | StatusCode int 16 | 17 | //WorkerStatus worker状态 18 | WorkerStatus struct { 19 | statusCode StatusCode 20 | } 21 | 22 | // DownloadStatusFunc 下载状态处理函数 23 | DownloadStatusFunc func(status transfer.DownloadStatuser, workersCallback func(RangeWorkerFunc)) 24 | ) 25 | 26 | const ( 27 | //StatusCodeInit 初始化 28 | StatusCodeInit StatusCode = iota 29 | //StatusCodeSuccessed 成功 30 | StatusCodeSuccessed 31 | //StatusCodePending 等待响应 32 | StatusCodePending 33 | //StatusCodeDownloading 下载中 34 | StatusCodeDownloading 35 | //StatusCodeWaitToWrite 等待写入数据 36 | StatusCodeWaitToWrite 37 | //StatusCodeInternalError 内部错误 38 | StatusCodeInternalError 39 | //StatusCodeTooManyConnections 连接数太多 40 | StatusCodeTooManyConnections 41 | //StatusCodeNetError 网络错误 42 | StatusCodeNetError 43 | //StatusCodeFailed 下载失败 44 | StatusCodeFailed 45 | //StatusCodePaused 已暂停 46 | StatusCodePaused 47 | //StatusCodeReseted 已重设连接 48 | StatusCodeReseted 49 | //StatusCodeCanceled 已取消 50 | StatusCodeCanceled 51 | ) 52 | 53 | //GetStatusText 根据状态码获取状态信息 54 | func GetStatusText(sc StatusCode) string { 55 | switch sc { 56 | case StatusCodeInit: 57 | return "初始化" 58 | case StatusCodeSuccessed: 59 | return "成功" 60 | case StatusCodePending: 61 | return "等待响应" 62 | case StatusCodeDownloading: 63 | return "下载中" 64 | case StatusCodeWaitToWrite: 65 | return "等待写入数据" 66 | case StatusCodeInternalError: 67 | return "内部错误" 68 | case StatusCodeTooManyConnections: 69 | return "连接数太多" 70 | case StatusCodeNetError: 71 | return "网络错误" 72 | case StatusCodeFailed: 73 | return "下载失败" 74 | case StatusCodePaused: 75 | return "已暂停" 76 | case StatusCodeReseted: 77 | return "已重设连接" 78 | case StatusCodeCanceled: 79 | return "已取消" 80 | default: 81 | return "未知状态码" 82 | } 83 | } 84 | 85 | //NewWorkerStatus 初始化WorkerStatus 86 | func NewWorkerStatus() *WorkerStatus { 87 | return &WorkerStatus{ 88 | statusCode: StatusCodeInit, 89 | } 90 | } 91 | 92 | //SetStatusCode 设置worker状态码 93 | func (ws *WorkerStatus) SetStatusCode(sc StatusCode) { 94 | ws.statusCode = sc 95 | } 96 | 97 | //StatusCode 返回状态码 98 | func (ws *WorkerStatus) StatusCode() StatusCode { 99 | return ws.statusCode 100 | } 101 | 102 | //StatusText 返回状态信息 103 | func (ws *WorkerStatus) StatusText() string { 104 | return GetStatusText(ws.statusCode) 105 | } 106 | -------------------------------------------------------------------------------- /cmd/AndroidNDKBuild/main.go: -------------------------------------------------------------------------------- 1 | // AndroidNDKBuild 2 | // go build -ldflags "-X main.APILevel=15 -X main.Arch=x86_64" 3 | // env ANDROID_API_LEVEL NDK ANDROID_NDK_ROOT GOARCH 4 | 5 | package main 6 | 7 | import ( 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | "runtime" 12 | "syscall" 13 | ) 14 | 15 | var ( 16 | // NDKPath path to Android NDK 17 | NDKPath string 18 | // APILevel Android api level 19 | APILevel string 20 | // Arch arch 21 | Arch string 22 | ) 23 | 24 | func getNDKPath() string { 25 | ndkPath, ok := os.LookupEnv("NDK") 26 | if ok { 27 | return ndkPath 28 | } 29 | ndkPath, ok = os.LookupEnv("ANDROID_NDK_ROOT") 30 | if ok { 31 | return ndkPath 32 | } 33 | ndkPath, ok = os.LookupEnv("ANDROID_NDK_DIR") 34 | if ok { 35 | return ndkPath 36 | } 37 | return "" 38 | } 39 | 40 | func getAPILevel() string { 41 | apiLevelStr, ok := os.LookupEnv("ANDROID_API_LEVEL") 42 | if ok { 43 | return apiLevelStr 44 | } 45 | return "21" 46 | } 47 | 48 | func getGoarch() string { 49 | arch, ok := os.LookupEnv("GOARCH") 50 | if ok { 51 | return arch 52 | } 53 | 54 | return runtime.GOARCH 55 | } 56 | 57 | func getArch() string { 58 | if Arch != "" { 59 | return Arch 60 | } 61 | goarch := getGoarch() 62 | switch goarch { 63 | case "386": 64 | return "x86" 65 | case "amd64": 66 | return "x86_64" 67 | case "arm64": 68 | return "aarch64" 69 | } 70 | return goarch 71 | } 72 | 73 | func getPlatformsArch() string { 74 | arch := getArch() 75 | switch arch { 76 | case "aarch64": 77 | return "arm64" 78 | } 79 | return arch 80 | } 81 | 82 | func main() { 83 | if NDKPath == "" { 84 | NDKPath = getNDKPath() 85 | } 86 | if APILevel == "" { 87 | APILevel = getAPILevel() 88 | } 89 | if Arch == "" { 90 | Arch = getArch() 91 | } 92 | 93 | lastPattern := "*-gcc" 94 | if runtime.GOOS == "windows" { 95 | lastPattern += ".exe" 96 | } 97 | 98 | gccPaths, err := filepath.Glob(filepath.Join(NDKPath, "toolchains", getArch()+"-*", "prebuilt", runtime.GOOS+"-*", "bin", lastPattern)) 99 | checkErr(err) 100 | if len(gccPaths) == 0 { 101 | panic("no match gcc") 102 | } 103 | 104 | args := make([]string, len(os.Args)) 105 | copy(args[1:], os.Args[1:]) 106 | args[0] = "--sysroot=" + filepath.Join(NDKPath, "platforms", "android-"+APILevel, "arch-"+getPlatformsArch()) 107 | 108 | gccExec := exec.Command(gccPaths[0], args...) 109 | gccExec.Stdout = os.Stdout 110 | gccExec.Stderr = os.Stderr 111 | 112 | err = gccExec.Run() 113 | exitError, ok := err.(*exec.ExitError) 114 | if ok { 115 | status := exitError.ProcessState.Sys().(syscall.WaitStatus) 116 | os.Exit(status.ExitStatus()) 117 | } 118 | 119 | if err != nil { 120 | println(err.Error()) 121 | } 122 | 123 | return 124 | } 125 | 126 | func checkErr(err error) { 127 | if err != nil { 128 | panic(err) 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /requester/uploader/uploader.go: -------------------------------------------------------------------------------- 1 | // Package uploader 上传包 2 | package uploader 3 | 4 | import ( 5 | "github.com/iikira/iikira-go-utils/pcsverbose" 6 | "github.com/iikira/iikira-go-utils/requester" 7 | "github.com/iikira/iikira-go-utils/requester/rio" 8 | "github.com/iikira/iikira-go-utils/utils" 9 | "github.com/iikira/iikira-go-utils/utils/converter" 10 | "net/http" 11 | "time" 12 | ) 13 | 14 | const ( 15 | // BufioReadSize bufio 缓冲区大小, 用于上传时读取文件 16 | BufioReadSize = int(64 * converter.KB) // 64KB 17 | ) 18 | 19 | type ( 20 | //CheckFunc 上传完成的检测函数 21 | CheckFunc func(resp *http.Response, uploadErr error) 22 | 23 | // Uploader 上传 24 | Uploader struct { 25 | url string // 上传地址 26 | readed64 Readed64 // 要上传的对象 27 | contentType string 28 | 29 | client *requester.HTTPClient 30 | 31 | executeTime time.Time 32 | executed bool 33 | finished chan struct{} 34 | 35 | checkFunc CheckFunc 36 | onExecute func() 37 | onFinish func() 38 | } 39 | ) 40 | 41 | var ( 42 | uploaderVerbose = pcsverbose.New("UPLOADER") 43 | ) 44 | 45 | // NewUploader 返回 uploader 对象, url: 上传地址, readerlen64: 实现 rio.ReaderLen64 接口的对象, 例如文件 46 | func NewUploader(url string, readerlen64 rio.ReaderLen64) (uploader *Uploader) { 47 | uploader = &Uploader{ 48 | url: url, 49 | readed64: NewReaded64(readerlen64), 50 | } 51 | 52 | return 53 | } 54 | 55 | func (u *Uploader) lazyInit() { 56 | if u.finished == nil { 57 | u.finished = make(chan struct{}) 58 | } 59 | if u.client == nil { 60 | u.client = requester.NewHTTPClient() 61 | } 62 | u.client.SetTimeout(0) 63 | u.client.SetResponseHeaderTimeout(0) 64 | } 65 | 66 | // SetClient 设置http客户端 67 | func (u *Uploader) SetClient(c *requester.HTTPClient) { 68 | u.client = c 69 | } 70 | 71 | //SetContentType 设置Content-Type 72 | func (u *Uploader) SetContentType(contentType string) { 73 | u.contentType = contentType 74 | } 75 | 76 | //SetCheckFunc 设置上传完成的检测函数 77 | func (u *Uploader) SetCheckFunc(checkFunc CheckFunc) { 78 | u.checkFunc = checkFunc 79 | } 80 | 81 | // Execute 执行上传, 收到返回值信号则为上传结束 82 | func (u *Uploader) Execute() { 83 | utils.Trigger(u.onExecute) 84 | 85 | // 开始上传 86 | u.executeTime = time.Now() 87 | u.executed = true 88 | resp, _, err := u.execute() 89 | 90 | // 上传结束 91 | close(u.finished) 92 | 93 | if u.checkFunc != nil { 94 | u.checkFunc(resp, err) 95 | } 96 | 97 | utils.Trigger(u.onFinish) // 触发上传结束的事件 98 | } 99 | 100 | func (u *Uploader) execute() (resp *http.Response, code int, err error) { 101 | u.lazyInit() 102 | header := map[string]string{} 103 | if u.contentType != "" { 104 | header["Content-Type"] = u.contentType 105 | } 106 | 107 | resp, err = u.client.Req(http.MethodPost, u.url, u.readed64, header) 108 | if err != nil { 109 | return nil, 2, err 110 | } 111 | 112 | return resp, 0, nil 113 | } 114 | 115 | // OnExecute 任务开始时触发的事件 116 | func (u *Uploader) OnExecute(fn func()) { 117 | u.onExecute = fn 118 | } 119 | 120 | // OnFinish 任务完成时触发的事件 121 | func (u *Uploader) OnFinish(fn func()) { 122 | u.onFinish = fn 123 | } 124 | -------------------------------------------------------------------------------- /requester/http_client.go: -------------------------------------------------------------------------------- 1 | package requester 2 | 3 | import ( 4 | "crypto/tls" 5 | "net/http" 6 | "net/http/cookiejar" 7 | "time" 8 | ) 9 | 10 | // HTTPClient http client 11 | type HTTPClient struct { 12 | http.Client 13 | transport *http.Transport 14 | https bool 15 | UserAgent string 16 | } 17 | 18 | // NewHTTPClient 返回 HTTPClient 的指针, 19 | // 预设了一些配置 20 | func NewHTTPClient() *HTTPClient { 21 | h := &HTTPClient{ 22 | Client: http.Client{ 23 | Timeout: 30 * time.Second, 24 | }, 25 | UserAgent: UserAgent, 26 | } 27 | h.Client.Jar, _ = cookiejar.New(nil) 28 | return h 29 | } 30 | 31 | func (h *HTTPClient) lazyInit() { 32 | if h.transport == nil { 33 | h.transport = &http.Transport{ 34 | Proxy: proxyFunc, 35 | DialContext: dialContext, 36 | Dial: dial, 37 | // DialTLS: h.dialTLSFunc(), 38 | TLSClientConfig: &tls.Config{ 39 | InsecureSkipVerify: !h.https, 40 | }, 41 | TLSHandshakeTimeout: 10 * time.Second, 42 | DisableKeepAlives: false, 43 | DisableCompression: false, // gzip 44 | MaxIdleConns: 100, 45 | IdleConnTimeout: 90 * time.Second, 46 | ResponseHeaderTimeout: 10 * time.Second, 47 | ExpectContinueTimeout: 10 * time.Second, 48 | } 49 | h.Client.Transport = h.transport 50 | } 51 | } 52 | 53 | // SetUserAgent 设置 UserAgent 浏览器标识 54 | func (h *HTTPClient) SetUserAgent(ua string) { 55 | h.UserAgent = ua 56 | } 57 | 58 | // SetProxy 设置代理 59 | func (h *HTTPClient) SetProxy(proxyAddr string) { 60 | h.lazyInit() 61 | u, err := checkProxyAddr(proxyAddr) 62 | if err != nil { 63 | h.transport.Proxy = http.ProxyFromEnvironment 64 | return 65 | } 66 | 67 | h.transport.Proxy = http.ProxyURL(u) 68 | } 69 | 70 | // SetCookiejar 设置 cookie 71 | func (h *HTTPClient) SetCookiejar(jar http.CookieJar) { 72 | h.Client.Jar = jar 73 | } 74 | 75 | // ResetCookiejar 清空 cookie 76 | func (h *HTTPClient) ResetCookiejar() { 77 | h.Jar, _ = cookiejar.New(nil) 78 | } 79 | 80 | // SetHTTPSecure 是否启用 https 安全检查, 默认不检查 81 | func (h *HTTPClient) SetHTTPSecure(b bool) { 82 | h.https = b 83 | h.lazyInit() 84 | if b { 85 | h.transport.TLSClientConfig = nil 86 | } else { 87 | h.transport.TLSClientConfig = &tls.Config{ 88 | InsecureSkipVerify: !b, 89 | } 90 | } 91 | } 92 | 93 | // SetKeepAlive 设置 Keep-Alive 94 | func (h *HTTPClient) SetKeepAlive(b bool) { 95 | h.lazyInit() 96 | h.transport.DisableKeepAlives = !b 97 | } 98 | 99 | // SetGzip 是否启用Gzip 100 | func (h *HTTPClient) SetGzip(b bool) { 101 | h.lazyInit() 102 | h.transport.DisableCompression = !b 103 | } 104 | 105 | // SetResponseHeaderTimeout 设置目标服务器响应超时时间 106 | func (h *HTTPClient) SetResponseHeaderTimeout(t time.Duration) { 107 | h.lazyInit() 108 | h.transport.ResponseHeaderTimeout = t 109 | } 110 | 111 | // SetTLSHandshakeTimeout 设置tls握手超时时间 112 | func (h *HTTPClient) SetTLSHandshakeTimeout(t time.Duration) { 113 | h.lazyInit() 114 | h.transport.TLSHandshakeTimeout = t 115 | } 116 | 117 | // SetTimeout 设置 http 请求超时时间, 默认30s 118 | func (h *HTTPClient) SetTimeout(t time.Duration) { 119 | h.Client.Timeout = t 120 | } 121 | -------------------------------------------------------------------------------- /utils/file.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "github.com/iikira/iikira-go-utils/pcsverbose" 5 | "github.com/kardianos/osext" 6 | "os" 7 | "path" 8 | "path/filepath" 9 | "runtime" 10 | "strings" 11 | ) 12 | 13 | func IsPipeInput() bool { 14 | fileInfo, err := os.Stdin.Stat() 15 | if err != nil { 16 | return false 17 | } 18 | return (fileInfo.Mode() & os.ModeNamedPipe) == os.ModeNamedPipe 19 | } 20 | 21 | // IsIPhoneOS 是否为苹果移动设备 22 | func IsIPhoneOS() bool { 23 | if runtime.GOOS == "darwin" && (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64") { 24 | _, err := os.Stat("Info.plist") 25 | return err == nil 26 | } 27 | return false 28 | } 29 | 30 | // ChWorkDir 切换回工作目录 31 | func ChWorkDir() { 32 | if !IsIPhoneOS() { 33 | return 34 | } 35 | 36 | dir, err := filepath.Abs("") 37 | if err != nil { 38 | return 39 | } 40 | 41 | subPath := filepath.Dir(os.Args[0]) 42 | os.Chdir(strings.TrimSuffix(dir, subPath)) 43 | } 44 | 45 | // Executable 获取程序所在的真实目录或真实相对路径 46 | func Executable() string { 47 | executablePath, err := osext.Executable() 48 | if err != nil { 49 | pcsverbose.Verbosef("DEBUG: osext.Executable: %s\n", err) 50 | executablePath, err = filepath.Abs(filepath.Dir(os.Args[0])) 51 | if err != nil { 52 | pcsverbose.Verbosef("DEBUG: filepath.Abs: %s\n", err) 53 | executablePath = filepath.Dir(os.Args[0]) 54 | } 55 | } 56 | 57 | if IsIPhoneOS() { 58 | executablePath = filepath.Join(strings.TrimSuffix(executablePath, os.Args[0]), filepath.Base(os.Args[0])) 59 | } 60 | 61 | // 读取链接 62 | linkedExecutablePath, err := filepath.EvalSymlinks(executablePath) 63 | if err != nil { 64 | pcsverbose.Verbosef("DEBUG: filepath.EvalSymlinks: %s\n", err) 65 | return executablePath 66 | } 67 | return linkedExecutablePath 68 | } 69 | 70 | // ExecutablePath 获取程序所在目录 71 | func ExecutablePath() string { 72 | return filepath.Dir(Executable()) 73 | } 74 | 75 | // ExecutablePathJoin 返回程序所在目录的子目录 76 | func ExecutablePathJoin(subPath string) string { 77 | return filepath.Join(ExecutablePath(), subPath) 78 | } 79 | 80 | // WalkDir 获取指定目录及所有子目录下的所有文件，可以匹配后缀过滤。 81 | // 支持 Linux/macOS 软链接 82 | func WalkDir(dirPth, suffix string) (files []string, err error) { 83 | files = make([]string, 0, 32) 84 | suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写 85 | 86 | var walkFunc filepath.WalkFunc 87 | walkFunc = func(filename string, fi os.FileInfo, err error) error { //遍历目录 88 | if err != nil { 89 | return err 90 | } 91 | if fi.IsDir() { // 忽略目录 92 | return nil 93 | } 94 | if fi.Mode()&os.ModeSymlink != 0 { // 读取 symbol link 95 | err = filepath.Walk(filename+string(os.PathSeparator), walkFunc) 96 | return err 97 | } 98 | 99 | if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) { 100 | files = append(files, path.Clean(filename)) 101 | } 102 | return nil 103 | } 104 | 105 | err = filepath.Walk(dirPth, walkFunc) 106 | return files, err 107 | } 108 | 109 | // ConvertToUnixPathSeparator 将 windows 目录分隔符转换为 Unix 的 110 | func ConvertToUnixPathSeparator(p string) string { 111 | return strings.Replace(p, "\\", "/", -1) 112 | } 113 | -------------------------------------------------------------------------------- /utils/converter/converter.go: -------------------------------------------------------------------------------- 1 | // Package converter 格式, 类型转换包 2 | package converter 3 | 4 | import ( 5 | "github.com/mattn/go-runewidth" 6 | "reflect" 7 | "strconv" 8 | "strings" 9 | "unicode" 10 | "unsafe" 11 | ) 12 | 13 | const ( 14 | // InvalidChars 文件名中的非法字符 15 | InvalidChars = `\/:*?"<>|` 16 | ) 17 | 18 | // ToString unsafe 转换, 将 []byte 转换为 string 19 | func ToString(p []byte) string { 20 | return *(*string)(unsafe.Pointer(&p)) 21 | } 22 | 23 | // ToBytes unsafe 转换, 将 string 转换为 []byte 24 | func ToBytes(str string) []byte { 25 | strHeader := (*reflect.StringHeader)(unsafe.Pointer(&str)) 26 | return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ 27 | Data: strHeader.Data, 28 | Len: strHeader.Len, 29 | Cap: strHeader.Len, 30 | })) 31 | } 32 | 33 | // ToBytesUnsafe unsafe 转换, 请确保转换后的 []byte 不涉及 cap() 操作, 将 string 转换为 []byte 34 | func ToBytesUnsafe(str string) []byte { 35 | return *(*[]byte)(unsafe.Pointer(&str)) 36 | } 37 | 38 | // IntToBool int 类型转换为 bool 39 | func IntToBool(i int) bool { 40 | return i != 0 41 | } 42 | 43 | // SliceInt64ToString []int64 转换为 []string 44 | func SliceInt64ToString(si []int64) (ss []string) { 45 | ss = make([]string, 0, len(si)) 46 | for k := range si { 47 | ss = append(ss, strconv.FormatInt(si[k], 10)) 48 | } 49 | return ss 50 | } 51 | 52 | // SliceStringToInt64 []string 转换为 []int64 53 | func SliceStringToInt64(ss []string) (si []int64) { 54 | si = make([]int64, 0, len(ss)) 55 | var ( 56 | i int64 57 | err error 58 | ) 59 | for k := range ss { 60 | i, err = strconv.ParseInt(ss[k], 10, 64) 61 | if err != nil { 62 | continue 63 | } 64 | si = append(si, i) 65 | } 66 | return 67 | } 68 | 69 | // SliceStringToInt []string 转换为 []int 70 | func SliceStringToInt(ss []string) (si []int) { 71 | si = make([]int, 0, len(ss)) 72 | var ( 73 | i int 74 | err error 75 | ) 76 | for k := range ss { 77 | i, err = strconv.Atoi(ss[k]) 78 | if err != nil { 79 | continue 80 | } 81 | si = append(si, i) 82 | } 83 | return 84 | } 85 | 86 | // MustInt 将string转换为int, 忽略错误 87 | func MustInt(s string) (n int) { 88 | n, _ = strconv.Atoi(s) 89 | return 90 | } 91 | 92 | // MustInt64 将string转换为int64, 忽略错误 93 | func MustInt64(s string) (i int64) { 94 | i, _ = strconv.ParseInt(s, 10, 64) 95 | return 96 | } 97 | 98 | // ShortDisplay 缩略显示字符串s, 显示长度为num, 缩略的内容用"..."填充 99 | func ShortDisplay(s string, num int) string { 100 | var ( 101 | sb = strings.Builder{} 102 | n int 103 | ) 104 | for _, v := range s { 105 | if unicode.Is(unicode.C, v) { // 去除无效字符 106 | continue 107 | } 108 | n += runewidth.RuneWidth(v) 109 | if n > num { 110 | sb.WriteString("...") 111 | break 112 | } 113 | sb.WriteRune(v) 114 | } 115 | 116 | return sb.String() 117 | } 118 | 119 | // TrimPathInvalidChars 清除文件名中的非法字符 120 | func TrimPathInvalidChars(fpath string) string { 121 | buf := make([]byte, 0, len(fpath)) 122 | 123 | for _, c := range ToBytesUnsafe(fpath) { 124 | if strings.ContainsRune(InvalidChars, rune(c)) { 125 | continue 126 | } 127 | 128 | buf = append(buf, c) 129 | } 130 | 131 | return ToString(buf) 132 | } 133 | -------------------------------------------------------------------------------- /requester/uploader/multiworker.go: -------------------------------------------------------------------------------- 1 | package uploader 2 | 3 | import ( 4 | "context" 5 | "github.com/iikira/iikira-go-utils/utils/waitgroup" 6 | "github.com/oleiade/lane" 7 | "os" 8 | ) 9 | 10 | type ( 11 | worker struct { 12 | id int 13 | partOffset int64 14 | splitUnit SplitUnit 15 | checksum string 16 | } 17 | 18 | workerList []*worker 19 | ) 20 | 21 | // CheckSumList 返回所以worker的checksum 22 | // TODO: 实现sort 23 | func (werl *workerList) CheckSumList() []string { 24 | checksumList := make([]string, 0, len(*werl)) 25 | for _, wer := range *werl { 26 | checksumList = append(checksumList, wer.checksum) 27 | } 28 | return checksumList 29 | } 30 | 31 | func (werl *workerList) Readed() int64 { 32 | var readed int64 33 | for _, wer := range *werl { 34 | readed += wer.splitUnit.Readed() 35 | } 36 | return readed 37 | } 38 | 39 | func (muer *MultiUploader) upload() (uperr error) { 40 | err := muer.multiUpload.Precreate() 41 | if err != nil { 42 | return err 43 | } 44 | 45 | var ( 46 | uploadDeque = lane.NewDeque() 47 | ) 48 | 49 | // 加入队列 50 | for _, wer := range muer.workers { 51 | if wer.checksum == "" { 52 | uploadDeque.Append(wer) 53 | } 54 | } 55 | 56 | for { 57 | wg := waitgroup.NewWaitGroup(muer.config.Parallel) 58 | for { 59 | e := uploadDeque.Shift() 60 | if e == nil { // 任务为空 61 | break 62 | } 63 | 64 | wer := e.(*worker) 65 | wg.AddDelta() 66 | go func() { 67 | defer wg.Done() 68 | 69 | var ( 70 | ctx, cancel = context.WithCancel(context.Background()) 71 | doneChan = make(chan struct{}) 72 | checksum string 73 | terr error 74 | ) 75 | go func() { 76 | checksum, terr = muer.multiUpload.TmpFile(ctx, int(wer.id), wer.partOffset, wer.splitUnit) 77 | close(doneChan) 78 | }() 79 | select { 80 | case <-muer.canceled: 81 | cancel() 82 | return 83 | case <-doneChan: 84 | // continue 85 | } 86 | cancel() 87 | if terr != nil { 88 | if me, ok := terr.(*MultiError); ok { 89 | if me.Terminated { // 终止 90 | muer.closeCanceledOnce.Do(func() { // 只关闭一次 91 | close(muer.canceled) 92 | }) 93 | uperr = me.Err 94 | return 95 | } 96 | } 97 | 98 | uploaderVerbose.Warnf("upload err: %s, id: %d\n", terr, wer.id) 99 | wer.splitUnit.Seek(0, os.SEEK_SET) 100 | uploadDeque.Append(wer) 101 | return 102 | } 103 | wer.checksum = checksum 104 | 105 | // 通知更新 106 | if muer.updateInstanceStateChan != nil && len(muer.updateInstanceStateChan) < cap(muer.updateInstanceStateChan) { 107 | muer.updateInstanceStateChan <- struct{}{} 108 | } 109 | }() 110 | } 111 | wg.Wait() 112 | 113 | // 没有任务了 114 | if uploadDeque.Size() == 0 { 115 | break 116 | } 117 | } 118 | 119 | select { 120 | case <-muer.canceled: 121 | if uperr != nil { 122 | return uperr 123 | } 124 | return context.Canceled 125 | default: 126 | } 127 | 128 | cerr := muer.multiUpload.CreateSuperFile(muer.workers.CheckSumList()...) 129 | if cerr != nil { 130 | return cerr 131 | } 132 | 133 | return 134 | } 135 | -------------------------------------------------------------------------------- /utils/checksum/checksum_write.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import ( 4 | "hash" 5 | "io" 6 | ) 7 | 8 | type ( 9 | ChecksumWriter interface { 10 | io.Writer 11 | Sum() interface{} 12 | } 13 | 14 | ChecksumWriteUnit struct { 15 | SliceEnd int64 16 | End int64 17 | SliceSum interface{} 18 | Sum interface{} 19 | OnlySliceSum bool 20 | ChecksumWriter ChecksumWriter 21 | 22 | ptr int64 23 | } 24 | 25 | hashChecksumWriter struct { 26 | h hash.Hash 27 | } 28 | 29 | hash32ChecksumWriter struct { 30 | h hash.Hash32 31 | } 32 | ) 33 | 34 | func (wi *ChecksumWriteUnit) handleEnd() error { 35 | if wi.ptr >= wi.End { 36 | // 已写完 37 | if !wi.OnlySliceSum { 38 | wi.Sum = wi.ChecksumWriter.Sum() 39 | } 40 | return ErrChecksumWriteStop 41 | } 42 | return nil 43 | } 44 | 45 | func (wi *ChecksumWriteUnit) write(p []byte) (n int, err error) { 46 | if wi.End <= 0 { 47 | // do nothing 48 | err = ErrChecksumWriteStop 49 | return 50 | } 51 | err = wi.handleEnd() 52 | if err != nil { 53 | return 54 | } 55 | 56 | var ( 57 | i int 58 | left = wi.End - wi.ptr 59 | lenP = len(p) 60 | ) 61 | if left < int64(lenP) { 62 | // 读取即将完毕 63 | i = int(left) 64 | } else { 65 | i = lenP 66 | } 67 | n, err = wi.ChecksumWriter.Write(p[:i]) 68 | if err != nil { 69 | return 70 | } 71 | wi.ptr += int64(n) 72 | if left < int64(lenP) { 73 | err = wi.handleEnd() 74 | return 75 | } 76 | return 77 | } 78 | 79 | func (wi *ChecksumWriteUnit) Write(p []byte) (n int, err error) { 80 | if wi.SliceEnd <= 0 { // 忽略Slice 81 | // 读取全部 82 | n, err = wi.write(p) 83 | return 84 | } 85 | 86 | // 要计算Slice的情况 87 | // 调整slice 88 | if wi.SliceEnd > wi.End { 89 | wi.SliceEnd = wi.End 90 | } 91 | 92 | // 计算剩余Slice 93 | var ( 94 | sliceLeft = wi.SliceEnd - wi.ptr 95 | ) 96 | if sliceLeft <= 0 { 97 | // 已处理完Slice 98 | if wi.OnlySliceSum { 99 | err = ErrChecksumWriteStop 100 | return 101 | } 102 | 103 | // 继续处理 104 | n, err = wi.write(p) 105 | return 106 | } 107 | 108 | var ( 109 | lenP = len(p) 110 | ) 111 | if sliceLeft <= int64(lenP) { 112 | var n1, n2 int 113 | n1, err = wi.write(p[:sliceLeft]) 114 | n += n1 115 | if err != nil { 116 | return 117 | } 118 | wi.SliceSum = wi.ChecksumWriter.Sum().([]byte) 119 | n2, err = wi.write(p[sliceLeft:]) 120 | n += n2 121 | if err != nil { 122 | return 123 | } 124 | return 125 | } 126 | n, err = wi.write(p) 127 | return 128 | } 129 | 130 | func NewHashChecksumWriter(h hash.Hash) ChecksumWriter { 131 | return &hashChecksumWriter{ 132 | h: h, 133 | } 134 | } 135 | 136 | func (hc *hashChecksumWriter) Write(p []byte) (n int, err error) { 137 | return hc.h.Write(p) 138 | } 139 | 140 | func (hc *hashChecksumWriter) Sum() interface{} { 141 | return hc.h.Sum(nil) 142 | } 143 | 144 | func NewHash32ChecksumWriter(h32 hash.Hash32) ChecksumWriter { 145 | return &hash32ChecksumWriter{ 146 | h: h32, 147 | } 148 | } 149 | 150 | func (hc *hash32ChecksumWriter) Write(p []byte) (n int, err error) { 151 | return hc.h.Write(p) 152 | } 153 | 154 | func (hc *hash32ChecksumWriter) Sum() interface{} { 155 | return hc.h.Sum32() 156 | } 157 | -------------------------------------------------------------------------------- /requester/uploader/block.go: -------------------------------------------------------------------------------- 1 | package uploader 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "github.com/iikira/iikira-go-utils/requester/rio/speeds" 7 | "github.com/iikira/iikira-go-utils/requester/transfer" 8 | "io" 9 | "os" 10 | "sync" 11 | ) 12 | 13 | type ( 14 | // SplitUnit 将 io.ReaderAt 分割单元 15 | SplitUnit interface { 16 | Readed64 17 | io.Seeker 18 | Range() transfer.Range 19 | Left() int64 20 | } 21 | 22 | fileBlock struct { 23 | readRange transfer.Range 24 | readed int64 25 | readerAt io.ReaderAt 26 | speedsStatRef *speeds.Speeds 27 | rateLimit *speeds.RateLimit 28 | mu sync.Mutex 29 | } 30 | 31 | bufioFileBlock struct { 32 | *fileBlock 33 | bufio *bufio.Reader 34 | } 35 | ) 36 | 37 | // SplitBlock 文件分块 38 | func SplitBlock(fileSize, blockSize int64) (blockList []*BlockState) { 39 | gen := transfer.NewRangeListGenBlockSize(fileSize, 0, blockSize) 40 | rangeCount := gen.RangeCount() 41 | blockList = make([]*BlockState, 0, rangeCount) 42 | for i := 0; i < rangeCount; i++ { 43 | id, r := gen.GenRange() 44 | blockList = append(blockList, &BlockState{ 45 | ID: id, 46 | Range: *r, 47 | }) 48 | } 49 | return 50 | } 51 | 52 | // NewBufioSplitUnit io.ReaderAt实现SplitUnit接口, 有Buffer支持 53 | func NewBufioSplitUnit(readerAt io.ReaderAt, readRange transfer.Range, speedsStat *speeds.Speeds, rateLimit *speeds.RateLimit) SplitUnit { 54 | su := &fileBlock{ 55 | readerAt: readerAt, 56 | readRange: readRange, 57 | speedsStatRef: speedsStat, 58 | rateLimit: rateLimit, 59 | } 60 | return &bufioFileBlock{ 61 | fileBlock: su, 62 | bufio: bufio.NewReaderSize(su, BufioReadSize), 63 | } 64 | } 65 | 66 | func (bfb *bufioFileBlock) Read(b []byte) (n int, err error) { 67 | return bfb.bufio.Read(b) // 间接调用fileBlock 的Read 68 | } 69 | 70 | // Read 只允许一个线程读同一个文件 71 | func (fb *fileBlock) Read(b []byte) (n int, err error) { 72 | fb.mu.Lock() 73 | defer fb.mu.Unlock() 74 | 75 | left := int(fb.Left()) 76 | if left <= 0 { 77 | return 0, io.EOF 78 | } 79 | 80 | if len(b) > left { 81 | n, err = fb.readerAt.ReadAt(b[:left], fb.readed+fb.readRange.Begin) 82 | } else { 83 | n, err = fb.readerAt.ReadAt(b, fb.readed+fb.readRange.Begin) 84 | } 85 | 86 | n64 := int64(n) 87 | fb.readed += n64 88 | if fb.rateLimit != nil { 89 | fb.rateLimit.Add(n64) // 限速阻塞 90 | } 91 | if fb.speedsStatRef != nil { 92 | fb.speedsStatRef.Add(n64) 93 | } 94 | return 95 | } 96 | 97 | func (fb *fileBlock) Seek(offset int64, whence int) (int64, error) { 98 | fb.mu.Lock() 99 | defer fb.mu.Unlock() 100 | 101 | switch whence { 102 | case os.SEEK_SET: 103 | fb.readed = offset 104 | case os.SEEK_CUR: 105 | fb.readed += offset 106 | case os.SEEK_END: 107 | fb.readed = fb.readRange.End - fb.readRange.Begin + offset 108 | default: 109 | return 0, fmt.Errorf("unsupport whence: %d", whence) 110 | } 111 | if fb.readed < 0 { 112 | fb.readed = 0 113 | } 114 | return fb.readed, nil 115 | } 116 | 117 | func (fb *fileBlock) Len() int64 { 118 | return fb.readRange.End - fb.readRange.Begin 119 | } 120 | 121 | func (fb *fileBlock) Left() int64 { 122 | return fb.readRange.End - fb.readRange.Begin - fb.readed 123 | } 124 | 125 | func (fb *fileBlock) Range() transfer.Range { 126 | return fb.readRange 127 | } 128 | 129 | func (fb *fileBlock) Readed() int64 { 130 | return fb.readed 131 | } 132 | -------------------------------------------------------------------------------- /requester/transfer/download_status.go: -------------------------------------------------------------------------------- 1 | package transfer 2 | 3 | import ( 4 | "github.com/iikira/iikira-go-utils/requester/rio/speeds" 5 | "sync" 6 | "sync/atomic" 7 | "time" 8 | ) 9 | 10 | type ( 11 | //DownloadStatuser 下载状态接口 12 | DownloadStatuser interface { 13 | TotalSize() int64 14 | Downloaded() int64 15 | SpeedsPerSecond() int64 16 | TimeElapsed() time.Duration // 已开始时间 17 | TimeLeft() time.Duration // 预计剩余时间, 负数代表未知 18 | } 19 | 20 | //DownloadStatus 下载状态及统计信息 21 | DownloadStatus struct { 22 | totalSize int64 // 总大小 23 | downloaded int64 // 已下载的数据量 24 | speedsDownloaded int64 // 用于统计速度的downloaded 25 | maxSpeeds int64 // 最大下载速度 26 | tmpSpeeds int64 // 缓存的速度 27 | speedsStat speeds.Speeds // 速度统计 (注意对齐) 28 | 29 | startTime time.Time // 开始下载的时间 30 | 31 | rateLimit *speeds.RateLimit // 限速控制 32 | 33 | gen *RangeListGen // Range生成状态 34 | mu sync.Mutex 35 | } 36 | ) 37 | 38 | //NewDownloadStatus 初始化DownloadStatus 39 | func NewDownloadStatus() *DownloadStatus { 40 | return &DownloadStatus{ 41 | startTime: time.Now(), 42 | } 43 | } 44 | 45 | // SetRateLimit 设置限速 46 | func (ds *DownloadStatus) SetRateLimit(rl *speeds.RateLimit) { 47 | ds.rateLimit = rl 48 | } 49 | 50 | //SetTotalSize 返回总大小 51 | func (ds *DownloadStatus) SetTotalSize(size int64) { 52 | ds.totalSize = size 53 | } 54 | 55 | //AddDownloaded 增加已下载数据量 56 | func (ds *DownloadStatus) AddDownloaded(d int64) { 57 | atomic.AddInt64(&ds.downloaded, d) 58 | } 59 | 60 | //AddTotalSize 增加总大小 (不支持多线程) 61 | func (ds *DownloadStatus) AddTotalSize(size int64) { 62 | ds.totalSize += size 63 | } 64 | 65 | //AddSpeedsDownloaded 增加已下载数据量, 用于统计速度 66 | func (ds *DownloadStatus) AddSpeedsDownloaded(d int64) { 67 | if ds.rateLimit != nil { 68 | ds.rateLimit.Add(d) 69 | } 70 | ds.speedsStat.Add(d) 71 | } 72 | 73 | //SetMaxSpeeds 设置最大速度, 原子操作 74 | func (ds *DownloadStatus) SetMaxSpeeds(speeds int64) { 75 | if speeds > atomic.LoadInt64(&ds.maxSpeeds) { 76 | atomic.StoreInt64(&ds.maxSpeeds, speeds) 77 | } 78 | } 79 | 80 | //ClearMaxSpeeds 清空统计最大速度, 原子操作 81 | func (ds *DownloadStatus) ClearMaxSpeeds() { 82 | atomic.StoreInt64(&ds.maxSpeeds, 0) 83 | } 84 | 85 | //TotalSize 返回总大小 86 | func (ds *DownloadStatus) TotalSize() int64 { 87 | return ds.totalSize 88 | } 89 | 90 | //Downloaded 返回已下载数据量 91 | func (ds *DownloadStatus) Downloaded() int64 { 92 | return atomic.LoadInt64(&ds.downloaded) 93 | } 94 | 95 | // UpdateSpeeds 更新speeds 96 | func (ds *DownloadStatus) UpdateSpeeds() { 97 | atomic.StoreInt64(&ds.tmpSpeeds, ds.speedsStat.GetSpeeds()) 98 | } 99 | 100 | //SpeedsPerSecond 返回每秒速度 101 | func (ds *DownloadStatus) SpeedsPerSecond() int64 { 102 | return atomic.LoadInt64(&ds.tmpSpeeds) 103 | } 104 | 105 | //MaxSpeeds 返回最大速度 106 | func (ds *DownloadStatus) MaxSpeeds() int64 { 107 | return atomic.LoadInt64(&ds.maxSpeeds) 108 | } 109 | 110 | //TimeElapsed 返回花费的时间 111 | func (ds *DownloadStatus) TimeElapsed() (elapsed time.Duration) { 112 | return time.Since(ds.startTime) 113 | } 114 | 115 | //TimeLeft 返回预计剩余时间 116 | func (ds *DownloadStatus) TimeLeft() (left time.Duration) { 117 | speeds := atomic.LoadInt64(&ds.tmpSpeeds) 118 | if speeds <= 0 { 119 | left = -1 120 | } else { 121 | left = time.Duration((ds.totalSize-ds.downloaded)/(speeds)) * time.Second 122 | } 123 | return 124 | } 125 | 126 | // RangeListGen 返回RangeListGen 127 | func (ds *DownloadStatus) RangeListGen() *RangeListGen { 128 | return ds.gen 129 | } 130 | 131 | // SetRangeListGen 设置RangeListGen 132 | func (ds *DownloadStatus) SetRangeListGen(gen *RangeListGen) { 133 | ds.gen = gen 134 | } 135 | -------------------------------------------------------------------------------- /utils/taskframework/executor.go: -------------------------------------------------------------------------------- 1 | package taskframework 2 | 3 | import ( 4 | "github.com/GeertJohan/go.incremental" 5 | "github.com/iikira/iikira-go-utils/utils/waitgroup" 6 | "github.com/oleiade/lane" 7 | "strconv" 8 | "time" 9 | ) 10 | 11 | type ( 12 | TaskExecutor struct { 13 | incr *incremental.Int // 任务id生成 14 | deque *lane.Deque // 队列 15 | parallel int // 任务的最大并发量 16 | 17 | // 是否统计失败队列 18 | IsFailedDeque bool 19 | failedDeque *lane.Deque 20 | } 21 | ) 22 | 23 | func NewTaskExecutor() *TaskExecutor { 24 | return &TaskExecutor{} 25 | } 26 | 27 | func (te *TaskExecutor) lazyInit() { 28 | if te.deque == nil { 29 | te.deque = lane.NewDeque() 30 | } 31 | if te.incr == nil { 32 | te.incr = &incremental.Int{} 33 | } 34 | if te.parallel < 1 { 35 | te.parallel = 1 36 | } 37 | if te.IsFailedDeque { 38 | te.failedDeque = lane.NewDeque() 39 | } 40 | } 41 | 42 | // 设置任务的最大并发量 43 | func (te *TaskExecutor) SetParallel(parallel int) { 44 | te.parallel = parallel 45 | } 46 | 47 | //Append 将任务加到任务队列末尾 48 | func (te *TaskExecutor) Append(unit TaskUnit, maxRetry int) *TaskInfo { 49 | te.lazyInit() 50 | taskInfo := &TaskInfo{ 51 | id: strconv.Itoa(te.incr.Next()), 52 | maxRetry: maxRetry, 53 | } 54 | unit.SetTaskInfo(taskInfo) 55 | te.deque.Append(&TaskInfoItem{ 56 | Info: taskInfo, 57 | Unit: unit, 58 | }) 59 | return taskInfo 60 | } 61 | 62 | //AppendNoRetry 将任务加到任务队列末尾, 不重试 63 | func (te *TaskExecutor) AppendNoRetry(unit TaskUnit) { 64 | te.Append(unit, 0) 65 | } 66 | 67 | //Count 返回任务数量 68 | func (te *TaskExecutor) Count() int { 69 | if te.deque == nil { 70 | return 0 71 | } 72 | return te.deque.Size() 73 | } 74 | 75 | //Execute 执行任务 76 | func (te *TaskExecutor) Execute() { 77 | te.lazyInit() 78 | 79 | for { 80 | wg := waitgroup.NewWaitGroup(te.parallel) 81 | for { 82 | e := te.deque.Shift() 83 | if e == nil { // 任务为空 84 | break 85 | } 86 | 87 | // 获取任务 88 | task := e.(*TaskInfoItem) 89 | wg.AddDelta() 90 | 91 | go func(task *TaskInfoItem) { 92 | defer wg.Done() 93 | 94 | result := task.Unit.Run() 95 | 96 | // 返回结果为空 97 | if result == nil { 98 | task.Unit.OnComplete(result) 99 | return 100 | } 101 | 102 | if result.Succeed { 103 | task.Unit.OnSuccess(result) 104 | task.Unit.OnComplete(result) 105 | return 106 | } 107 | 108 | // 需要进行重试 109 | if result.NeedRetry { 110 | // 重试次数超出限制 111 | // 执行失败 112 | if task.Info.IsExceedRetry() { 113 | task.Unit.OnFailed(result) 114 | if te.IsFailedDeque { 115 | // 加入失败队列 116 | te.failedDeque.Append(task) 117 | } 118 | task.Unit.OnComplete(result) 119 | return 120 | } 121 | 122 | task.Info.retry++ // 增加重试次数 123 | task.Unit.OnRetry(result) // 调用重试 124 | task.Unit.OnComplete(result) 125 | 126 | time.Sleep(task.Unit.RetryWait()) // 等待 127 | te.deque.Append(task) // 重新加入队列末尾 128 | return 129 | } 130 | 131 | // 执行失败 132 | task.Unit.OnFailed(result) 133 | if te.IsFailedDeque { 134 | // 加入失败队列 135 | te.failedDeque.Append(task) 136 | } 137 | task.Unit.OnComplete(result) 138 | }(task) 139 | } 140 | 141 | wg.Wait() 142 | 143 | // 没有任务了 144 | if te.deque.Size() == 0 { 145 | break 146 | } 147 | } 148 | } 149 | 150 | //FailedDeque 获取失败队列 151 | func (te *TaskExecutor) FailedDeque() *lane.Deque { 152 | return te.failedDeque 153 | } 154 | 155 | //Stop 停止执行 156 | func (te *TaskExecutor) Stop() { 157 | 158 | } 159 | 160 | //Pause 暂停执行 161 | func (te *TaskExecutor) Pause() { 162 | 163 | } 164 | 165 | //Resume 恢复执行 166 | func (te *TaskExecutor) Resume() { 167 | } 168 | -------------------------------------------------------------------------------- /requester/fetch.go: -------------------------------------------------------------------------------- 1 | package requester 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/iikira/iikira-go-utils/requester/rio" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "net/url" 11 | "strings" 12 | ) 13 | 14 | // HTTPGet 简单实现 http 访问 GET 请求 15 | func HTTPGet(urlStr string) (body []byte, err error) { 16 | resp, err := DefaultClient.Get(urlStr) 17 | if resp != nil { 18 | defer resp.Body.Close() 19 | } 20 | if err != nil { 21 | return nil, err 22 | } 23 | return ioutil.ReadAll(resp.Body) 24 | } 25 | 26 | // Req 参见 *HTTPClient.Req, 使用默认 http 客户端 27 | func Req(method string, urlStr string, post interface{}, header map[string]string) (resp *http.Response, err error) { 28 | return DefaultClient.Req(method, urlStr, post, header) 29 | } 30 | 31 | // Fetch 参见 *HTTPClient.Fetch, 使用默认 http 客户端 32 | func Fetch(method string, urlStr string, post interface{}, header map[string]string) (body []byte, err error) { 33 | return DefaultClient.Fetch(method, urlStr, post, header) 34 | } 35 | 36 | // Req 实现 http／https 访问， 37 | // 根据给定的 method (GET, POST, HEAD, PUT 等等), urlStr (网址), 38 | // post (post 数据), header (header 请求头数据), 进行网站访问。 39 | // 返回值分别为 *http.Response, 错误信息 40 | func (h *HTTPClient) Req(method string, urlStr string, post interface{}, header map[string]string) (resp *http.Response, err error) { 41 | h.lazyInit() 42 | var ( 43 | req *http.Request 44 | obody io.Reader 45 | contentLength int64 46 | contentType string 47 | ) 48 | 49 | if post != nil { 50 | switch value := post.(type) { 51 | case io.Reader: 52 | obody = value 53 | case map[string]string: 54 | query := url.Values{} 55 | for k := range value { 56 | query.Set(k, value[k]) 57 | } 58 | obody = strings.NewReader(query.Encode()) 59 | contentType = "application/x-www-form-urlencoded" 60 | case map[string]interface{}: 61 | query := url.Values{} 62 | for k := range value { 63 | query.Set(k, fmt.Sprint(value[k])) 64 | } 65 | obody = strings.NewReader(query.Encode()) 66 | contentType = "application/x-www-form-urlencoded" 67 | case map[interface{}]interface{}: 68 | query := url.Values{} 69 | for k := range value { 70 | query.Set(fmt.Sprint(k), fmt.Sprint(value[k])) 71 | } 72 | obody = strings.NewReader(query.Encode()) 73 | contentType = "application/x-www-form-urlencoded" 74 | case string: 75 | obody = strings.NewReader(value) 76 | case []byte: 77 | obody = bytes.NewReader(value[:]) 78 | default: 79 | return nil, fmt.Errorf("requester.Req: unknown post type: %s", value) 80 | } 81 | 82 | switch value := post.(type) { 83 | case ContentLengther: 84 | contentLength = value.ContentLength() 85 | case rio.Lener: 86 | contentLength = int64(value.Len()) 87 | case rio.Lener64: 88 | contentLength = value.Len() 89 | } 90 | 91 | switch value := post.(type) { 92 | case ContentTyper: 93 | contentType = value.ContentType() 94 | } 95 | } 96 | req, err = http.NewRequest(method, urlStr, obody) 97 | if err != nil { 98 | return nil, err 99 | } 100 | 101 | if req.ContentLength <= 0 && contentLength != 0 { 102 | req.ContentLength = contentLength 103 | } 104 | 105 | // 设置浏览器标识 106 | req.Header.Set("User-Agent", h.UserAgent) 107 | 108 | // 设置Content-Type 109 | if contentType != "" { 110 | req.Header.Set("Content-Type", contentType) 111 | } 112 | 113 | if header != nil { 114 | // 处理Host 115 | if host, ok := header["Host"]; ok { 116 | req.Host = host 117 | } 118 | 119 | for key := range header { 120 | req.Header.Set(key, header[key]) 121 | } 122 | } 123 | 124 | return h.Client.Do(req) 125 | } 126 | 127 | // Fetch 实现 http／https 访问， 128 | // 根据给定的 method (GET, POST, HEAD, PUT 等等), urlStr (网址), 129 | // post (post 数据), header (header 请求头数据), 进行网站访问。 130 | // 返回值分别为 网站主体, 错误信息 131 | func (h *HTTPClient) Fetch(method string, urlStr string, post interface{}, header map[string]string) (body []byte, err error) { 132 | h.lazyInit() 133 | resp, err := h.Req(method, urlStr, post, header) 134 | if resp != nil { 135 | defer resp.Body.Close() 136 | } 137 | if err != nil { 138 | return nil, err 139 | } 140 | 141 | return ioutil.ReadAll(resp.Body) 142 | } 143 | -------------------------------------------------------------------------------- /requester/multipartreader/multipartreader.go: -------------------------------------------------------------------------------- 1 | // Package multipartreader helps you encode large files in MIME multipart format 2 | // without reading the entire content into memory. 3 | package multipartreader 4 | 5 | import ( 6 | "bytes" 7 | "errors" 8 | "fmt" 9 | "github.com/iikira/iikira-go-utils/requester/rio" 10 | "io" 11 | "mime/multipart" 12 | "strings" 13 | "sync" 14 | "sync/atomic" 15 | ) 16 | 17 | type ( 18 | // MultipartReader MIME multipart format 19 | MultipartReader struct { 20 | length int64 21 | contentType string 22 | boundary string 23 | 24 | formBody string 25 | parts []*part 26 | part64s []*part64 27 | formClose string 28 | 29 | mu sync.Mutex 30 | closed bool 31 | multiReader io.Reader 32 | } 33 | 34 | part struct { 35 | form string 36 | readerlen rio.ReaderLen 37 | } 38 | 39 | part64 struct { 40 | form string 41 | readerlen64 rio.ReaderLen64 42 | } 43 | ) 44 | 45 | // NewMultipartReader 返回初始化的 *MultipartReader 46 | func NewMultipartReader() (mr *MultipartReader) { 47 | builder := &strings.Builder{} 48 | writer := multipart.NewWriter(builder) 49 | mr = &MultipartReader{ 50 | contentType: writer.FormDataContentType(), 51 | boundary: writer.Boundary(), 52 | } 53 | 54 | mr.length += int64(builder.Len()) 55 | mr.formBody = builder.String() 56 | return 57 | } 58 | 59 | // AddFormField 增加 form 表单 60 | func (mr *MultipartReader) AddFormField(fieldname string, readerlen rio.ReaderLen) { 61 | if readerlen == nil { 62 | return 63 | } 64 | 65 | mpart := &part{ 66 | form: fmt.Sprintf("--%s\r\nContent-Disposition: form-data; name=\"%s\"\r\n\r\n", mr.boundary, fieldname), 67 | readerlen: readerlen, 68 | } 69 | atomic.AddInt64(&mr.length, int64(len(mpart.form)+mpart.readerlen.Len())) 70 | mr.parts = append(mr.parts, mpart) 71 | } 72 | 73 | func (mr *MultipartReader) AddFormFieldBytes(fieldname string, b []byte) { 74 | mr.AddFormField(fieldname, bytes.NewReader(b)) 75 | } 76 | 77 | func (mr *MultipartReader) AddFormFieldString(fieldname string, s string) { 78 | mr.AddFormField(fieldname, strings.NewReader(s)) 79 | } 80 | 81 | // AddFormFile 增加 form 文件表单 82 | func (mr *MultipartReader) AddFormFile(fieldname, filename string, readerlen64 rio.ReaderLen64) { 83 | if readerlen64 == nil { 84 | return 85 | } 86 | 87 | mpart64 := &part64{ 88 | form: fmt.Sprintf("--%s\r\nContent-Disposition: form-data; name=\"%s\"; filename=\"%s\"\r\n\r\n", mr.boundary, fieldname, filename), 89 | readerlen64: readerlen64, 90 | } 91 | atomic.AddInt64(&mr.length, int64(len(mpart64.form))+mpart64.readerlen64.Len()) 92 | mr.part64s = append(mr.part64s, mpart64) 93 | } 94 | 95 | //CloseMultipart 关闭multipartreader 96 | func (mr *MultipartReader) CloseMultipart() error { 97 | mr.mu.Lock() 98 | defer mr.mu.Unlock() 99 | if mr.closed { 100 | return errors.New("multipartreader already closed") 101 | } 102 | 103 | mr.formClose = "\r\n--" + mr.boundary + "--\r\n" 104 | atomic.AddInt64(&mr.length, int64(len(mr.formClose))) 105 | 106 | numReaders := 0 107 | if mr.formBody != "" { 108 | numReaders++ 109 | } 110 | numReaders += 2*len(mr.parts) + 2*len(mr.part64s) 111 | if mr.formClose != "" { 112 | numReaders++ 113 | } 114 | 115 | readers := make([]io.Reader, 0, numReaders) 116 | readers = append(readers, strings.NewReader(mr.formBody)) 117 | for k := range mr.parts { 118 | readers = append(readers, strings.NewReader(mr.parts[k].form), mr.parts[k].readerlen) 119 | } 120 | for k := range mr.part64s { 121 | readers = append(readers, strings.NewReader(mr.part64s[k].form), mr.part64s[k].readerlen64) 122 | } 123 | readers = append(readers, strings.NewReader(mr.formClose)) 124 | mr.multiReader = io.MultiReader(readers...) 125 | 126 | mr.closed = true 127 | return nil 128 | } 129 | 130 | //ContentType 返回Content-Type 131 | func (mr *MultipartReader) ContentType() string { 132 | return mr.contentType 133 | } 134 | 135 | func (mr *MultipartReader) Read(p []byte) (n int, err error) { 136 | if !mr.closed { 137 | return 0, errors.New("multipartreader not closed") 138 | } 139 | n, err = mr.multiReader.Read(p) 140 | return n, err 141 | } 142 | 143 | // Len 返回表单内容总长度 144 | func (mr *MultipartReader) Len() int64 { 145 | return atomic.LoadInt64(&mr.length) 146 | } 147 | -------------------------------------------------------------------------------- /requester/downloader/instance_state.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "errors" 5 | "github.com/golang/protobuf/proto" 6 | "github.com/iikira/iikira-go-utils/pcsverbose" 7 | "github.com/iikira/iikira-go-utils/requester/transfer" 8 | "github.com/iikira/iikira-go-utils/utils/cachepool" 9 | "github.com/json-iterator/go" 10 | "os" 11 | "sync" 12 | ) 13 | 14 | type ( 15 | //InstanceState 状态, 断点续传信息 16 | InstanceState struct { 17 | saveFile *os.File 18 | format InstanceStateStorageFormat 19 | ii transfer.DownloadInstanceInfoExporter 20 | mu sync.Mutex 21 | } 22 | 23 | // InstanceStateStorageFormat 断点续传储存类型 24 | InstanceStateStorageFormat int 25 | ) 26 | 27 | const ( 28 | // InstanceStateStorageFormatJSON json 格式 29 | InstanceStateStorageFormatJSON = iota 30 | // InstanceStateStorageFormatProto3 protobuf 格式 31 | InstanceStateStorageFormatProto3 32 | ) 33 | 34 | //NewInstanceState 初始化InstanceState 35 | func NewInstanceState(saveFile *os.File, format InstanceStateStorageFormat) *InstanceState { 36 | return &InstanceState{ 37 | saveFile: saveFile, 38 | format: format, 39 | } 40 | } 41 | 42 | func (is *InstanceState) checkSaveFile() bool { 43 | return is.saveFile != nil 44 | } 45 | 46 | func (is *InstanceState) getSaveFileContents() []byte { 47 | if !is.checkSaveFile() { 48 | return nil 49 | } 50 | 51 | finfo, err := is.saveFile.Stat() 52 | if err != nil { 53 | panic(err) 54 | } 55 | 56 | size := finfo.Size() 57 | if size > 0xffffffff { 58 | panic("savePath too large") 59 | } 60 | intSize := int(size) 61 | 62 | buf := cachepool.RawMallocByteSlice(intSize) 63 | 64 | n, _ := is.saveFile.ReadAt(buf, 0) 65 | return buf[:n] 66 | } 67 | 68 | //Get 获取断点续传信息 69 | func (is *InstanceState) Get() (eii *transfer.DownloadInstanceInfo) { 70 | if !is.checkSaveFile() { 71 | return nil 72 | } 73 | 74 | is.mu.Lock() 75 | defer is.mu.Unlock() 76 | 77 | contents := is.getSaveFileContents() 78 | if len(contents) <= 0 { 79 | return 80 | } 81 | 82 | is.ii = &transfer.DownloadInstanceInfoExport{} 83 | var err error 84 | switch is.format { 85 | case InstanceStateStorageFormatProto3: 86 | err = proto.Unmarshal(contents, is.ii.(*transfer.DownloadInstanceInfoExport)) 87 | default: 88 | err = jsoniter.Unmarshal(contents, is.ii) 89 | } 90 | 91 | if err != nil { 92 | pcsverbose.Verbosef("DEBUG: InstanceInfo unmarshal error: %s\n", err) 93 | return 94 | } 95 | 96 | eii = is.ii.GetInstanceInfo() 97 | return 98 | } 99 | 100 | //Put 提交断点续传信息 101 | func (is *InstanceState) Put(eii *transfer.DownloadInstanceInfo) { 102 | if !is.checkSaveFile() { 103 | return 104 | } 105 | 106 | is.mu.Lock() 107 | defer is.mu.Unlock() 108 | 109 | if is.ii == nil { 110 | is.ii = &transfer.DownloadInstanceInfoExport{} 111 | } 112 | is.ii.SetInstanceInfo(eii) 113 | var ( 114 | data []byte 115 | err error 116 | ) 117 | switch is.format { 118 | case InstanceStateStorageFormatProto3: 119 | data, err = proto.Marshal(is.ii.(*transfer.DownloadInstanceInfoExport)) 120 | default: 121 | data, err = jsoniter.Marshal(is.ii) 122 | } 123 | if err != nil { 124 | panic(err) 125 | } 126 | 127 | err = is.saveFile.Truncate(int64(len(data))) 128 | if err != nil { 129 | pcsverbose.Verbosef("DEBUG: truncate file error: %s\n", err) 130 | } 131 | 132 | _, err = is.saveFile.WriteAt(data, 0) 133 | if err != nil { 134 | pcsverbose.Verbosef("DEBUG: write instance state error: %s\n", err) 135 | } 136 | } 137 | 138 | //Close 关闭 139 | func (is *InstanceState) Close() error { 140 | if !is.checkSaveFile() { 141 | return nil 142 | } 143 | 144 | return is.saveFile.Close() 145 | } 146 | 147 | func (der *Downloader) initInstanceState(format InstanceStateStorageFormat) (err error) { 148 | if der.instanceState != nil { 149 | return errors.New("already initInstanceState") 150 | } 151 | 152 | var saveFile *os.File 153 | if !der.config.IsTest && der.config.InstanceStatePath != "" { 154 | saveFile, err = os.OpenFile(der.config.InstanceStatePath, os.O_RDWR|os.O_CREATE, 0777) 155 | if err != nil { 156 | return err 157 | } 158 | } 159 | 160 | der.instanceState = NewInstanceState(saveFile, format) 161 | return nil 162 | } 163 | 164 | func (der *Downloader) removeInstanceState() error { 165 | der.instanceState.Close() 166 | if !der.config.IsTest && der.config.InstanceStatePath != "" { 167 | return os.Remove(der.config.InstanceStatePath) 168 | } 169 | return nil 170 | } 171 | -------------------------------------------------------------------------------- /requester/dial.go: -------------------------------------------------------------------------------- 1 | package requester 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "errors" 7 | "github.com/iikira/iikira-go-utils/utils/expires" 8 | "github.com/iikira/iikira-go-utils/utils/expires/cachemap" 9 | mathrand "math/rand" 10 | "net" 11 | "net/http" 12 | "net/url" 13 | "strconv" 14 | "time" 15 | ) 16 | 17 | const ( 18 | // MaxDuration 最大的Duration 19 | MaxDuration = 1<<63 - 1 20 | ) 21 | 22 | var ( 23 | localTCPAddrList = []*net.TCPAddr{} 24 | 25 | // ProxyAddr 代理地址 26 | ProxyAddr string 27 | 28 | // ErrProxyAddrEmpty 代理地址为空 29 | ErrProxyAddrEmpty = errors.New("proxy addr is empty") 30 | 31 | tcpCache = cachemap.GlobalCacheOpMap.LazyInitCachePoolOp("requester/tcp") 32 | ) 33 | 34 | // SetLocalTCPAddrList 设置网卡地址 35 | func SetLocalTCPAddrList(ips ...string) { 36 | list := make([]*net.TCPAddr, 0, len(ips)) 37 | for k := range ips { 38 | p := net.ParseIP(ips[k]) 39 | if p == nil { 40 | continue 41 | } 42 | 43 | list = append(list, &net.TCPAddr{ 44 | IP: p, 45 | }) 46 | } 47 | localTCPAddrList = list 48 | } 49 | 50 | func proxyFunc(req *http.Request) (*url.URL, error) { 51 | u, err := checkProxyAddr(ProxyAddr) 52 | if err != nil { 53 | return http.ProxyFromEnvironment(req) 54 | } 55 | 56 | return u, err 57 | } 58 | 59 | func getLocalTCPAddr() *net.TCPAddr { 60 | if len(localTCPAddrList) == 0 { 61 | return nil 62 | } 63 | i := mathrand.Intn(len(localTCPAddrList)) 64 | return localTCPAddrList[i] 65 | } 66 | 67 | func getDialer() *net.Dialer { 68 | return &net.Dialer{ 69 | Timeout: 30 * time.Second, 70 | KeepAlive: 30 * time.Second, 71 | LocalAddr: getLocalTCPAddr(), 72 | DualStack: true, 73 | } 74 | } 75 | 76 | func checkProxyAddr(proxyAddr string) (u *url.URL, err error) { 77 | if proxyAddr == "" { 78 | return nil, ErrProxyAddrEmpty 79 | } 80 | 81 | host, port, err := net.SplitHostPort(proxyAddr) 82 | if err == nil { 83 | u = &url.URL{ 84 | Host: net.JoinHostPort(host, port), 85 | } 86 | return 87 | } 88 | 89 | u, err = url.Parse(proxyAddr) 90 | if err == nil { 91 | return 92 | } 93 | 94 | return 95 | } 96 | 97 | // SetGlobalProxy 设置代理 98 | func SetGlobalProxy(proxyAddr string) { 99 | ProxyAddr = proxyAddr 100 | } 101 | 102 | // SetTCPHostBind 设置host绑定ip 103 | func SetTCPHostBind(host, ip string) { 104 | tcpCache.Store(host, expires.NewDataExpires(net.ParseIP(ip), MaxDuration)) 105 | return 106 | } 107 | 108 | func getServerName(address string) string { 109 | host, _, err := net.SplitHostPort(address) 110 | if err != nil { 111 | return address 112 | } 113 | return host 114 | } 115 | 116 | // resolveTCPHost 117 | // 解析的tcpaddr没有port!!! 118 | func resolveTCPHost(ctx context.Context, host string) (ip net.IP, err error) { 119 | addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host) 120 | if err != nil { 121 | return 122 | } 123 | 124 | return addrs[0].IP, nil 125 | } 126 | 127 | func dialContext(ctx context.Context, network, address string) (conn net.Conn, err error) { 128 | switch network { 129 | case "tcp", "tcp4", "tcp6": 130 | host, portStr, err := net.SplitHostPort(address) 131 | if err != nil { 132 | return nil, err 133 | } 134 | data, err := cachemap.GlobalCacheOpMap.CacheOperationWithError("requester/tcp", host, func() (expires.DataExpires, error) { 135 | ip, err := resolveTCPHost(ctx, host) 136 | if err != nil { 137 | return nil, err 138 | } 139 | return expires.NewDataExpires(ip, 10*time.Minute), nil // 传值 140 | }) 141 | if err != nil { 142 | return nil, err 143 | } 144 | 145 | port, err := strconv.Atoi(portStr) 146 | if err != nil { 147 | return nil, err 148 | } 149 | 150 | return net.DialTCP(network, getLocalTCPAddr(), &net.TCPAddr{ 151 | IP: data.Data().(net.IP), 152 | Port: port, // 设置端口 153 | }) 154 | } 155 | 156 | // 非 tcp 请求 157 | conn, err = getDialer().DialContext(ctx, network, address) 158 | return 159 | } 160 | 161 | func dial(network, address string) (conn net.Conn, err error) { 162 | return dialContext(context.Background(), network, address) 163 | } 164 | 165 | func (h *HTTPClient) dialTLSFunc() func(network, address string) (tlsConn net.Conn, err error) { 166 | return func(network, address string) (tlsConn net.Conn, err error) { 167 | conn, err := dialContext(context.Background(), network, address) 168 | if err != nil { 169 | return nil, err 170 | } 171 | 172 | return tls.Client(conn, &tls.Config{ 173 | ServerName: getServerName(address), 174 | InsecureSkipVerify: !h.https, 175 | }), nil 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /requester/transfer/rangelist.go: -------------------------------------------------------------------------------- 1 | package transfer 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/iikira/iikira-go-utils/utils/converter" 7 | "sync" 8 | "sync/atomic" 9 | ) 10 | 11 | type ( 12 | //RangeList 请求范围列表 13 | RangeList []*Range 14 | 15 | //RangeListGen Range 生成器 16 | RangeListGen struct { 17 | total int64 18 | begin int64 19 | blockSize int64 20 | parallel int 21 | count int // 已生成次数 22 | rangeGenMode RangeGenMode 23 | mu sync.Mutex 24 | } 25 | ) 26 | 27 | const ( 28 | // DefaultBlockSize 默认的BlockSize 29 | DefaultBlockSize = 256 * converter.KB 30 | ) 31 | 32 | var ( 33 | // ErrUnknownRangeGenMode RangeGenMode 非法 34 | ErrUnknownRangeGenMode = errors.New("Unknown RangeGenMode") 35 | ) 36 | 37 | //Len 长度 38 | func (r *Range) Len() int64 { 39 | return r.LoadEnd() - r.LoadBegin() 40 | } 41 | 42 | //LoadBegin 读取Begin, 原子操作 43 | func (r *Range) LoadBegin() int64 { 44 | return atomic.LoadInt64(&r.Begin) 45 | } 46 | 47 | //AddBegin 增加Begin, 原子操作 48 | func (r *Range) AddBegin(i int64) (newi int64) { 49 | return atomic.AddInt64(&r.Begin, i) 50 | } 51 | 52 | //LoadEnd 读取End, 原子操作 53 | func (r *Range) LoadEnd() int64 { 54 | return atomic.LoadInt64(&r.End) 55 | } 56 | 57 | //StoreBegin 储存End, 原子操作 58 | func (r *Range) StoreBegin(end int64) { 59 | atomic.StoreInt64(&r.Begin, end) 60 | } 61 | 62 | //StoreEnd 储存End, 原子操作 63 | func (r *Range) StoreEnd(end int64) { 64 | atomic.StoreInt64(&r.End, end) 65 | } 66 | 67 | // ShowDetails 显示Range细节 68 | func (r *Range) ShowDetails() string { 69 | return fmt.Sprintf("{%d-%d}", r.LoadBegin(), r.LoadEnd()) 70 | } 71 | 72 | //Len 获取所有的Range的剩余长度 73 | func (rl *RangeList) Len() int64 { 74 | var l int64 75 | for _, wrange := range *rl { 76 | if wrange == nil { 77 | continue 78 | } 79 | l += wrange.Len() 80 | } 81 | return l 82 | } 83 | 84 | // NewRangeListGenDefault 初始化默认Range生成器, 根据parallel平均生成 85 | func NewRangeListGenDefault(totalSize, begin int64, count, parallel int) *RangeListGen { 86 | return &RangeListGen{ 87 | total: totalSize, 88 | begin: begin, 89 | parallel: parallel, 90 | count: count, 91 | rangeGenMode: RangeGenMode_Default, 92 | } 93 | } 94 | 95 | // NewRangeListGenBlockSize 初始化Range生成器, 根据blockSize生成 96 | func NewRangeListGenBlockSize(totalSize, begin, blockSize int64) *RangeListGen { 97 | return &RangeListGen{ 98 | total: totalSize, 99 | begin: begin, 100 | blockSize: blockSize, 101 | rangeGenMode: RangeGenMode_BlockSize, 102 | } 103 | } 104 | 105 | // RangeGenMode 返回Range生成方式 106 | func (gen *RangeListGen) RangeGenMode() RangeGenMode { 107 | return gen.rangeGenMode 108 | } 109 | 110 | // RangeCount 返回预计生成的Range数量 111 | func (gen *RangeListGen) RangeCount() (rangeCount int) { 112 | switch gen.rangeGenMode { 113 | case RangeGenMode_Default: 114 | rangeCount = gen.parallel - gen.count 115 | case RangeGenMode_BlockSize: 116 | rangeCount = int((gen.total - gen.begin) / gen.blockSize) 117 | if gen.total%gen.blockSize != 0 { 118 | rangeCount++ 119 | } 120 | } 121 | return 122 | } 123 | 124 | // LoadBegin 返回begin 125 | func (gen *RangeListGen) LoadBegin() (begin int64) { 126 | gen.mu.Lock() 127 | begin = gen.begin 128 | gen.mu.Unlock() 129 | return 130 | } 131 | 132 | // LoadBlockSize 返回blockSize 133 | func (gen *RangeListGen) LoadBlockSize() (blockSize int64) { 134 | switch gen.rangeGenMode { 135 | case RangeGenMode_Default: 136 | if gen.blockSize <= 0 { 137 | gen.blockSize = (gen.total - gen.begin) / int64(gen.parallel) 138 | } 139 | blockSize = gen.blockSize 140 | case RangeGenMode_BlockSize: 141 | blockSize = gen.blockSize 142 | } 143 | return 144 | } 145 | 146 | // IsDone 是否已分配完成 147 | func (gen *RangeListGen) IsDone() bool { 148 | return gen.begin >= gen.total 149 | } 150 | 151 | // GenRange 生成 Range 152 | func (gen *RangeListGen) GenRange() (index int, r *Range) { 153 | var ( 154 | end int64 155 | ) 156 | if gen.parallel < 1 { 157 | gen.parallel = 1 158 | } 159 | switch gen.rangeGenMode { 160 | case RangeGenMode_Default: 161 | gen.LoadBlockSize() 162 | gen.mu.Lock() 163 | defer gen.mu.Unlock() 164 | 165 | if gen.IsDone() { 166 | return gen.count, nil 167 | } 168 | 169 | gen.count++ 170 | if gen.count >= gen.parallel { 171 | end = gen.total 172 | } else { 173 | end = gen.begin + gen.blockSize 174 | } 175 | r = &Range{ 176 | Begin: gen.begin, 177 | End: end, 178 | } 179 | 180 | gen.begin = end 181 | index = gen.count - 1 182 | return 183 | case RangeGenMode_BlockSize: 184 | if gen.blockSize <= 0 { 185 | gen.blockSize = DefaultBlockSize 186 | } 187 | gen.mu.Lock() 188 | defer gen.mu.Unlock() 189 | 190 | if gen.IsDone() { 191 | return gen.count, nil 192 | } 193 | 194 | gen.count++ 195 | end = gen.begin + gen.blockSize 196 | if end >= gen.total { 197 | end = gen.total 198 | } 199 | r = &Range{ 200 | Begin: gen.begin, 201 | End: end, 202 | } 203 | gen.begin = end 204 | index = gen.count - 1 205 | return 206 | } 207 | 208 | return 0, nil 209 | } 210 | -------------------------------------------------------------------------------- /utils/bdcrypto/des_test.go: -------------------------------------------------------------------------------- 1 | package bdcrypto_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/iikira/iikira-go-utils/utils/bdcrypto" 6 | "testing" 7 | ) 8 | 9 | func TestDESCBCDecrypt3(t *testing.T) { 10 | data := bdcrypto.Base64Decode([]byte( 11 | "4D2a4IsjSyD1GHh+w2IfMCTdO0uxHBFbzKVP4l81z+UZY2LzKqhovEcymeYeFW2lDWZSa56cS16lHjbN7woyyFNu+DMAeUaFA+4JWI/7zNXgqicIYpAcPtPZgdiLRpZzKB55q4+df9BSKC+dHge7dIK+Jv2fmV8vIWwvDL/1JYLB/LSB55IZ+BQQS9kPSCLju/snryQFPzHMlgHDNsTjqZWGDmOfMfXR/WriX+9Qkh1fu1zld28T7H+GgVSkyD1ldzDsTZU6vwxJk1Fyqzl2BtoB7OAFxbPWAGipKtTycN+ss/4sQpDLKPrnUJWcWfKb635v3wK+t1bfS7U+shWPKU5DoPLfXch/xEakF2Q7mXSGFxSs//ajvpS5YnovGCTd4iX2FsMOqB9sdCAx8l5CTGYWXuyDsrwxaJKeENav0lBLLqtd5X83VIv2YAs86pJUQEUPBI0wNlPNkEVFH+XFku/xIMavUvXC1hhwEwQy2apntyd+jiq0rwgDd+tJpcDTAzUa6SvgmMT97DN0FogPuqQ3Dar9HsvjNkv87RI+TCJSABWUVJvw+mzjfgaJL5cF", 12 | )) 13 | ciphertext, _ := bdcrypto.DESCBCDecrypt3(data, []byte("fHSgdfH326XJ@328%521DSAg"), []byte("01234567")) 14 | fmt.Printf("%s\n", ciphertext) 15 | 16 | data = bdcrypto.Base64Decode([]byte( 17 | "LuuPxoAsuEkjRyUD+W2/AiDc/f2cpNukHi+OmlhzuTkmN+41o3DHgw2e9J9LF9Y/8TuwikZcN1byG2oQq+KYaFOQrTqWzBSAWEAanJPHn1tH0PvhsiJkhawt9jLdqdzZo5AkzJcGj8/LmSDsmX8aSQRp0F1alnp1rMHI3Y1vbgTHa+2sppbasHeUyZlgJWynnvekG5jP2/34pkbQcRNpCcEAGADDi6oiFM9U0R0uTUQzN6w5eTp1pjyApRkebtlh9Tj69goD+2roZHY56xGUDETGGm/sOQDyXptIYNbl93rN3JpQWzDEtXzphQqGQN6q+b51HzkMkJ96G4q3ivfFliFmIGqJWSFKKsgactR5nM+MNWOQ4M4Pp6SqJwv782GDW8qjSJ0Qyj4+vyc92jbBtQr/u3OoHAOoVB+yIZrihOk0MsiMF6z2+djp9c5s75so3LNAtPmLj4Y9ekUQYq/qhZEuieqELg7cRayMZt2o2FnnBwrjOuImoRjx4nbcSZmHvD0WTgRG03nHVfdRIpXQFL7yKGWixTnAN/pl4Tw3te/K51uIFDJZauhmk62ezYhvqPTLtNcs95yQ9iM+PbiuAkqPiLnonOWu16c0gdDl3qYjyyTNBZrWl4U7y/3gZmA8588bD4Qp2qix78u9LKkFDFFltzav6FZXcZeCq5bc9oZXioh3pFcGX3PvAylkheGVVOhZawNEqmUHBboXkQJkEzTHi5JJpCaobJ7zWzP6jkD3yyHLk/ufRtS1TBumXq+hjZd3m4g3hFsqDXcxl8kzqQ22UyOMa3Yskz1un5X7R9nyZM2QFEsQLZ3kVlwDJfyiIOWxXvFxddyNjQatxx2vvGe+cE2OhYAWSmq4BanDZ21TQc0McfJ5qvPbET2QVU/zkzR5XwvaQe+gn/HLJ4U/Cb3mBwg2+2uwCRF3olQSuLyeTM6j9gcnouqMprc9bnhrRiVxvqBmDQPj40qorQZFti7AoDoJ0WfR3pnOvPL5rd/pMiEx2H0eMly9Bs/CWj219mM/WGHa83pSv3rvbQAd2yvVXSPg28kQF2aXosGFf8ze1rNRaKfKHrctt/3iu9wnNvD04z75kgj+yVsjRjqTdx2Q+F3e6TMHDrUnfEyKk8fSkk5L5XWs0w5iBy4xuwcgUu/gxpatviusgeGVLY6hpZPSVCXf8kOui/FvaKNVsPkKHJignO5uwGQin1d2t6pOOxRnfGPr1NQyRmH9qlA8Piq/pR5vQI2PRr32EOASllTg/DVCcyKm3YKLe+GyWDItp+LuHc8K5fh2bjUBRCc5B33rWG4eFO34v+wTqvFF5m6H0VBzQ317CKe1i4dABJzLlh7n3Yuc6fv4oGXtw4c1nnvQiN4GYAVql1li0w5qdiGyGnIyLblBkK31sQLdFxSJ+qHKHDlndriid4wQuOVEKDe58g5VBq7Niqyuf1kalYAV82HeJb4SIG3X91OVbquJcPNQvOMLoXe8sRbuafr8tvAhBUvAfxUZKD3UAGmFMxHQGq8cY19MfQkRhunTs68O4wo6evpw7DDKagxMAEvx9znQTXInNHeBc8UpeA+SbbS8CVa6Rdf2yWxhMbBsp/cEnOl7CqqVJJgga176TcMP6YZsbmYdARO6TsDQ3dhP9jMgYjtJg9vPgrGV+uY6jhPG15FPmbMrOmKfAJu8H+Y/imSNVDxjkku6MTQleqiN0T6nt+S/gk79a94p0EVEN7NdZJkd+Hd+J9Qd7algQG8Hoeiv/5MI4MX/5zZvjPCpOeeBbuoBZca009sMNMr0Mf0hFH4sKkrSLtAuhjmI2/aTHFddSa6WtmT+dXXXog6Ik4+hrW7qfz0P5aepvVRP5YEMTRaF0vCV222eA1ivwBUXANjnxeW791CjLMSy/tUxkCBRvfo3DSxPmVsMV5kk85YmI/qJm1uRFMPpfdUbzKSHewvuqD9dTULlhBSOOfOwDLglVhq+tETW9oUgRfvvlnzyCO9tMcHjDJQOi9gyhjbqNBHwfrHzf9Qz+QH4WZJYpA4iwJqLNuf6axz+NwefGUJWNkwDL+O6YRPr1MUcVO2HsYJ+GuqFDTkzsbKO0bfF2EwbPS0mZSWiR2cqBkrAiKjEImsYf9L1uKM2kSlr3djzIowJPHP1k6rEaMWUM/tzuosL2agYOqby/GlRQy++ZW+tIroI5o3P6UWcQVjKAYk0KqbD29Ud9USBT0Uf3lcpln1A9uLN7LIEMgENa1OhaN2AxttMs7dkuvmL9MttgxEKdeZWzLcdKQznkiUZqoQjvQK6T9RwmSSjaTQoAF37bHFu73T1eRguohRLTGczCst8npAucEGjTyvgTzjkw5nJ79ZVRms9D7OM4iiLWXtWyn6Jx+6Lb17+eD+WPj45MXa9sdK+Z11edXKBAT9Y0ENLw4WmpaT+TZcp+YY6RT4DWEYIARN2gS3jyezQYRRXbGPK0tcDbHfAmW9R82ula7xiAewV+umAgYl1IrPHmN+wiLasirL8jOzt+FEyFHpcoLZcgcOj9kNGzzjThmClXKjSIEWxf2Ol5s0wYgEn/Lk5odV+vG2ODxRw5ubugaXxmkoJdBhkLCiC68gnB/jRzXPib1o/XI36Njgk2h7/opPFg2J6lYCnKeniSmht6bOuhb3T9RMXrh9gyYYQEiX2DXMitRS0QoZyEzG7rG6td0F4LPSzPWWFoZovhf/hweJzv+lrLFkbBLqLUkJxFMJrxCJlMOPp8KQDviRBH1BqcE0VyUg36Or80+eByxi38JHOCq7DpsCMM1uRdXqezBDYN5/g58YrW+BpPbmr7769KXL19zk+qCOHnTPQJldE3Uy8AfI5MOHEYsqvonCH0Vl1fzG8qULRfjAJeF2XhTqvN88c8ZJJf9dynmUWBrRGVKDvHHChtNbHwELMSivtRQZKM3bq0LEBPUs8YmSC4+LgF3PlLIdme4DiEhBgUdzn53F9CPyuC50s+kv7PBJX4CteRdQz/argtFPksY3F+2ZbnNskzM+kDPtoD0AG9KD4wiuiIoTJp3l2WVgHrZmuIos+3Fwg5d0JAWMGcDVsECGpfq4wGn9bD39lfu7/wC0n+AXWrjf6HwXf4Tcu/KHpn2TMK+A9aRrhaABIPdyET12dQ2opz9kLWxLfx9vYh0QnnAxrJDCyRs4SJpNpQt27cFPsYNWNn7C1qGlF2Vtyc2LtzChZzuPXFlo3mcGBCe/dnFMZ5w7nQUI3XC92Hh2QTUw262DQa1oEMpIc7JzxIKxIAp0KB2wRiTHAjSuKiGeweBjn8wT7Ve5Upzv+GNEZrgXjSm17H3vXgU10uDXVh2MlVGFBTY9wVyq5Cax+k0jA1w00NDMwPDI6Emvj1NGVqKSFFHKOvwhMVBk5e9U8dQZHHVPsSIZS7NCgf25cA568i9A/w0CyyssoxD92g07Tc6oyQyemMs1+vnorOVbRBDn9o3XKNX7K2INS+GPzCvvbpghbpjZSbiTTcvD0XfwouJfyPK0Cp9ZL+9s1ou7pTZMmGHZyyW4zT6uMsBIwyWt+CdFrG6GsM84IY4r6BlJ1jrSlvEhvB5BJnDkd1ILkf02Ob1LwslS9nse3r6bas+qWBB6cAT+umWqb0hdALJS3KpOiluXlcWf24OWcsuJ9CR/+iDpcvvVjHkhg7W11ZN1VawvmlYtCqQAuCoAoNnTdSttN9au8KLRzZHG9Mh4cYGiXu2RrzjmiQuNFlNEN3/oEpscr8BKPexguhRPKd5Y2ooXH1Hh0TrPUAUM7Xb+lTulALtrFyBrQ+Sg7QEhEhH5tWP1OgHKqPkK2sfEG1f9UkASe30Vtymw9+6AsjNVbOk0N+9k8+lFnmqe46+O8PUYd681ZOHIPaWYhgk3i7ZdzYeRWTYaTn0o/iExXLmh8HMNG/SQCiSJR55KYvAAORyNur7ldFl9vtw1jyH5JYyVSZvuw5K6306CNvBDcGXc5bEyXGrCcixfFSRg9LdCoDR2PWbNncoR3eT5C10vFoSwY1xNP1PxoISbrbAQxaKzGQFHDPDfGwCZXmmAarETXMcbi+eddurcDrbUym5FIoizZ7IqMRUuKe5m/P//Wrjg1tFkVck/8t0nqXWH7ylyO5HD7XxCmxPB3PplwNKlLkHZq2w7BK0WYnTFCpJ9dhDyGKbZimoHRQONoNsAlrbH0NKSbxPlMB0SiVz0KaPwyr/bOfQ+lXH3SiHWQ3jg/FjisCsDvb19AN2tVN5yRsGnkMrAwSLWi/QOLchMWwJgBc9dM8c5gcRy2SrNpBhnHLZKy/Yk8qTuXI1sieKhYjxpDNMISyrDeKIpUH8cqRoRscaNtcSPiIEZCOFAPA50JS5O8WdZnbzb0r9GTnnvYRAjjdOtqGQdMHMT+8MiWVstQgQ/Uw8yOWTWTCR07jMej13Z9Sw==", 18 | )) 19 | 20 | ciphertext, _ = bdcrypto.DESCBCDecrypt3(data, []byte("common3DESCode$%^_wangsu"), []byte("01234567")) 21 | fmt.Printf("%s\n", ciphertext) 22 | } 23 | -------------------------------------------------------------------------------- /utils/crypto.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "github.com/iikira/iikira-go-utils/utils/bdcrypto" 6 | "io" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | // CryptoMethodSupport 检测是否支持加密解密方法 12 | func CryptoMethodSupport(method string) bool { 13 | switch method { 14 | case "aes-128-ctr", "aes-192-ctr", "aes-256-ctr", "aes-128-cfb", "aes-192-cfb", "aes-256-cfb", "aes-128-ofb", "aes-192-ofb", "aes-256-ofb": 15 | return true 16 | } 17 | 18 | return false 19 | } 20 | 21 | // EncryptFile 加密本地文件 22 | func EncryptFile(method string, key []byte, filePath string, isGzip bool) (encryptedFilePath string, err error) { 23 | if !CryptoMethodSupport(method) { 24 | return "", fmt.Errorf("unknown encrypt method: %s", method) 25 | } 26 | 27 | if isGzip { 28 | err = bdcrypto.GZIPCompressFile(filePath) 29 | if err != nil { 30 | return 31 | } 32 | } 33 | 34 | plainFile, err := os.OpenFile(filePath, os.O_RDONLY, 0) 35 | if err != nil { 36 | return 37 | } 38 | 39 | defer plainFile.Close() 40 | 41 | var cipherReader io.Reader 42 | switch method { 43 | case "aes-128-ctr": 44 | cipherReader, err = bdcrypto.Aes128CTREncrypt(bdcrypto.Convert16bytes(key), plainFile) 45 | case "aes-192-ctr": 46 | cipherReader, err = bdcrypto.Aes192CTREncrypt(bdcrypto.Convert24bytes(key), plainFile) 47 | case "aes-256-ctr": 48 | cipherReader, err = bdcrypto.Aes256CTREncrypt(bdcrypto.Convert32bytes(key), plainFile) 49 | case "aes-128-cfb": 50 | cipherReader, err = bdcrypto.Aes128CFBEncrypt(bdcrypto.Convert16bytes(key), plainFile) 51 | case "aes-192-cfb": 52 | cipherReader, err = bdcrypto.Aes192CFBEncrypt(bdcrypto.Convert24bytes(key), plainFile) 53 | case "aes-256-cfb": 54 | cipherReader, err = bdcrypto.Aes256CFBEncrypt(bdcrypto.Convert32bytes(key), plainFile) 55 | case "aes-128-ofb": 56 | cipherReader, err = bdcrypto.Aes128OFBEncrypt(bdcrypto.Convert16bytes(key), plainFile) 57 | case "aes-192-ofb": 58 | cipherReader, err = bdcrypto.Aes192OFBEncrypt(bdcrypto.Convert24bytes(key), plainFile) 59 | case "aes-256-ofb": 60 | cipherReader, err = bdcrypto.Aes256OFBEncrypt(bdcrypto.Convert32bytes(key), plainFile) 61 | default: 62 | return "", fmt.Errorf("unknown encrypt method: %s", method) 63 | } 64 | 65 | if err != nil { 66 | return 67 | } 68 | 69 | plainFileInfo, err := plainFile.Stat() 70 | if err != nil { 71 | return 72 | } 73 | 74 | encryptedFilePath = filePath + ".encrypt" 75 | encryptedFile, err := os.OpenFile(encryptedFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, plainFileInfo.Mode()) 76 | if err != nil { 77 | return 78 | } 79 | 80 | defer encryptedFile.Close() 81 | 82 | _, err = io.Copy(encryptedFile, cipherReader) 83 | if err != nil { 84 | return 85 | } 86 | 87 | os.Remove(filePath) 88 | 89 | return encryptedFilePath, nil 90 | } 91 | 92 | // DecryptFile 加密本地文件 93 | func DecryptFile(method string, key []byte, filePath string, isGzip bool) (decryptedFilePath string, err error) { 94 | if !CryptoMethodSupport(method) { 95 | return "", fmt.Errorf("unknown decrypt method: %s", method) 96 | } 97 | 98 | cipherFile, err := os.OpenFile(filePath, os.O_RDONLY, 0644) 99 | if err != nil { 100 | return 101 | } 102 | 103 | defer cipherFile.Close() 104 | 105 | var plainReader io.Reader 106 | switch method { 107 | case "aes-128-ctr": 108 | plainReader, err = bdcrypto.Aes128CTRDecrypt(bdcrypto.Convert16bytes(key), cipherFile) 109 | case "aes-192-ctr": 110 | plainReader, err = bdcrypto.Aes192CTRDecrypt(bdcrypto.Convert24bytes(key), cipherFile) 111 | case "aes-256-ctr": 112 | plainReader, err = bdcrypto.Aes256CTRDecrypt(bdcrypto.Convert32bytes(key), cipherFile) 113 | case "aes-128-cfb": 114 | plainReader, err = bdcrypto.Aes128CFBDecrypt(bdcrypto.Convert16bytes(key), cipherFile) 115 | case "aes-192-cfb": 116 | plainReader, err = bdcrypto.Aes192CFBDecrypt(bdcrypto.Convert24bytes(key), cipherFile) 117 | case "aes-256-cfb": 118 | plainReader, err = bdcrypto.Aes256CFBDecrypt(bdcrypto.Convert32bytes(key), cipherFile) 119 | case "aes-128-ofb": 120 | plainReader, err = bdcrypto.Aes128OFBDecrypt(bdcrypto.Convert16bytes(key), cipherFile) 121 | case "aes-192-ofb": 122 | plainReader, err = bdcrypto.Aes192OFBDecrypt(bdcrypto.Convert24bytes(key), cipherFile) 123 | case "aes-256-ofb": 124 | plainReader, err = bdcrypto.Aes256OFBDecrypt(bdcrypto.Convert32bytes(key), cipherFile) 125 | default: 126 | return "", fmt.Errorf("unknown decrypt method: %s", method) 127 | } 128 | 129 | if err != nil { 130 | return 131 | } 132 | 133 | cipherFileInfo, err := cipherFile.Stat() 134 | if err != nil { 135 | return 136 | } 137 | 138 | decryptedFilePath = strings.TrimSuffix(filePath, ".encrypt") 139 | decryptedTmpFilePath := decryptedFilePath + ".decrypted" 140 | decryptedTmpFile, err := os.OpenFile(decryptedTmpFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, cipherFileInfo.Mode()) 141 | if err != nil { 142 | return 143 | } 144 | 145 | _, err = io.Copy(decryptedTmpFile, plainReader) 146 | if err != nil { 147 | return 148 | } 149 | 150 | defer decryptedTmpFile.Close() 151 | 152 | if isGzip { 153 | err = bdcrypto.GZIPUnompressFile(decryptedTmpFilePath) 154 | if err != nil { 155 | os.Remove(decryptedTmpFilePath) 156 | return 157 | } 158 | 159 | // 删除已加密的文件 160 | os.Remove(filePath) 161 | } 162 | 163 | if filePath != decryptedFilePath { 164 | os.Rename(decryptedTmpFilePath, decryptedFilePath) 165 | } else { 166 | decryptedFilePath = decryptedTmpFilePath 167 | } 168 | 169 | return decryptedFilePath, nil 170 | } 171 | -------------------------------------------------------------------------------- /utils/checksum/checksum.go: -------------------------------------------------------------------------------- 1 | // Package checksum 校验本地文件包 2 | package checksum 3 | 4 | import ( 5 | "crypto/md5" 6 | "github.com/iikira/iikira-go-utils/utils/cachepool" 7 | "github.com/iikira/iikira-go-utils/utils/converter" 8 | "hash/crc32" 9 | "io" 10 | "os" 11 | ) 12 | 13 | const ( 14 | // DefaultBufSize 默认的bufSize 15 | DefaultBufSize = int(256 * converter.KB) 16 | ) 17 | 18 | const ( 19 | // CHECKSUM_MD5 获取文件的 md5 值 20 | CHECKSUM_MD5 int = 1 << iota 21 | // CHECKSUM_SLICE_MD5 获取文件前 sliceSize 切片的 md5 值 22 | CHECKSUM_SLICE_MD5 23 | // CHECKSUM_CRC32 获取文件的 crc32 值 24 | CHECKSUM_CRC32 25 | ) 26 | 27 | type ( 28 | // LocalFileMeta 本地文件元信息 29 | LocalFileMeta struct { 30 | Path string `json:"path"` // 本地路径 31 | Length int64 `json:"length"` // 文件大小 32 | SliceMD5 []byte `json:"slicemd5"` // 文件前 requiredSliceLen (256KB) 切片的 md5 值 33 | MD5 []byte `json:"md5"` // 文件的 md5 34 | CRC32 uint32 `json:"crc32"` // 文件的 crc32 35 | ModTime int64 `json:"modtime"` // 修改日期 36 | } 37 | 38 | // LocalFileChecksum 校验本地文件 39 | LocalFileChecksum struct { 40 | LocalFileMeta 41 | bufSize int 42 | sliceSize int 43 | buf []byte 44 | file *os.File // 文件 45 | } 46 | ) 47 | 48 | func NewLocalFileChecksum(localPath string, sliceSize int) *LocalFileChecksum { 49 | return NewLocalFileChecksumWithBufSize(localPath, DefaultBufSize, sliceSize) 50 | } 51 | 52 | func NewLocalFileChecksumWithBufSize(localPath string, bufSize, sliceSize int) *LocalFileChecksum { 53 | return &LocalFileChecksum{ 54 | LocalFileMeta: LocalFileMeta{ 55 | Path: localPath, 56 | }, 57 | bufSize: bufSize, 58 | sliceSize: sliceSize, 59 | } 60 | } 61 | 62 | // OpenPath 检查文件状态并获取文件的大小 (Length) 63 | func (lfc *LocalFileChecksum) OpenPath() error { 64 | if lfc.file != nil { 65 | lfc.file.Close() 66 | } 67 | 68 | var err error 69 | lfc.file, err = os.Open(lfc.Path) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | info, err := lfc.file.Stat() 75 | if err != nil { 76 | return err 77 | } 78 | 79 | lfc.Length = info.Size() 80 | lfc.ModTime = info.ModTime().Unix() 81 | return nil 82 | } 83 | 84 | // GetFile 获取文件 85 | func (lfc *LocalFileChecksum) GetFile() *os.File { 86 | return lfc.file 87 | } 88 | 89 | // Close 关闭文件 90 | func (lfc *LocalFileChecksum) Close() error { 91 | if lfc.file == nil { 92 | return ErrFileIsNil 93 | } 94 | 95 | return lfc.file.Close() 96 | } 97 | 98 | func (lfc *LocalFileChecksum) initBuf() { 99 | if lfc.buf == nil { 100 | lfc.buf = cachepool.RawMallocByteSlice(lfc.bufSize) 101 | } 102 | } 103 | 104 | func (lfc *LocalFileChecksum) writeChecksum(data []byte, wus ...*ChecksumWriteUnit) (err error) { 105 | doneCount := 0 106 | for _, wu := range wus { 107 | _, err := wu.Write(data) 108 | switch err { 109 | case ErrChecksumWriteStop: 110 | doneCount++ 111 | continue 112 | case nil: 113 | default: 114 | return err 115 | } 116 | } 117 | if doneCount == len(wus) { 118 | return ErrChecksumWriteAllStop 119 | } 120 | return nil 121 | } 122 | 123 | func (lfc *LocalFileChecksum) repeatRead(wus ...*ChecksumWriteUnit) (err error) { 124 | if lfc.file == nil { 125 | return ErrFileIsNil 126 | } 127 | 128 | lfc.initBuf() 129 | 130 | defer func() { 131 | _, err = lfc.file.Seek(0, os.SEEK_SET) // 恢复文件指针 132 | if err != nil { 133 | return 134 | } 135 | }() 136 | 137 | // 读文件 138 | var ( 139 | n int 140 | ) 141 | read: 142 | for { 143 | n, err = lfc.file.Read(lfc.buf) 144 | switch err { 145 | case io.EOF: 146 | err = lfc.writeChecksum(lfc.buf[:n], wus...) 147 | break read 148 | case nil: 149 | err = lfc.writeChecksum(lfc.buf[:n], wus...) 150 | default: 151 | return 152 | } 153 | } 154 | switch err { 155 | case ErrChecksumWriteAllStop: // 全部结束 156 | err = nil 157 | } 158 | return 159 | } 160 | 161 | func (lfc *LocalFileChecksum) createChecksumWriteUnit(cw ChecksumWriter, isAll, isSlice bool, getSumFunc func(sliceSum interface{}, sum interface{})) (wu *ChecksumWriteUnit, deferFunc func(err error)) { 162 | wu = &ChecksumWriteUnit{ 163 | ChecksumWriter: cw, 164 | End: lfc.LocalFileMeta.Length, 165 | OnlySliceSum: !isAll, 166 | } 167 | 168 | if isSlice { 169 | wu.SliceEnd = int64(lfc.sliceSize) 170 | } 171 | 172 | return wu, func(err error) { 173 | if err != nil { 174 | return 175 | } 176 | getSumFunc(wu.SliceSum, wu.Sum) 177 | } 178 | } 179 | 180 | // Sum 计算文件摘要值 181 | func (lfc *LocalFileChecksum) Sum(checkSumFlag int) (err error) { 182 | lfc.fix() 183 | wus := make([]*ChecksumWriteUnit, 0, 2) 184 | if (checkSumFlag & (CHECKSUM_MD5 | CHECKSUM_SLICE_MD5)) != 0 { 185 | md5w := md5.New() 186 | wu, d := lfc.createChecksumWriteUnit( 187 | NewHashChecksumWriter(md5w), 188 | (checkSumFlag&CHECKSUM_MD5) != 0, 189 | (checkSumFlag&CHECKSUM_SLICE_MD5) != 0, 190 | func(sliceSum interface{}, sum interface{}) { 191 | if sliceSum != nil { 192 | lfc.SliceMD5 = sliceSum.([]byte) 193 | } 194 | if sum != nil { 195 | lfc.MD5 = sum.([]byte) 196 | } 197 | }, 198 | ) 199 | 200 | wus = append(wus, wu) 201 | defer d(err) 202 | } 203 | if (checkSumFlag & CHECKSUM_CRC32) != 0 { 204 | crc32w := crc32.NewIEEE() 205 | wu, d := lfc.createChecksumWriteUnit( 206 | NewHash32ChecksumWriter(crc32w), 207 | true, 208 | false, 209 | func(sliceSum interface{}, sum interface{}) { 210 | if sum != nil { 211 | lfc.CRC32 = sum.(uint32) 212 | } 213 | }, 214 | ) 215 | 216 | wus = append(wus, wu) 217 | defer d(err) 218 | } 219 | 220 | err = lfc.repeatRead(wus...) 221 | return 222 | } 223 | 224 | func (lfc *LocalFileChecksum) fix() { 225 | if lfc.sliceSize <= 0 { 226 | lfc.sliceSize = DefaultBufSize 227 | } 228 | if lfc.bufSize < DefaultBufSize { 229 | lfc.bufSize = DefaultBufSize 230 | } 231 | } 232 | -------------------------------------------------------------------------------- /requester/uploader/multiuploader.go: -------------------------------------------------------------------------------- 1 | package uploader 2 | 3 | import ( 4 | "context" 5 | "github.com/iikira/iikira-go-utils/requester" 6 | "github.com/iikira/iikira-go-utils/requester/rio" 7 | "github.com/iikira/iikira-go-utils/requester/rio/speeds" 8 | "github.com/iikira/iikira-go-utils/utils" 9 | "github.com/iikira/iikira-go-utils/utils/converter" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | type ( 15 | // MultiUpload 支持多线程的上传, 可用于断点续传 16 | MultiUpload interface { 17 | Precreate() (perr error) 18 | TmpFile(ctx context.Context, partseq int, partOffset int64, readerlen64 rio.ReaderLen64) (checksum string, terr error) 19 | CreateSuperFile(checksumList ...string) (cerr error) 20 | } 21 | 22 | // MultiUploader 多线程上传 23 | MultiUploader struct { 24 | onExecuteEvent requester.Event //开始上传事件 25 | onSuccessEvent requester.Event //成功上传事件 26 | onFinishEvent requester.Event //结束上传事件 27 | onCancelEvent requester.Event //取消上传事件 28 | onErrorEvent requester.EventOnError //上传出错事件 29 | onUploadStatusEvent UploadStatusFunc //上传状态事件 30 | 31 | instanceState *InstanceState 32 | 33 | multiUpload MultiUpload // 上传体接口 34 | file rio.ReaderAtLen64 // 上传 35 | config *MultiUploaderConfig 36 | workers workerList 37 | speedsStat *speeds.Speeds 38 | rateLimit *speeds.RateLimit 39 | 40 | executeTime time.Time 41 | finished chan struct{} 42 | canceled chan struct{} 43 | closeCanceledOnce sync.Once 44 | updateInstanceStateChan chan struct{} 45 | } 46 | 47 | // MultiUploaderConfig 多线程上传配置 48 | MultiUploaderConfig struct { 49 | Parallel int // 上传并发量 50 | BlockSize int64 // 上传分块 51 | MaxRate int64 // 限制最大上传速度 52 | } 53 | ) 54 | 55 | // NewMultiUploader 初始化上传 56 | func NewMultiUploader(multiUpload MultiUpload, file rio.ReaderAtLen64, config *MultiUploaderConfig) *MultiUploader { 57 | return &MultiUploader{ 58 | multiUpload: multiUpload, 59 | file: file, 60 | config: config, 61 | } 62 | } 63 | 64 | // SetInstanceState 设置InstanceState, 断点续传信息 65 | func (muer *MultiUploader) SetInstanceState(is *InstanceState) { 66 | muer.instanceState = is 67 | } 68 | 69 | func (muer *MultiUploader) lazyInit() { 70 | if muer.finished == nil { 71 | muer.finished = make(chan struct{}, 1) 72 | } 73 | if muer.canceled == nil { 74 | muer.canceled = make(chan struct{}) 75 | } 76 | if muer.updateInstanceStateChan == nil { 77 | muer.updateInstanceStateChan = make(chan struct{}, 1) 78 | } 79 | if muer.config == nil { 80 | muer.config = &MultiUploaderConfig{} 81 | } 82 | if muer.config.Parallel <= 0 { 83 | muer.config.Parallel = 4 84 | } 85 | if muer.config.BlockSize <= 0 { 86 | muer.config.BlockSize = 1 * converter.GB 87 | } 88 | if muer.speedsStat == nil { 89 | muer.speedsStat = &speeds.Speeds{} 90 | } 91 | } 92 | 93 | func (muer *MultiUploader) check() { 94 | if muer.file == nil { 95 | panic("file is nil") 96 | } 97 | if muer.multiUpload == nil { 98 | panic("multiUpload is nil") 99 | } 100 | } 101 | 102 | // Execute 执行上传 103 | func (muer *MultiUploader) Execute() { 104 | muer.check() 105 | muer.lazyInit() 106 | 107 | // 初始化限速 108 | if muer.config.MaxRate > 0 { 109 | muer.rateLimit = speeds.NewRateLimit(muer.config.MaxRate) 110 | defer muer.rateLimit.Stop() 111 | } 112 | 113 | // 分配任务 114 | if muer.instanceState != nil { 115 | muer.workers = muer.getWorkerListByInstanceState(muer.instanceState) 116 | uploaderVerbose.Infof("upload task CREATED from instance state\n") 117 | } else { 118 | muer.workers = muer.getWorkerListByInstanceState(&InstanceState{ 119 | BlockList: SplitBlock(muer.file.Len(), muer.config.BlockSize), 120 | }) 121 | 122 | uploaderVerbose.Infof("upload task CREATED: block size: %d, num: %d\n", muer.config.BlockSize, len(muer.workers)) 123 | } 124 | 125 | // 开始上传 126 | muer.executeTime = time.Now() 127 | utils.Trigger(muer.onExecuteEvent) 128 | 129 | muer.uploadStatusEvent() 130 | 131 | err := muer.upload() 132 | 133 | // 完成 134 | muer.finished <- struct{}{} 135 | if err != nil { 136 | if err == context.Canceled { 137 | if muer.onCancelEvent != nil { 138 | muer.onCancelEvent() 139 | } 140 | } else if muer.onErrorEvent != nil { 141 | muer.onErrorEvent(err) 142 | } 143 | } else { 144 | utils.TriggerOnSync(muer.onSuccessEvent) 145 | } 146 | utils.TriggerOnSync(muer.onFinishEvent) 147 | } 148 | 149 | // InstanceState 返回断点续传信息 150 | func (muer *MultiUploader) InstanceState() *InstanceState { 151 | blockStates := make([]*BlockState, 0, len(muer.workers)) 152 | for _, wer := range muer.workers { 153 | blockStates = append(blockStates, &BlockState{ 154 | ID: wer.id, 155 | Range: wer.splitUnit.Range(), 156 | CheckSum: wer.checksum, 157 | }) 158 | } 159 | return &InstanceState{ 160 | BlockList: blockStates, 161 | } 162 | } 163 | 164 | // Cancel 取消上传 165 | func (muer *MultiUploader) Cancel() { 166 | close(muer.canceled) 167 | } 168 | 169 | //OnExecute 设置开始上传事件 170 | func (muer *MultiUploader) OnExecute(onExecuteEvent requester.Event) { 171 | muer.onExecuteEvent = onExecuteEvent 172 | } 173 | 174 | //OnSuccess 设置成功上传事件 175 | func (muer *MultiUploader) OnSuccess(onSuccessEvent requester.Event) { 176 | muer.onSuccessEvent = onSuccessEvent 177 | } 178 | 179 | //OnFinish 设置结束上传事件 180 | func (muer *MultiUploader) OnFinish(onFinishEvent requester.Event) { 181 | muer.onFinishEvent = onFinishEvent 182 | } 183 | 184 | //OnCancel 设置取消上传事件 185 | func (muer *MultiUploader) OnCancel(onCancelEvent requester.Event) { 186 | muer.onCancelEvent = onCancelEvent 187 | } 188 | 189 | //OnError 设置上传发生错误事件 190 | func (muer *MultiUploader) OnError(onErrorEvent requester.EventOnError) { 191 | muer.onErrorEvent = onErrorEvent 192 | } 193 | 194 | //OnUploadStatusEvent 设置上传状态事件 195 | func (muer *MultiUploader) OnUploadStatusEvent(f UploadStatusFunc) { 196 | muer.onUploadStatusEvent = f 197 | } 198 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= 2 | github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= 7 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 8 | github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= 9 | github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= 10 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 11 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 12 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 13 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 14 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 15 | github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= 16 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 17 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 18 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 19 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 20 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 21 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 22 | github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= 23 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 24 | github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= 25 | github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= 26 | github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= 27 | github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 28 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 29 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 30 | github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg= 31 | github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= 32 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 33 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 34 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= 35 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 36 | github.com/oleiade/lane v1.0.1 h1:hXofkn7GEOubzTwNpeL9MaNy8WxolCYb9cInAIeqShU= 37 | github.com/oleiade/lane v1.0.1/go.mod h1:IyTkraa4maLfjq/GmHR+Dxb4kCMtEGeb+qmhlrQ5Mk4= 38 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 39 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 40 | github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY= 41 | github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 42 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 43 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 44 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 45 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 46 | golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= 47 | golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= 48 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 49 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 50 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 51 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 52 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 53 | golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78 h1:nVuTkr9L6Bq62qpUqKo/RnZCFfzDBL0bYo6w9OJUqZY= 54 | golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 55 | golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= 56 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 57 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 58 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 59 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 60 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 61 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 62 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 63 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 64 | google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= 65 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 66 | -------------------------------------------------------------------------------- /requester/transfer/transfer.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // source: transfer/transfer.proto 3 | 4 | package transfer 5 | 6 | import ( 7 | fmt "fmt" 8 | proto "github.com/golang/protobuf/proto" 9 | math "math" 10 | ) 11 | 12 | // Reference imports to suppress errors if they are not otherwise used. 13 | var _ = proto.Marshal 14 | var _ = fmt.Errorf 15 | var _ = math.Inf 16 | 17 | // This is a compile-time assertion to ensure that this generated file 18 | // is compatible with the proto package it is being compiled against. 19 | // A compilation error at this line likely means your copy of the 20 | // proto package needs to be updated. 21 | const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package 22 | 23 | // RangeGenMode 线程分配方式 24 | type RangeGenMode int32 25 | 26 | const ( 27 | // RangeGenMode_Default 根据parallel平均生成 28 | RangeGenMode_Default RangeGenMode = 0 29 | // RangeGenMode_BlockSize 根据blockSize生成 30 | RangeGenMode_BlockSize RangeGenMode = 1 31 | ) 32 | 33 | var RangeGenMode_name = map[int32]string{ 34 | 0: "Default", 35 | 1: "BlockSize", 36 | } 37 | 38 | var RangeGenMode_value = map[string]int32{ 39 | "Default": 0, 40 | "BlockSize": 1, 41 | } 42 | 43 | func (x RangeGenMode) String() string { 44 | return proto.EnumName(RangeGenMode_name, int32(x)) 45 | } 46 | 47 | func (RangeGenMode) EnumDescriptor() ([]byte, []int) { 48 | return fileDescriptor_44038b0c710d7f2f, []int{0} 49 | } 50 | 51 | //Range 请求范围 52 | type Range struct { 53 | Begin int64 `protobuf:"varint,1,opt,name=begin,proto3" json:"begin,omitempty"` 54 | End int64 `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` 55 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 56 | XXX_unrecognized []byte `json:"-"` 57 | XXX_sizecache int32 `json:"-"` 58 | } 59 | 60 | func (m *Range) Reset() { *m = Range{} } 61 | func (m *Range) String() string { return proto.CompactTextString(m) } 62 | func (*Range) ProtoMessage() {} 63 | func (*Range) Descriptor() ([]byte, []int) { 64 | return fileDescriptor_44038b0c710d7f2f, []int{0} 65 | } 66 | 67 | func (m *Range) XXX_Unmarshal(b []byte) error { 68 | return xxx_messageInfo_Range.Unmarshal(m, b) 69 | } 70 | func (m *Range) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 71 | return xxx_messageInfo_Range.Marshal(b, m, deterministic) 72 | } 73 | func (m *Range) XXX_Merge(src proto.Message) { 74 | xxx_messageInfo_Range.Merge(m, src) 75 | } 76 | func (m *Range) XXX_Size() int { 77 | return xxx_messageInfo_Range.Size(m) 78 | } 79 | func (m *Range) XXX_DiscardUnknown() { 80 | xxx_messageInfo_Range.DiscardUnknown(m) 81 | } 82 | 83 | var xxx_messageInfo_Range proto.InternalMessageInfo 84 | 85 | func (m *Range) GetBegin() int64 { 86 | if m != nil { 87 | return m.Begin 88 | } 89 | return 0 90 | } 91 | 92 | func (m *Range) GetEnd() int64 { 93 | if m != nil { 94 | return m.End 95 | } 96 | return 0 97 | } 98 | 99 | // DownloadInstanceInfoExport 断点续传 100 | type DownloadInstanceInfoExport struct { 101 | RangeGenMode RangeGenMode `protobuf:"varint,1,opt,name=range_gen_mode,json=rangeGenMode,proto3,enum=transfer.RangeGenMode" json:"range_gen_mode,omitempty"` 102 | TotalSize int64 `protobuf:"varint,2,opt,name=total_size,json=totalSize,proto3" json:"total_size,omitempty"` 103 | GenBegin int64 `protobuf:"varint,3,opt,name=gen_begin,json=genBegin,proto3" json:"gen_begin,omitempty"` 104 | BlockSize int64 `protobuf:"varint,4,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"` 105 | Ranges []*Range `protobuf:"bytes,5,rep,name=ranges,proto3" json:"ranges,omitempty"` 106 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 107 | XXX_unrecognized []byte `json:"-"` 108 | XXX_sizecache int32 `json:"-"` 109 | } 110 | 111 | func (m *DownloadInstanceInfoExport) Reset() { *m = DownloadInstanceInfoExport{} } 112 | func (m *DownloadInstanceInfoExport) String() string { return proto.CompactTextString(m) } 113 | func (*DownloadInstanceInfoExport) ProtoMessage() {} 114 | func (*DownloadInstanceInfoExport) Descriptor() ([]byte, []int) { 115 | return fileDescriptor_44038b0c710d7f2f, []int{1} 116 | } 117 | 118 | func (m *DownloadInstanceInfoExport) XXX_Unmarshal(b []byte) error { 119 | return xxx_messageInfo_DownloadInstanceInfoExport.Unmarshal(m, b) 120 | } 121 | func (m *DownloadInstanceInfoExport) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 122 | return xxx_messageInfo_DownloadInstanceInfoExport.Marshal(b, m, deterministic) 123 | } 124 | func (m *DownloadInstanceInfoExport) XXX_Merge(src proto.Message) { 125 | xxx_messageInfo_DownloadInstanceInfoExport.Merge(m, src) 126 | } 127 | func (m *DownloadInstanceInfoExport) XXX_Size() int { 128 | return xxx_messageInfo_DownloadInstanceInfoExport.Size(m) 129 | } 130 | func (m *DownloadInstanceInfoExport) XXX_DiscardUnknown() { 131 | xxx_messageInfo_DownloadInstanceInfoExport.DiscardUnknown(m) 132 | } 133 | 134 | var xxx_messageInfo_DownloadInstanceInfoExport proto.InternalMessageInfo 135 | 136 | func (m *DownloadInstanceInfoExport) GetRangeGenMode() RangeGenMode { 137 | if m != nil { 138 | return m.RangeGenMode 139 | } 140 | return RangeGenMode_Default 141 | } 142 | 143 | func (m *DownloadInstanceInfoExport) GetTotalSize() int64 { 144 | if m != nil { 145 | return m.TotalSize 146 | } 147 | return 0 148 | } 149 | 150 | func (m *DownloadInstanceInfoExport) GetGenBegin() int64 { 151 | if m != nil { 152 | return m.GenBegin 153 | } 154 | return 0 155 | } 156 | 157 | func (m *DownloadInstanceInfoExport) GetBlockSize() int64 { 158 | if m != nil { 159 | return m.BlockSize 160 | } 161 | return 0 162 | } 163 | 164 | func (m *DownloadInstanceInfoExport) GetRanges() []*Range { 165 | if m != nil { 166 | return m.Ranges 167 | } 168 | return nil 169 | } 170 | 171 | func init() { 172 | proto.RegisterEnum("transfer.RangeGenMode", RangeGenMode_name, RangeGenMode_value) 173 | proto.RegisterType((*Range)(nil), "transfer.Range") 174 | proto.RegisterType((*DownloadInstanceInfoExport)(nil), "transfer.DownloadInstanceInfoExport") 175 | } 176 | 177 | func init() { proto.RegisterFile("transfer/transfer.proto", fileDescriptor_44038b0c710d7f2f) } 178 | 179 | var fileDescriptor_44038b0c710d7f2f = []byte{ 180 | // 260 bytes of a gzipped FileDescriptorProto 181 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x90, 0x41, 0x4b, 0xc3, 0x40, 182 | 0x10, 0x85, 0x8d, 0x31, 0xb5, 0x99, 0xd6, 0x1a, 0x16, 0xd1, 0xa0, 0x14, 0x4a, 0x2f, 0x96, 0x1e, 183 | 0x5a, 0xa8, 0x57, 0x4f, 0xa5, 0x22, 0x3d, 0x78, 0x89, 0x3f, 0x20, 0x6c, 0x9a, 0x49, 0x08, 0xc6, 184 | 0x99, 0xb2, 0x59, 0x51, 0xfa, 0x63, 0xfd, 0x2d, 0x92, 0x49, 0x23, 0xa1, 0xb7, 0x7d, 0x6f, 0x78, 185 | 0xef, 0x7d, 0x2c, 0xdc, 0x59, 0xa3, 0xa9, 0xca, 0xd0, 0x2c, 0xdb, 0xc7, 0x62, 0x6f, 0xd8, 0xb2, 186 | 0xea, 0xb7, 0x7a, 0xba, 0x04, 0x2f, 0xd2, 0x94, 0xa3, 0xba, 0x01, 0x2f, 0xc1, 0xbc, 0xa0, 0xd0, 187 | 0x99, 0x38, 0x33, 0x37, 0x6a, 0x84, 0x0a, 0xc0, 0x45, 0x4a, 0xc3, 0x73, 0xf1, 0xea, 0xe7, 0xf4, 188 | 0xd7, 0x81, 0xfb, 0x0d, 0x7f, 0x53, 0xc9, 0x3a, 0xdd, 0x52, 0x65, 0x35, 0xed, 0x70, 0x4b, 0x19, 189 | 0xbf, 0xfc, 0xec, 0xd9, 0x58, 0xf5, 0x0c, 0x23, 0x53, 0xf7, 0xc5, 0x39, 0x52, 0xfc, 0xc9, 0x29, 190 | 0x4a, 0xdf, 0x68, 0x75, 0xbb, 0xf8, 0x47, 0x90, 0xbd, 0x57, 0xa4, 0x37, 0x4e, 0x31, 0x1a, 0x9a, 191 | 0x8e, 0x52, 0x63, 0x00, 0xcb, 0x56, 0x97, 0x71, 0x55, 0x1c, 0xf0, 0xb8, 0xea, 0x8b, 0xf3, 0x5e, 192 | 0x1c, 0x50, 0x3d, 0x80, 0x5f, 0xd7, 0x36, 0x9c, 0xae, 0x5c, 0xfb, 0x39, 0xd2, 0x5a, 0x50, 0xc7, 193 | 0x00, 0x49, 0xc9, 0xbb, 0x8f, 0x26, 0x7b, 0xd1, 0x64, 0xc5, 0x91, 0xec, 0x23, 0xf4, 0x64, 0xaa, 194 | 0x0a, 0xbd, 0x89, 0x3b, 0x1b, 0xac, 0xae, 0x4f, 0x80, 0xa2, 0xe3, 0x79, 0x3e, 0x87, 0x61, 0x97, 195 | 0x50, 0x0d, 0xe0, 0x72, 0x83, 0x99, 0xfe, 0x2a, 0x6d, 0x70, 0xa6, 0xae, 0xc0, 0x5f, 0xb7, 0x95, 196 | 0x81, 0x93, 0xf4, 0xe4, 0x3b, 0x9f, 0xfe, 0x02, 0x00, 0x00, 0xff, 0xff, 0xe8, 0x5c, 0x09, 0xcc, 197 | 0x69, 0x01, 0x00, 0x00, 198 | } 199 | -------------------------------------------------------------------------------- /requester/downloader/monitor.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "github.com/iikira/iikira-go-utils/pcsverbose" 7 | "github.com/iikira/iikira-go-utils/requester/transfer" 8 | "sort" 9 | "time" 10 | ) 11 | 12 | var ( 13 | //ErrNoWokers no workers 14 | ErrNoWokers = errors.New("no workers") 15 | ) 16 | 17 | type ( 18 | //Monitor 线程监控器 19 | Monitor struct { 20 | workers WorkerList 21 | status *transfer.DownloadStatus 22 | instanceState *InstanceState 23 | completed chan struct{} 24 | err error 25 | resetController *ResetController 26 | isReloadWorker bool //是否重载worker, 单线程模式不重载 27 | 28 | // 临时变量 29 | lastAvaliableIndex int 30 | } 31 | 32 | // RangeWorkerFunc 遍历workers的函数 33 | RangeWorkerFunc func(key int, worker *Worker) bool 34 | ) 35 | 36 | //NewMonitor 初始化Monitor 37 | func NewMonitor() *Monitor { 38 | monitor := &Monitor{} 39 | return monitor 40 | } 41 | 42 | func (mt *Monitor) lazyInit() { 43 | if mt.workers == nil { 44 | mt.workers = make(WorkerList, 0, 100) 45 | } 46 | if mt.status == nil { 47 | mt.status = transfer.NewDownloadStatus() 48 | } 49 | if mt.resetController == nil { 50 | mt.resetController = NewResetController(80) 51 | } 52 | } 53 | 54 | //InitMonitorCapacity 初始化workers, 用于Append 55 | func (mt *Monitor) InitMonitorCapacity(capacity int) { 56 | mt.workers = make(WorkerList, 0, capacity) 57 | } 58 | 59 | //Append 增加Worker 60 | func (mt *Monitor) Append(worker *Worker) { 61 | if worker == nil { 62 | return 63 | } 64 | mt.workers = append(mt.workers, worker) 65 | } 66 | 67 | //SetWorkers 设置workers, 此操作会覆盖原有的workers 68 | func (mt *Monitor) SetWorkers(workers WorkerList) { 69 | mt.workers = workers 70 | } 71 | 72 | //SetStatus 设置DownloadStatus 73 | func (mt *Monitor) SetStatus(status *transfer.DownloadStatus) { 74 | mt.status = status 75 | } 76 | 77 | //SetInstanceState 设置状态 78 | func (mt *Monitor) SetInstanceState(instanceState *InstanceState) { 79 | mt.instanceState = instanceState 80 | } 81 | 82 | //Status 返回DownloadStatus 83 | func (mt *Monitor) Status() *transfer.DownloadStatus { 84 | return mt.status 85 | } 86 | 87 | //Err 返回遇到的错误 88 | func (mt *Monitor) Err() error { 89 | return mt.err 90 | } 91 | 92 | //CompletedChan 获取completed chan 93 | func (mt *Monitor) CompletedChan() <-chan struct{} { 94 | return mt.completed 95 | } 96 | 97 | //GetAvailableWorker 获取空闲的worker 98 | func (mt *Monitor) GetAvailableWorker() *Worker { 99 | workerCount := len(mt.workers) 100 | for i := mt.lastAvaliableIndex; i < mt.lastAvaliableIndex+workerCount; i++ { 101 | index := i % workerCount 102 | worker := mt.workers[index] 103 | if worker.Completed() { 104 | mt.lastAvaliableIndex = index 105 | return worker 106 | } 107 | } 108 | return nil 109 | } 110 | 111 | //GetAllWorkersRange 获取所有worker的范围 112 | func (mt *Monitor) GetAllWorkersRange() transfer.RangeList { 113 | allWorkerRanges := make(transfer.RangeList, 0, len(mt.workers)) 114 | for _, worker := range mt.workers { 115 | allWorkerRanges = append(allWorkerRanges, worker.GetRange()) 116 | } 117 | return allWorkerRanges 118 | } 119 | 120 | //NumLeftWorkers 剩余的worker数量 121 | func (mt *Monitor) NumLeftWorkers() (num int) { 122 | for _, worker := range mt.workers { 123 | if !worker.Completed() { 124 | num++ 125 | } 126 | } 127 | return 128 | } 129 | 130 | //SetReloadWorker 是否重载worker 131 | func (mt *Monitor) SetReloadWorker(b bool) { 132 | mt.isReloadWorker = b 133 | } 134 | 135 | //IsLeftWorkersAllFailed 剩下的线程是否全部失败 136 | func (mt *Monitor) IsLeftWorkersAllFailed() bool { 137 | failedNum := 0 138 | for _, worker := range mt.workers { 139 | if worker.Completed() { 140 | continue 141 | } 142 | 143 | if !worker.Failed() { 144 | failedNum++ 145 | return false 146 | } 147 | } 148 | return failedNum != 0 149 | } 150 | 151 | //registerAllCompleted 全部完成则发送消息 152 | func (mt *Monitor) registerAllCompleted() { 153 | mt.completed = make(chan struct{}, 0) 154 | var ( 155 | workerNum = len(mt.workers) 156 | completeNum = 0 157 | ) 158 | 159 | go func() { 160 | for { 161 | time.Sleep(1 * time.Second) 162 | 163 | completeNum = 0 164 | for _, worker := range mt.workers { 165 | switch worker.GetStatus().StatusCode() { 166 | case StatusCodeInternalError: 167 | // 检测到内部错误 168 | // 马上停止执行 169 | mt.err = worker.Err() 170 | close(mt.completed) 171 | return 172 | case StatusCodeSuccessed, StatusCodeCanceled: 173 | completeNum++ 174 | } 175 | } 176 | // status 在 lazyInit 之后, 不可能为空 177 | // 完成条件: 所有worker 都已经完成, 且 rangeGen 已生成完毕 178 | gen := mt.status.RangeListGen() 179 | if completeNum >= workerNum && (gen == nil || gen.IsDone()) { // 已完成 180 | close(mt.completed) 181 | return 182 | } 183 | } 184 | }() 185 | } 186 | 187 | //ResetFailedAndNetErrorWorkers 重设部分网络错误的worker 188 | func (mt *Monitor) ResetFailedAndNetErrorWorkers() { 189 | for k := range mt.workers { 190 | if !mt.resetController.CanReset() { 191 | continue 192 | } 193 | 194 | switch mt.workers[k].GetStatus().StatusCode() { 195 | case StatusCodeNetError: 196 | pcsverbose.Verbosef("DEBUG: monitor: ResetFailedAndNetErrorWorkers: reset StatusCodeNetError worker, id: %d\n", mt.workers[k].id) 197 | goto reset 198 | case StatusCodeFailed: 199 | pcsverbose.Verbosef("DEBUG: monitor: ResetFailedAndNetErrorWorkers: reset StatusCodeFailed worker, id: %d\n", mt.workers[k].id) 200 | goto reset 201 | default: 202 | continue 203 | } 204 | 205 | reset: 206 | mt.workers[k].Reset() 207 | mt.resetController.AddResetNum() 208 | } 209 | } 210 | 211 | //RangeWorker 遍历worker 212 | func (mt *Monitor) RangeWorker(f RangeWorkerFunc) { 213 | for k := range mt.workers { 214 | if !f(k, mt.workers[k]) { 215 | break 216 | } 217 | } 218 | } 219 | 220 | //Pause 暂停所有的下载 221 | func (mt *Monitor) Pause() { 222 | for k := range mt.workers { 223 | mt.workers[k].Pause() 224 | } 225 | } 226 | 227 | //Resume 恢复所有的下载 228 | func (mt *Monitor) Resume() { 229 | for k := range mt.workers { 230 | mt.workers[k].Resume() 231 | } 232 | } 233 | 234 | // TryAddNewWork 尝试加入新range 235 | func (mt *Monitor) TryAddNewWork() { 236 | if mt.status == nil { 237 | return 238 | } 239 | gen := mt.status.RangeListGen() 240 | if gen == nil || gen.IsDone() { 241 | return 242 | } 243 | 244 | if !mt.resetController.CanReset() { //能否建立新连接 245 | return 246 | } 247 | 248 | availableWorker := mt.GetAvailableWorker() 249 | if availableWorker == nil { 250 | return 251 | } 252 | 253 | // 有空闲的range, 执行 254 | _, r := gen.GenRange() 255 | if r == nil { 256 | // 没有range了 257 | return 258 | } 259 | 260 | availableWorker.SetRange(r) 261 | availableWorker.ClearStatus() 262 | 263 | mt.resetController.AddResetNum() 264 | pcsverbose.Verbosef("MONITER: worker[%d] add new range: %s\n", availableWorker.ID(), r.ShowDetails()) 265 | go availableWorker.Execute() 266 | } 267 | 268 | // DynamicSplitWorker 动态分配线程 269 | func (mt *Monitor) DynamicSplitWorker(worker *Worker) { 270 | if !mt.resetController.CanReset() { 271 | return 272 | } 273 | 274 | switch worker.status.statusCode { 275 | case StatusCodeDownloading, StatusCodeFailed, StatusCodeNetError: 276 | //pass 277 | default: 278 | return 279 | } 280 | 281 | // 筛选空闲的Worker 282 | availableWorker := mt.GetAvailableWorker() 283 | if availableWorker == nil || worker == availableWorker { // 没有空的 284 | return 285 | } 286 | 287 | workerRange := worker.GetRange() 288 | 289 | end := workerRange.LoadEnd() 290 | middle := (workerRange.LoadBegin() + end) / 2 291 | 292 | if end-middle < MinParallelSize/5 { // 如果线程剩余的下载量太少, 不分配空闲线程 293 | return 294 | } 295 | 296 | // 折半 297 | availableWorkerRange := availableWorker.GetRange() 298 | availableWorkerRange.StoreBegin(middle) // middle不能加1 299 | availableWorkerRange.StoreEnd(end) 300 | availableWorker.ClearStatus() 301 | 302 | workerRange.StoreEnd(middle) 303 | 304 | mt.resetController.AddResetNum() 305 | pcsverbose.Verbosef("MONITOR: worker duplicated: %d <- %d\n", availableWorker.ID(), worker.ID()) 306 | go availableWorker.Execute() 307 | } 308 | 309 | // ResetWorker 重设长时间无响应, 和下载速度为 0 的 Worker 310 | func (mt *Monitor) ResetWorker(worker *Worker) { 311 | if !mt.resetController.CanReset() { //达到最大重载次数 312 | return 313 | } 314 | 315 | if worker.Completed() { 316 | return 317 | } 318 | 319 | // 忽略正在写入数据到硬盘的 320 | // 过滤速度有变化的线程 321 | status := worker.GetStatus() 322 | speeds := worker.GetSpeedsPerSecond() 323 | if speeds != 0 { 324 | return 325 | } 326 | 327 | switch status.StatusCode() { 328 | case StatusCodePending, StatusCodeReseted: 329 | fallthrough 330 | case StatusCodeWaitToWrite: // 正在写入数据 331 | fallthrough 332 | case StatusCodePaused: // 已暂停 333 | // 忽略, 返回 334 | return 335 | } 336 | 337 | mt.resetController.AddResetNum() 338 | 339 | // 重设连接 340 | pcsverbose.Verbosef("MONITOR: worker[%d] reload\n", worker.ID()) 341 | worker.Reset() 342 | } 343 | 344 | //Execute 执行任务 345 | func (mt *Monitor) Execute(cancelCtx context.Context) { 346 | if len(mt.workers) == 0 { 347 | mt.err = ErrNoWokers 348 | return 349 | } 350 | 351 | mt.lazyInit() 352 | for _, worker := range mt.workers { 353 | worker.SetDownloadStatus(mt.status) 354 | go worker.Execute() 355 | } 356 | 357 | mt.registerAllCompleted() // 注册completed 358 | ticker := time.NewTicker(990 * time.Millisecond) 359 | defer ticker.Stop() 360 | 361 | //开始监控 362 | for { 363 | select { 364 | case <-cancelCtx.Done(): 365 | for _, worker := range mt.workers { 366 | err := worker.Cancel() 367 | if err != nil { 368 | pcsverbose.Verbosef("DEBUG: cancel failed, worker id: %d, err: %s\n", worker.ID(), err) 369 | } 370 | } 371 | return 372 | case <-mt.completed: 373 | return 374 | case <-ticker.C: 375 | // 初始化监控工作 376 | mt.ResetFailedAndNetErrorWorkers() 377 | 378 | mt.status.UpdateSpeeds() // 更新速度 379 | 380 | // 保存断点信息到文件 381 | if mt.instanceState != nil { 382 | mt.instanceState.Put(&transfer.DownloadInstanceInfo{ 383 | DownloadStatus: mt.status, 384 | Ranges: mt.GetAllWorkersRange(), 385 | }) 386 | } 387 | 388 | // 加入新range 389 | mt.TryAddNewWork() 390 | 391 | // 不重载worker 392 | if !mt.isReloadWorker { 393 | continue 394 | } 395 | 396 | // 更新maxSpeeds 397 | mt.status.SetMaxSpeeds(mt.status.SpeedsPerSecond()) 398 | 399 | // 速度减慢或者全部失败, 开始监控 400 | // 只有一个worker时不重设连接 401 | isLeftWorkersAllFailed := mt.IsLeftWorkersAllFailed() 402 | if mt.status.SpeedsPerSecond() < mt.status.MaxSpeeds()/6 || isLeftWorkersAllFailed { 403 | if isLeftWorkersAllFailed { 404 | pcsverbose.Verbosef("DEBUG: monitor: All workers failed\n") 405 | } 406 | mt.status.ClearMaxSpeeds() //清空最大速度的统计 407 | 408 | // 先进行动态分配线程 409 | pcsverbose.Verbosef("DEBUG: monitor: start duplicate.\n") 410 | sort.Sort(ByLeftDesc{mt.workers}) 411 | for _, worker := range mt.workers { 412 | //动态分配线程 413 | mt.DynamicSplitWorker(worker) 414 | } 415 | 416 | // 重设长时间无响应, 和下载速度为 0 的线程 417 | pcsverbose.Verbosef("DEBUG: monitor: start reload.\n") 418 | for _, worker := range mt.workers { 419 | mt.ResetWorker(worker) 420 | } 421 | } // end if 2 422 | } //end select 423 | } //end for 424 | } 425 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------