├── .github ├── FUNDING.yml └── workflows │ └── main.yaml ├── .gitignore ├── .goreleaser.yml ├── LICENSE ├── Makefile ├── README.md ├── _testdata ├── resume.tar.gz └── test.tar.gz ├── bin └── .gitkeep ├── client.go ├── cmd └── pget │ └── main.go ├── download.go ├── error.go ├── error_test.go ├── go.mod ├── go.sum ├── io.go ├── option.go ├── parse_test.go ├── pget.go ├── pget_test.go ├── requests.go ├── run_test.go └── util.go /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: Code-Hex 4 | -------------------------------------------------------------------------------- /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - "master" 5 | tags: 6 | - "v*.*.*" 7 | pull_request: 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Setup go 15 | uses: actions/setup-go@v3 16 | with: 17 | go-version: '1.21' 18 | - name: Declare some variables 19 | id: vars 20 | run: | 21 | echo "::set-output name=coverage_txt::${RUNNER_TEMP}/coverage.txt" 22 | - name: Test Coverage (pkg) 23 | run: go test ./... -coverprofile=${{ steps.vars.outputs.coverage_txt }} 24 | - name: Upload coverage 25 | uses: codecov/codecov-action@v3 26 | with: 27 | files: ${{ steps.vars.outputs.coverage_txt }} 28 | - name: Run GoReleaser 29 | if: contains(github.ref, 'tags/v') 30 | uses: goreleaser/goreleaser-action@v2 31 | with: 32 | version: latest 33 | args: release --clean 34 | env: 35 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 36 | HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }} 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | !.gitignore 3 | !.travis.yml 4 | !.github 5 | !.goreleaser.yml 6 | vendor 7 | bin/* 8 | !bin/.gitkeep -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | builds: 2 | - id: pget 3 | main: ./cmd/pget/main.go 4 | binary: pget 5 | env: 6 | - CGO_ENABLED=0 7 | ldflags: -s -w -X main.version={{.Version}} 8 | goos: 9 | - linux 10 | - darwin 11 | - windows 12 | goarch: 13 | - "386" 14 | - amd64 15 | - arm 16 | - arm64 17 | goarm: 18 | - "6" 19 | - "7" 20 | ignore: 21 | - goos: darwin 22 | goarch: "386" 23 | - goos: linux 24 | goarch: arm 25 | goarm: "7" 26 | - goos: windows 27 | goarch: arm 28 | goarm: "7" 29 | 30 | archives: 31 | - builds: 32 | - pget 33 | name_template: >- 34 | pget_ 35 | {{ .Version }}_ 36 | {{- if eq .Os "darwin" }}macOS 37 | {{- else if eq .Os "linux" }}Linux 38 | {{- else if eq .Os "windows" }}Windows 39 | {{- else }}{{ title .Os }}{{ end }}_ 40 | {{- if eq .Arch "amd64" }}x86_64 41 | {{- else if eq .Arch "386" }}i386 42 | {{- else }}{{ .Arch }}{{ end }} 43 | format_overrides: 44 | - goos: windows 45 | format: zip 46 | files: 47 | - LICENSE 48 | 49 | brews: 50 | - name: pget 51 | repository: 52 | owner: Code-Hex 53 | name: homebrew-tap 54 | token: "{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}" 55 | homepage: https://github.com/Code-Hex/pget 56 | description: The fastest file download client 57 | folder: Formula 58 | install: | 59 | bin.install "pget" 60 | nfpms: 61 | - license: MIT License 62 | maintainer: Kei Kamikawa 63 | homepage: https://github.com/Code-Hex/pget 64 | bindir: /usr/local/bin 65 | description: The fastest file download client 66 | formats: 67 | - apk 68 | - deb 69 | - rpm 70 | 71 | checksum: 72 | name_template: 'pget_checksums.txt' 73 | 74 | changelog: 75 | sort: asc 76 | filters: 77 | exclude: 78 | - '^docs:' 79 | - '^test:' 80 | - Merge pull request 81 | - Merge branch -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 K(Code-Hex) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GIT_REF := $(shell git describe --always --tag) 2 | VERSION ?= $(GIT_REF) 3 | 4 | .PHONY: clean 5 | build: 6 | go build -o ./bin/pget -trimpath -ldflags "-w -s -X main.version=$(VERSION)" -mod=readonly ./cmd/pget -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Pget - The fastest file download client 2 | ======= 3 | 4 | [![.github/workflows/main.yaml](https://github.com/Code-Hex/pget/actions/workflows/main.yaml/badge.svg)](https://github.com/Code-Hex/pget/actions/workflows/main.yaml) 5 | [![codecov](https://codecov.io/gh/Code-Hex/pget/branch/master/graph/badge.svg?token=jUVGnY7ZlG)](undefined) 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/Code-Hex/pget)](https://goreportcard.com/report/github.com/Code-Hex/pget) 7 | [![GitHub release](https://img.shields.io/github/release/Code-Hex/pget.svg)](https://github.com/Code-Hex/pget) 8 | [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE) 9 | 10 | **Ad**: I'm currently developing a new date and time library [synchro](https://github.com/Code-Hex/synchro) for the modern era. please give it ⭐!! 11 | 12 | ## Description 13 | 14 | Multi-Connection Download using parallel requests. 15 | 16 | - Fast 17 | - Resumable 18 | - Cross-compiled (windows, linux, macOS) 19 | 20 | This is an example to download [linux kernel](https://www.kernel.org/). It will be finished between 15s. 21 | 22 | ![pget](https://user-images.githubusercontent.com/6500104/147878414-321c57ad-cff2-40f3-b2a4-12c30ff1363f.gif) 23 | 24 | 25 | ## Disclaimer 26 | 27 | This program comes with no warranty. You must use this program at your own risk. 28 | 29 | ### Note 30 | 31 | - Using a large number of connections to a single URL can lead to DOS attacks. 32 | - The case is increasing that if you use multiple connections to 1 URL does not increase the download speed with the spread of CDNs. 33 | - I recommend to use multiple mirrors simultaneously for faster downloads (And the number of connections is 1 for each). 34 | 35 | ## Installation 36 | 37 | ### Homebrew 38 | 39 | $ brew install pget 40 | 41 | ### Go 42 | 43 | $ go install github.com/Code-Hex/pget/cmd/pget@latest 44 | 45 | ## Synopsis 46 | 47 | This example will be used 2 connections per URL. 48 | 49 | $ pget -p 2 MIRROR1 MIRROR2 MIRROR3 50 | 51 | If you have created such as this file 52 | 53 | cat list.txt 54 | MIRROR1 55 | MIRROR2 56 | MIRROR3 57 | 58 | You can do this 59 | 60 | cat list.txt | pget -p 2 61 | 62 | ## Options 63 | 64 | ``` 65 | Options: 66 | -h, --help print usage and exit 67 | -p, --procs the number of connections for a single URL (default 1) 68 | -o, --output output file to 69 | -t, --timeout timeout of checking request in seconds 70 | -u, --user-agent identify as 71 | -r, --referer identify as 72 | --check-update check if there is update available 73 | --trace display detail error messages 74 | ``` 75 | 76 | ## Pget vs Wget 77 | 78 | URL: https://mirror.internet.asn.au/pub/ubuntu/releases/21.10/ubuntu-21.10-desktop-amd64.iso 79 | 80 | Using 81 | ``` 82 | time wget https://mirror.internet.asn.au/pub/ubuntu/releases/21.10/ubuntu-21.10-desktop-amd64.iso 83 | time pget -p 6 https://mirror.internet.asn.au/pub/ubuntu/releases/21.10/ubuntu-21.10-desktop-amd64.iso 84 | ``` 85 | Results 86 | 87 | ``` 88 | wget 3.92s user 23.52s system 3% cpu 13:35.24 total 89 | pget -p 6 10.54s user 34.52s system 25% cpu 2:56.93 total 90 | ``` 91 | 92 | `wget` 13:35.24 total, `pget -p 6` **2:56.93 total (6x faster)** 93 | 94 | ## Binary 95 | 96 | You can download from [here](https://github.com/Code-Hex/pget/releases) 97 | 98 | ## Author 99 | 100 | [codehex](https://twitter.com/CodeHex) 101 | -------------------------------------------------------------------------------- /_testdata/resume.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Code-Hex/pget/7cb7dea95a0f9b510dc7a64e8c1f521178734e6f/_testdata/resume.tar.gz -------------------------------------------------------------------------------- /_testdata/test.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Code-Hex/pget/7cb7dea95a0f9b510dc7a64e8c1f521178734e6f/_testdata/test.tar.gz -------------------------------------------------------------------------------- /bin/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Code-Hex/pget/7cb7dea95a0f9b510dc7a64e8c1f521178734e6f/bin/.gitkeep -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package pget 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "net/http" 7 | "runtime" 8 | "time" 9 | ) 10 | 11 | func newDownloadClient(maxIdleConnsPerHost int) *http.Client { 12 | tr := http.DefaultTransport.(*http.Transport).Clone() 13 | dialer := newDialRateLimiter(&net.Dialer{ 14 | Timeout: 30 * time.Second, 15 | KeepAlive: 30 * time.Second, 16 | }) 17 | tr.DialContext = dialer.DialContext 18 | tr.MaxIdleConns = 0 // no limit 19 | tr.MaxIdleConnsPerHost = maxIdleConnsPerHost 20 | tr.DisableCompression = true 21 | return &http.Client{ 22 | Transport: tr, 23 | } 24 | } 25 | 26 | func newClient(client *http.Client) *http.Client { 27 | if client == nil { 28 | return http.DefaultClient 29 | } 30 | return client 31 | } 32 | 33 | // Prevents too many dials happening at once, because we've observed that that increases the thread 34 | // count in the app, to several times more than is actually necessary - presumably due to a blocking OS 35 | // call somewhere. It's tidier to avoid creating those excess OS threads. 36 | // Even our change from Dial (deprecated) to DialContext did not replicate the effect of dialRateLimiter. 37 | // 38 | // see: https://github.com/Azure/azure-storage-azcopy/blob/058bd5bc5b970074520e4ee088b15328d888c483/ste/mgr-JobPartMgr.go#L117-L124 39 | type dialRateLimiter struct { 40 | dialer *net.Dialer 41 | sem chan struct{} 42 | } 43 | 44 | func newDialRateLimiter(dialer *net.Dialer) *dialRateLimiter { 45 | // exact value doesn't matter too much, but too low will be too slow, 46 | // and too high will reduce the beneficial effect on thread count 47 | const concurrentDialsPerCpu = 10 48 | 49 | return &dialRateLimiter{ 50 | dialer: dialer, 51 | sem: make(chan struct{}, concurrentDialsPerCpu*runtime.NumCPU()), 52 | } 53 | } 54 | 55 | func (d *dialRateLimiter) DialContext(ctx context.Context, network, address string) (net.Conn, error) { 56 | d.sem <- struct{}{} 57 | defer func() { <-d.sem }() 58 | return d.dialer.DialContext(ctx, network, address) 59 | } 60 | -------------------------------------------------------------------------------- /cmd/pget/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/Code-Hex/pget" 9 | ) 10 | 11 | var version string 12 | 13 | func main() { 14 | cli := pget.New() 15 | if err := cli.Run(context.Background(), version, os.Args[1:]); err != nil { 16 | if cli.Trace { 17 | fmt.Fprintf(os.Stderr, "Error:\n%+v\n", err) 18 | } else { 19 | fmt.Fprintf(os.Stderr, "Error:\n %v\n", err) 20 | } 21 | os.Exit(1) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /download.go: -------------------------------------------------------------------------------- 1 | package pget 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "os" 9 | "path/filepath" 10 | 11 | "github.com/cheggaaa/pb/v3" 12 | "github.com/pkg/errors" 13 | "golang.org/x/sync/errgroup" 14 | ) 15 | 16 | type assignTasksConfig struct { 17 | Procs int 18 | TaskSize int64 // download filesize per task 19 | ContentLength int64 // full download filesize 20 | URLs []string 21 | PartialDir string 22 | Filename string 23 | Client *http.Client 24 | } 25 | 26 | type task struct { 27 | ID int 28 | Procs int 29 | URL string 30 | Range Range 31 | PartialDir string 32 | Filename string 33 | Client *http.Client 34 | } 35 | 36 | func (t *task) destPath() string { 37 | return getPartialFilePath(t.PartialDir, t.Filename, t.Procs, t.ID) 38 | } 39 | 40 | func (t *task) String() string { 41 | return fmt.Sprintf("task[%d]: %q", t.ID, t.destPath()) 42 | } 43 | 44 | type makeRequestOption struct { 45 | useragent string 46 | referer string 47 | } 48 | 49 | func (t *task) makeRequest(ctx context.Context, opt *makeRequestOption) (*http.Request, error) { 50 | req, err := http.NewRequest("GET", t.URL, nil) 51 | if err != nil { 52 | return nil, errors.Wrap(err, fmt.Sprintf("failed to make a new request: %d", t.ID)) 53 | } 54 | req = req.WithContext(ctx) 55 | 56 | // set download ranges 57 | req.Header.Set("Range", t.Range.BytesRange()) 58 | 59 | // set useragent 60 | req.Header.Set("User-Agent", opt.useragent) 61 | 62 | // set referer 63 | if opt.referer != "" { 64 | req.Header.Set("Referer", opt.referer) 65 | } 66 | 67 | return req, nil 68 | } 69 | 70 | // assignTasks creates task to assign it to each goroutines 71 | func assignTasks(c *assignTasksConfig) []*task { 72 | tasks := make([]*task, 0, c.Procs) 73 | 74 | var totalActiveProcs int 75 | for i := 0; i < c.Procs; i++ { 76 | 77 | r := makeRange(i, c.Procs, c.TaskSize, c.ContentLength) 78 | 79 | partName := getPartialFilePath(c.PartialDir, c.Filename, c.Procs, i) 80 | 81 | if info, err := os.Stat(partName); err == nil { 82 | infosize := info.Size() 83 | // check if the part is fully downloaded 84 | if i == c.Procs-1 { 85 | if infosize == r.high-r.low { 86 | continue 87 | } 88 | } else if infosize == c.TaskSize { 89 | // skip as the part is already downloaded 90 | continue 91 | } 92 | 93 | // make low range from this next byte 94 | r.low += infosize 95 | } 96 | 97 | tasks = append(tasks, &task{ 98 | ID: i, 99 | Procs: c.Procs, 100 | URL: c.URLs[totalActiveProcs%len(c.URLs)], 101 | Range: r, 102 | PartialDir: c.PartialDir, 103 | Filename: c.Filename, 104 | Client: c.Client, 105 | }) 106 | 107 | totalActiveProcs++ 108 | } 109 | 110 | return tasks 111 | } 112 | 113 | type DownloadConfig struct { 114 | Filename string 115 | Dirname string 116 | ContentLength int64 117 | Procs int 118 | URLs []string 119 | Client *http.Client 120 | 121 | *makeRequestOption 122 | } 123 | 124 | type DownloadOption func(c *DownloadConfig) 125 | 126 | func WithUserAgent(ua, version string) DownloadOption { 127 | return func(c *DownloadConfig) { 128 | if ua == "" { 129 | ua = "Pget/" + version 130 | } 131 | c.makeRequestOption.useragent = ua 132 | } 133 | } 134 | 135 | func WithReferer(referer string) DownloadOption { 136 | return func(c *DownloadConfig) { 137 | c.makeRequestOption.referer = referer 138 | } 139 | } 140 | 141 | func Download(ctx context.Context, c *DownloadConfig, opts ...DownloadOption) error { 142 | partialDir := getPartialDirname(c.Dirname, c.Filename, c.Procs) 143 | 144 | // create download location 145 | if err := os.MkdirAll(partialDir, 0755); err != nil { 146 | return errors.Wrap(err, "failed to mkdir for download location") 147 | } 148 | 149 | c.makeRequestOption = &makeRequestOption{} 150 | 151 | for _, opt := range opts { 152 | opt(c) 153 | } 154 | 155 | tasks := assignTasks(&assignTasksConfig{ 156 | Procs: c.Procs, 157 | TaskSize: c.ContentLength / int64(c.Procs), 158 | ContentLength: c.ContentLength, 159 | URLs: c.URLs, 160 | PartialDir: partialDir, 161 | Filename: c.Filename, 162 | Client: newClient(c.Client), 163 | }) 164 | 165 | if err := parallelDownload(ctx, ¶llelDownloadConfig{ 166 | ContentLength: c.ContentLength, 167 | Tasks: tasks, 168 | PartialDir: partialDir, 169 | makeRequestOption: c.makeRequestOption, 170 | }); err != nil { 171 | return err 172 | } 173 | 174 | return bindFiles(c, partialDir) 175 | } 176 | 177 | type parallelDownloadConfig struct { 178 | ContentLength int64 179 | Tasks []*task 180 | PartialDir string 181 | *makeRequestOption 182 | } 183 | 184 | func parallelDownload(ctx context.Context, c *parallelDownloadConfig) error { 185 | eg, ctx := errgroup.WithContext(ctx) 186 | 187 | bar := pb.Start64(c.ContentLength).SetWriter(stdout).Set(pb.Bytes, true) 188 | defer bar.Finish() 189 | 190 | // check file size already downloaded for resume 191 | size, err := checkProgress(c.PartialDir) 192 | if err != nil { 193 | return errors.Wrap(err, "failed to get directory size") 194 | } 195 | 196 | bar.SetCurrent(size) 197 | 198 | for _, task := range c.Tasks { 199 | task := task 200 | eg.Go(func() error { 201 | req, err := task.makeRequest(ctx, c.makeRequestOption) 202 | if err != nil { 203 | return err 204 | } 205 | return task.download(req, bar) 206 | }) 207 | } 208 | 209 | return eg.Wait() 210 | } 211 | 212 | func (t *task) download(req *http.Request, bar *pb.ProgressBar) error { 213 | resp, err := t.Client.Do(req) 214 | if err != nil { 215 | return errors.Wrapf(err, "failed to get response: %q", t.String()) 216 | } 217 | defer resp.Body.Close() 218 | 219 | output, err := os.OpenFile(t.destPath(), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 220 | if err != nil { 221 | return errors.Wrapf(err, "failed to create: %q", t.String()) 222 | } 223 | defer output.Close() 224 | 225 | rd := bar.NewProxyReader(resp.Body) 226 | 227 | if _, err := io.Copy(output, rd); err != nil { 228 | return errors.Wrapf(err, "failed to write response body: %q", t.String()) 229 | } 230 | 231 | return nil 232 | } 233 | 234 | func bindFiles(c *DownloadConfig, partialDir string) error { 235 | fmt.Fprintln(stdout, "\nbinding with files...") 236 | 237 | destPath := filepath.Join(c.Dirname, c.Filename) 238 | f, err := os.Create(destPath) 239 | if err != nil { 240 | return errors.Wrap(err, "failed to create a file in download location") 241 | } 242 | defer f.Close() 243 | 244 | bar := pb.Start64(c.ContentLength).SetWriter(stdout) 245 | 246 | copyFn := func(name string) error { 247 | subfp, err := os.Open(name) 248 | if err != nil { 249 | return errors.Wrapf(err, "failed to open %q in download location", name) 250 | } 251 | 252 | defer subfp.Close() 253 | 254 | proxy := bar.NewProxyReader(subfp) 255 | if _, err := io.Copy(f, proxy); err != nil { 256 | return errors.Wrapf(err, "failed to copy %q", name) 257 | } 258 | 259 | return nil 260 | } 261 | 262 | for i := 0; i < c.Procs; i++ { 263 | partialFilename := getPartialFilePath(partialDir, c.Filename, c.Procs, i) 264 | if err := copyFn(partialFilename); err != nil { 265 | return err 266 | } 267 | 268 | // remove a file in download location for join 269 | if err := os.Remove(partialFilename); err != nil { 270 | return errors.Wrapf(err, "failed to remove %q in download location", partialFilename) 271 | } 272 | } 273 | 274 | bar.Finish() 275 | 276 | // remove download location 277 | // RemoveAll reason: will create .DS_Store in download location if execute on mac 278 | if err := os.RemoveAll(partialDir); err != nil { 279 | return errors.Wrap(err, "failed to remove download location") 280 | } 281 | 282 | return nil 283 | } 284 | -------------------------------------------------------------------------------- /error.go: -------------------------------------------------------------------------------- 1 | package pget 2 | 3 | import "github.com/pkg/errors" 4 | 5 | type causer interface { 6 | Cause() error 7 | } 8 | 9 | type ignore struct { 10 | err error 11 | } 12 | 13 | func makeIgnoreErr() ignore { 14 | return ignore{ 15 | err: errors.New("this is ignore message"), 16 | } 17 | } 18 | 19 | // Error for options: version, usage 20 | func (i ignore) Error() string { 21 | return i.err.Error() 22 | } 23 | 24 | func (i ignore) Cause() error { 25 | return i.err 26 | } 27 | 28 | // errTop get important message from wrapped error message 29 | func errTop(err error) error { 30 | for e := err; e != nil; { 31 | switch e.(type) { 32 | case ignore: 33 | return nil 34 | case causer: 35 | e = e.(causer).Cause() 36 | default: 37 | return e 38 | } 39 | } 40 | 41 | return nil 42 | } 43 | -------------------------------------------------------------------------------- /error_test.go: -------------------------------------------------------------------------------- 1 | package pget 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | func TestErrors(t *testing.T) { 10 | err := errors.New("first") 11 | err = errors.Wrap(err, "second") 12 | err = errors.Wrap(err, "third") 13 | 14 | err = errTop(err) 15 | if err.Error() != "first" { 16 | t.Errorf("could not get top message") 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Code-Hex/pget 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/Code-Hex/updater v0.0.0-20160712085121-c3f278672520 7 | github.com/Songmu/prompter v0.5.0 8 | github.com/asaskevich/govalidator v0.0.0-20161001163130-7b3beb6df3c4 9 | github.com/cheggaaa/pb/v3 v3.0.8 10 | github.com/jessevdk/go-flags v0.0.0-20160903113131-4cc2832a6e6d 11 | github.com/mholt/archiver v3.1.1+incompatible 12 | github.com/pkg/errors v0.8.1-0.20161002052512-839d9e913e06 13 | github.com/stretchr/testify v1.6.1 14 | golang.org/x/sync v0.0.0-20161004233620-1ae7c7b29e06 15 | ) 16 | 17 | require ( 18 | github.com/VividCortex/ewma v1.1.1 // indirect 19 | github.com/antonholmquist/jason v1.0.1-0.20160829104012-962e09b85496 // indirect 20 | github.com/davecgh/go-spew v1.1.0 // indirect 21 | github.com/dsnet/compress v0.0.1 // indirect 22 | github.com/fatih/color v1.10.0 // indirect 23 | github.com/frankban/quicktest v1.11.1 // indirect 24 | github.com/golang/snappy v0.0.2 // indirect 25 | github.com/mattn/go-colorable v0.1.8 // indirect 26 | github.com/mattn/go-isatty v0.0.12 // indirect 27 | github.com/mattn/go-runewidth v0.0.12 // indirect 28 | github.com/mcuadros/go-version v0.0.0-20141206211339-d52711f8d6be // indirect 29 | github.com/nwaples/rardecode v1.1.0 // indirect 30 | github.com/pierrec/lz4 v2.5.2+incompatible // indirect 31 | github.com/pmezard/go-difflib v1.0.0 // indirect 32 | github.com/rivo/uniseg v0.2.0 // indirect 33 | github.com/ulikunitz/xz v0.5.8 // indirect 34 | github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect 35 | golang.org/x/net v0.0.0-20161013035702-8b4af36cd21a // indirect 36 | golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 // indirect 37 | golang.org/x/term v0.0.0-20210317153231-de623e64d2a6 // indirect 38 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect 39 | ) 40 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Code-Hex/updater v0.0.0-20160712085121-c3f278672520 h1:AhI5ytq4dAam2scBpgeQY/9kz/covK9/NMyzO3e8350= 2 | github.com/Code-Hex/updater v0.0.0-20160712085121-c3f278672520/go.mod h1:RZRMRhdqo/22EdcyGiDJdIdCrptsRDEbqQ8/bswHV1E= 3 | github.com/Songmu/prompter v0.5.0 h1:uf60xlFItY5nW+rlLJ2XIUfaUReo4gUEeftuUeHpio8= 4 | github.com/Songmu/prompter v0.5.0/go.mod h1:S4Eg25l60kPlnfB2ttFVpvBKYw7RKJexzB3gzpAansY= 5 | github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= 6 | github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= 7 | github.com/antonholmquist/jason v1.0.1-0.20160829104012-962e09b85496 h1:dESITdufxuiwgQh1YPiPupEXORHTYvY8tr40nvrWelo= 8 | github.com/antonholmquist/jason v1.0.1-0.20160829104012-962e09b85496/go.mod h1:+GxMEKI0Va2U8h3os6oiUAetHAlGMvxjdpAH/9uvUMA= 9 | github.com/asaskevich/govalidator v0.0.0-20161001163130-7b3beb6df3c4 h1:roUAANycAr9TS5tnrZboqlI+bGfcY8n9nDyD1WDgn74= 10 | github.com/asaskevich/govalidator v0.0.0-20161001163130-7b3beb6df3c4/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= 11 | github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA= 12 | github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= 13 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 14 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 15 | github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= 16 | github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= 17 | github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= 18 | github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= 19 | github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= 20 | github.com/frankban/quicktest v1.11.1 h1:stwUsXhUGliQs9t0ZS39BWCltFdOHgABiIlihop8AD4= 21 | github.com/frankban/quicktest v1.11.1/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= 22 | github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw= 23 | github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 24 | github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= 25 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 26 | github.com/jessevdk/go-flags v0.0.0-20160903113131-4cc2832a6e6d h1:i6fERqEEy9HDP6qIg93orNgisqEFTzR+U5pzavIAAhs= 27 | github.com/jessevdk/go-flags v0.0.0-20160903113131-4cc2832a6e6d/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 28 | github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 29 | github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 30 | github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= 31 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 32 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 33 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 34 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 35 | github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= 36 | github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 37 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 38 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 39 | github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow= 40 | github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= 41 | github.com/mcuadros/go-version v0.0.0-20141206211339-d52711f8d6be h1:tj81VrKAa9Vv71Ugze2mtnOiU2ozpJau8itbf3XP3fo= 42 | github.com/mcuadros/go-version v0.0.0-20141206211339-d52711f8d6be/go.mod h1:76rfSfYPWj01Z85hUf/ituArm797mNKcvINh1OlsZKo= 43 | github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= 44 | github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= 45 | github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= 46 | github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= 47 | github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= 48 | github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 49 | github.com/pkg/errors v0.8.1-0.20161002052512-839d9e913e06 h1:swlfMC08lNw0gC4UR7Xz9id7JZ3K+I1oEehKgRL399U= 50 | github.com/pkg/errors v0.8.1-0.20161002052512-839d9e913e06/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 51 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 52 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 53 | github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 54 | github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= 55 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 56 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 57 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 58 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 59 | github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= 60 | github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= 61 | github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= 62 | github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= 63 | github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= 64 | golang.org/x/net v0.0.0-20161013035702-8b4af36cd21a h1:YEFEcqrj8fWeC0px2Ha5IrK20xodii3wn+N+jzuFRKQ= 65 | golang.org/x/net v0.0.0-20161013035702-8b4af36cd21a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 66 | golang.org/x/sync v0.0.0-20161004233620-1ae7c7b29e06 h1:pRVhPB331E/b1+A7Y9d/3ZkgE5LNxnP/q5ChiqPf79Q= 67 | golang.org/x/sync v0.0.0-20161004233620-1ae7c7b29e06/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 68 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 69 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 70 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 71 | golang.org/x/sys v0.0.0-20210319071255-635bc2c9138d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 72 | golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A= 73 | golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 74 | golang.org/x/term v0.0.0-20210317153231-de623e64d2a6 h1:EC6+IGYTjPpRfv9a2b/6Puw0W+hLtAhkV1tPsXhutqs= 75 | golang.org/x/term v0.0.0-20210317153231-de623e64d2a6/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 76 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 77 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 78 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 79 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 80 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 81 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 82 | -------------------------------------------------------------------------------- /io.go: -------------------------------------------------------------------------------- 1 | package pget 2 | 3 | import ( 4 | "io" 5 | "os" 6 | ) 7 | 8 | var stdout io.Writer = os.Stdout 9 | -------------------------------------------------------------------------------- /option.go: -------------------------------------------------------------------------------- 1 | package pget 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "github.com/Code-Hex/updater" 8 | "github.com/jessevdk/go-flags" 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | // Options struct for parse command line arguments 13 | type Options struct { 14 | Help bool `short:"h" long:"help"` 15 | NumConnection int `short:"p" long:"procs" default:"1"` 16 | Output string `short:"o" long:"output"` 17 | Timeout int `short:"t" long:"timeout" default:"10"` 18 | UserAgent string `short:"u" long:"user-agent"` 19 | Referer string `short:"r" long:"referer"` 20 | Update bool `long:"check-update"` 21 | Trace bool `long:"trace"` 22 | } 23 | 24 | func (opts *Options) parse(argv []string, version string) ([]string, error) { 25 | p := flags.NewParser(opts, flags.PrintErrors) 26 | args, err := p.ParseArgs(argv) 27 | 28 | if err != nil { 29 | stdout.Write(opts.usage(version)) 30 | return nil, errors.Wrap(err, "invalid command line options") 31 | } 32 | 33 | return args, nil 34 | } 35 | 36 | func (opts Options) usage(version string) []byte { 37 | buf := bytes.Buffer{} 38 | 39 | msg := "Pget %s, The fastest file download client\n" 40 | fmt.Fprintf(&buf, msg+ 41 | `Usage: pget [options] URL 42 | Options: 43 | -h, --help print usage and exit 44 | -p, --procs the number of connections for a single URL (default 1) 45 | -o, --output output file to 46 | -t, --timeout timeout of checking request in seconds (default 10s) 47 | -u, --user-agent identify as 48 | -r, --referer identify as 49 | --check-update check if there is update available 50 | --trace display detail error messages 51 | `, version) 52 | return buf.Bytes() 53 | } 54 | 55 | func (opts Options) isupdate(version string) ([]byte, error) { 56 | buf := bytes.Buffer{} 57 | result, err := updater.CheckWithTag("Code-Hex", "pget", version) 58 | if err != nil { 59 | return nil, err 60 | } 61 | fmt.Fprintf(&buf, result+"\n") 62 | 63 | return buf.Bytes(), nil 64 | } 65 | -------------------------------------------------------------------------------- /parse_test.go: -------------------------------------------------------------------------------- 1 | package pget 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | const version = "test_version" 10 | 11 | func TestParts_of_ready(t *testing.T) { 12 | cases := []struct { 13 | name string 14 | args []string 15 | wantProcs int 16 | wantURLs int 17 | }{ 18 | { 19 | name: "one URL", 20 | args: []string{ 21 | "pget", 22 | "-p", 23 | "2", 24 | "http://example.com/filename.tar.gz", 25 | "--trace", 26 | "--output", 27 | "filename.tar.gz", 28 | }, 29 | wantProcs: 2, 30 | wantURLs: 1, 31 | }, 32 | { 33 | name: "two URLs", 34 | args: []string{ 35 | "pget", 36 | "-p", 37 | "2", 38 | "http://example.com/filename.tar.gz", 39 | "http://example2.com/filename.tar.gz", 40 | "--trace", 41 | "--output", 42 | "filename.tar.gz", 43 | }, 44 | wantProcs: 4, 45 | wantURLs: 2, 46 | }, 47 | } 48 | for _, tc := range cases { 49 | tc := tc 50 | t.Run(tc.name, func(t *testing.T) { 51 | p := New() 52 | 53 | if err := p.Ready(version, tc.args); err != nil { 54 | t.Errorf("failed to parse command line args: %s", err) 55 | } 56 | 57 | assert.Equal(t, true, p.Trace, "failed to parse arguments of trace") 58 | assert.Equal(t, tc.wantProcs, p.Procs, "failed to parse arguments of procs") 59 | assert.Equal(t, "filename.tar.gz", p.Output, "failed to parse output") 60 | 61 | assert.Len(t, p.URLs, tc.wantURLs) 62 | }) 63 | } 64 | } 65 | 66 | func TestShowhelp(t *testing.T) { 67 | args := []string{ 68 | "pget", 69 | "-h", 70 | } 71 | 72 | p := New() 73 | _, err := p.parseOptions(args, version) 74 | assert.NotNil(t, err) 75 | 76 | args = []string{ 77 | "pget", 78 | "--help", 79 | } 80 | 81 | p = New() 82 | _, err = p.parseOptions(args, version) 83 | assert.NotNil(t, err) 84 | } 85 | 86 | func TestShowisupdate(t *testing.T) { 87 | args := []string{ 88 | "pget", 89 | "--check-update", 90 | } 91 | 92 | p := New() 93 | _, err := p.parseOptions(args, version) 94 | assert.NotNil(t, err) 95 | } 96 | -------------------------------------------------------------------------------- /pget.go: -------------------------------------------------------------------------------- 1 | package pget 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "runtime" 10 | "strings" 11 | "time" 12 | 13 | "github.com/Songmu/prompter" 14 | "github.com/asaskevich/govalidator" 15 | "github.com/pkg/errors" 16 | ) 17 | 18 | // Pget structs 19 | type Pget struct { 20 | Trace bool 21 | Output string 22 | Procs int 23 | URLs []string 24 | 25 | args []string 26 | timeout int 27 | useragent string 28 | referer string 29 | } 30 | 31 | // New for pget package 32 | func New() *Pget { 33 | return &Pget{ 34 | Trace: false, 35 | Procs: runtime.NumCPU(), // default 36 | timeout: 10, 37 | } 38 | } 39 | 40 | // Run execute methods in pget package 41 | func (pget *Pget) Run(ctx context.Context, version string, args []string) error { 42 | if err := pget.Ready(version, args); err != nil { 43 | return errTop(err) 44 | } 45 | 46 | // TODO(codehex): calc maxIdleConnsPerHost 47 | client := newDownloadClient(16) 48 | 49 | target, err := Check(ctx, &CheckConfig{ 50 | URLs: pget.URLs, 51 | Timeout: time.Duration(pget.timeout) * time.Second, 52 | Client: client, 53 | }) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | filename := target.Filename 59 | 60 | var dir string 61 | if pget.Output != "" { 62 | fi, err := os.Stat(pget.Output) 63 | if err == nil && fi.IsDir() { 64 | dir = pget.Output 65 | } else { 66 | dir, filename = filepath.Split(pget.Output) 67 | if dir != "" { 68 | if err := os.MkdirAll(dir, 0755); err != nil { 69 | return errors.Wrapf(err, "failed to create diretory at %s", dir) 70 | } 71 | } 72 | } 73 | } 74 | 75 | opts := []DownloadOption{ 76 | WithUserAgent(pget.useragent, version), 77 | WithReferer(pget.referer), 78 | } 79 | 80 | return Download(ctx, &DownloadConfig{ 81 | Filename: filename, 82 | Dirname: dir, 83 | ContentLength: target.ContentLength, 84 | Procs: pget.Procs, 85 | URLs: target.URLs, 86 | Client: client, 87 | }, opts...) 88 | } 89 | 90 | const ( 91 | warningNumConnection = 4 92 | warningMessage = "[WARNING] Using a large number of connections to 1 URL can lead to DOS attacks.\n" + 93 | "In most cases, `4` or less is enough. In addition, the case is increasing that if you use multiple connections to 1 URL does not increase the download speed with the spread of CDNs.\n" + 94 | "See: https://github.com/Code-Hex/pget#disclaimer\n" + 95 | "\n" + 96 | "Would you execute knowing these?\n" 97 | ) 98 | 99 | // Ready method define the variables required to Download. 100 | func (pget *Pget) Ready(version string, args []string) error { 101 | opts, err := pget.parseOptions(args, version) 102 | if err != nil { 103 | return errors.Wrap(errTop(err), "failed to parse command line args") 104 | } 105 | 106 | if opts.Trace { 107 | pget.Trace = opts.Trace 108 | } 109 | 110 | if opts.Timeout > 0 { 111 | pget.timeout = opts.Timeout 112 | } 113 | 114 | if err := pget.parseURLs(); err != nil { 115 | return errors.Wrap(err, "failed to parse of url") 116 | } 117 | 118 | if opts.NumConnection > warningNumConnection && !prompter.YN(warningMessage, false) { 119 | return makeIgnoreErr() 120 | } 121 | 122 | pget.Procs = opts.NumConnection * len(pget.URLs) 123 | 124 | if opts.Output != "" { 125 | pget.Output = opts.Output 126 | } 127 | 128 | if opts.UserAgent != "" { 129 | pget.useragent = opts.UserAgent 130 | } 131 | 132 | if opts.Referer != "" { 133 | pget.referer = opts.Referer 134 | } 135 | 136 | return nil 137 | } 138 | 139 | func (pget *Pget) parseOptions(argv []string, version string) (*Options, error) { 140 | var opts Options 141 | if len(argv) == 0 { 142 | stdout.Write(opts.usage(version)) 143 | return nil, makeIgnoreErr() 144 | } 145 | 146 | o, err := opts.parse(argv, version) 147 | if err != nil { 148 | return nil, errors.Wrap(err, "failed to parse command line options") 149 | } 150 | 151 | if opts.Help { 152 | stdout.Write(opts.usage(version)) 153 | return nil, makeIgnoreErr() 154 | } 155 | 156 | if opts.Update { 157 | result, err := opts.isupdate(version) 158 | if err != nil { 159 | return nil, errors.Wrap(err, "failed to parse command line options") 160 | } 161 | 162 | stdout.Write(result) 163 | return nil, makeIgnoreErr() 164 | } 165 | 166 | pget.args = o 167 | 168 | return &opts, nil 169 | } 170 | 171 | func (pget *Pget) parseURLs() error { 172 | 173 | // find url in args 174 | for _, argv := range pget.args { 175 | if govalidator.IsURL(argv) { 176 | pget.URLs = append(pget.URLs, argv) 177 | } 178 | } 179 | 180 | if len(pget.URLs) < 1 { 181 | fmt.Fprintf(stdout, "Please input url separate with space or newline\n") 182 | fmt.Fprintf(stdout, "Start download with ^D\n") 183 | 184 | // scanning url from stdin 185 | scanner := bufio.NewScanner(os.Stdin) 186 | for scanner.Scan() { 187 | scan := scanner.Text() 188 | urls := strings.Split(scan, " ") 189 | for _, url := range urls { 190 | if govalidator.IsURL(url) { 191 | pget.URLs = append(pget.URLs, url) 192 | } 193 | } 194 | } 195 | 196 | if err := scanner.Err(); err != nil { 197 | return errors.Wrap(err, "failed to parse url from stdin") 198 | } 199 | 200 | if len(pget.URLs) < 1 { 201 | return errors.New("urls not found in the arguments passed") 202 | } 203 | } 204 | 205 | return nil 206 | } 207 | -------------------------------------------------------------------------------- /pget_test.go: -------------------------------------------------------------------------------- 1 | package pget 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "crypto/md5" 7 | "encoding/hex" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "net/http" 12 | "net/http/httptest" 13 | "os" 14 | "path/filepath" 15 | "testing" 16 | "time" 17 | 18 | "github.com/stretchr/testify/assert" 19 | ) 20 | 21 | func TestMain(m *testing.M) { 22 | stdout = ioutil.Discard 23 | os.Exit(m.Run()) 24 | } 25 | 26 | func TestPget(t *testing.T) { 27 | // listening file server 28 | mux := http.NewServeMux() 29 | mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 30 | http.Redirect(w, r, "/moo", http.StatusFound) 31 | }) 32 | 33 | mux.HandleFunc("/moo", func(w http.ResponseWriter, r *http.Request) { 34 | http.Redirect(w, r, "/mooo", http.StatusFound) 35 | }) 36 | 37 | mux.HandleFunc("/mooo", func(w http.ResponseWriter, r *http.Request) { 38 | http.Redirect(w, r, "/test.tar.gz", http.StatusFound) 39 | }) 40 | 41 | mux.HandleFunc("/test.tar.gz", func(w http.ResponseWriter, r *http.Request) { 42 | fp := "_testdata/test.tar.gz" 43 | data, err := ioutil.ReadFile(fp) 44 | if err != nil { 45 | t.Errorf("failed to readfile: %s", err) 46 | } 47 | http.ServeContent(w, r, fp, time.Now(), bytes.NewReader(data)) 48 | }) 49 | 50 | ts := httptest.NewServer(mux) 51 | defer ts.Close() 52 | 53 | // begin tests 54 | url := ts.URL 55 | 56 | tmpdir := t.TempDir() 57 | 58 | cfg := &DownloadConfig{ 59 | Filename: "test.tar.gz", 60 | ContentLength: 1719652, 61 | Dirname: tmpdir, 62 | Procs: 4, 63 | URLs: []string{ts.URL}, 64 | Client: newDownloadClient(1), 65 | } 66 | 67 | t.Run("check", func(t *testing.T) { 68 | target, err := Check(context.Background(), &CheckConfig{ 69 | URLs: []string{url}, 70 | Timeout: 10 * time.Second, 71 | }) 72 | 73 | if err != nil { 74 | t.Fatalf("failed to check header: %s", err) 75 | } 76 | 77 | if len(target.URLs) == 0 { 78 | t.Fatalf("invalid URL length %d", len(target.URLs)) 79 | } 80 | 81 | // could redirect? 82 | assert.NotEqual(t, target.URLs[0], url, "failed to get of the last url in the redirect") 83 | }) 84 | 85 | t.Run("download", func(t *testing.T) { 86 | err := Download(context.Background(), cfg) 87 | if err != nil { 88 | t.Fatal(err) 89 | } 90 | // check of the file to exists 91 | for i := 0; i < cfg.Procs; i++ { 92 | filename := filepath.Join(tmpdir, "_test.tar.gz.4", fmt.Sprintf("test.tar.gz.2.%d", i)) 93 | _, err := os.Stat(filename) 94 | if err == nil { 95 | t.Errorf("%q does not exist: %v", filename, err) 96 | } 97 | } 98 | 99 | cmpFileChecksum(t, "_testdata/test.tar.gz", filepath.Join(tmpdir, cfg.Filename)) 100 | }) 101 | } 102 | 103 | func get2md5(path string) (string, error) { 104 | f, err := os.Open(path) 105 | if err != nil { 106 | return "", err 107 | } 108 | 109 | defer f.Close() 110 | 111 | hash := md5.New() 112 | if _, err := io.Copy(hash, f); err != nil { 113 | return "", err 114 | } 115 | 116 | // get the 16 bytes hash 117 | bytes := hash.Sum(nil)[:16] 118 | 119 | return hex.EncodeToString(bytes), nil 120 | } 121 | 122 | func cmpFileChecksum(t *testing.T, wantPath, gotPath string) { 123 | t.Helper() 124 | want, err := get2md5(wantPath) 125 | 126 | if err != nil { 127 | t.Fatalf("failed to md5sum of original file: %s", err) 128 | } 129 | 130 | resultfp, err := get2md5(gotPath) 131 | if err != nil { 132 | t.Fatalf("failed to md5sum of result file: %s", err) 133 | } 134 | 135 | if want != resultfp { 136 | t.Errorf("expected %s got %s", want, resultfp) 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /requests.go: -------------------------------------------------------------------------------- 1 | package pget 2 | 3 | import ( 4 | "context" 5 | "mime" 6 | "net/http" 7 | "path" 8 | "sync" 9 | "time" 10 | 11 | "github.com/pkg/errors" 12 | "golang.org/x/sync/errgroup" 13 | ) 14 | 15 | // Range struct for range access 16 | type Range struct { 17 | low int64 18 | high int64 19 | } 20 | 21 | func isNotLastURL(url, purl string) bool { 22 | return url != purl && url != "" 23 | } 24 | 25 | // CheckConfig is a configuration to check download target. 26 | type CheckConfig struct { 27 | URLs []string 28 | Timeout time.Duration 29 | Client *http.Client 30 | } 31 | 32 | // Target represensts download target. 33 | type Target struct { 34 | Filename string 35 | ContentLength int64 36 | URLs []string 37 | } 38 | 39 | // Check checks be able to download from targets 40 | func Check(ctx context.Context, c *CheckConfig) (*Target, error) { 41 | ctx, cancel := context.WithTimeout(ctx, c.Timeout) 42 | defer cancel() 43 | 44 | if len(c.URLs) == 0 { 45 | return nil, errors.New("URL is required at least one") 46 | } 47 | 48 | client := newClient(c.Client) 49 | 50 | infos, err := getMirrorInfos(ctx, client, c.URLs) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | filename, err := checkEachContent(infos) 56 | if err != nil { 57 | return nil, err 58 | } 59 | if filename == "" { 60 | filename = path.Base(infos[0].RetrievedURL) 61 | } 62 | 63 | urls := make([]string, len(infos)) 64 | for i, info := range infos { 65 | urls[i] = info.RetrievedURL 66 | } 67 | 68 | return &Target{ 69 | Filename: filename, 70 | ContentLength: infos[0].ContentLength, 71 | URLs: urls, 72 | }, nil 73 | } 74 | 75 | func getMirrorInfos(ctx context.Context, client *http.Client, urls []string) ([]*mirrorInfo, error) { 76 | var mu sync.Mutex 77 | eg, ctx := errgroup.WithContext(ctx) 78 | 79 | infos := make([]*mirrorInfo, 0, len(urls)) 80 | 81 | for _, url := range urls { 82 | url := url 83 | eg.Go(func() error { 84 | info, err := getMirrorInfo(ctx, client, url) 85 | if err != nil { 86 | return errors.Wrap(err, url) 87 | } 88 | 89 | mu.Lock() 90 | infos = append(infos, info) 91 | mu.Unlock() 92 | 93 | return nil 94 | }) 95 | } 96 | 97 | if err := eg.Wait(); err != nil { 98 | return nil, err 99 | } 100 | 101 | return infos, nil 102 | } 103 | 104 | type mirrorInfo struct { 105 | RetrievedURL string 106 | ContentLength int64 107 | Filename string 108 | } 109 | 110 | func getMirrorInfo(ctx context.Context, client *http.Client, url string) (*mirrorInfo, error) { 111 | req, err := http.NewRequest("HEAD", url, nil) 112 | if err != nil { 113 | return nil, errors.Wrap(err, "failed to make head request") 114 | } 115 | req = req.WithContext(ctx) 116 | 117 | resp, err := client.Do(req) 118 | if err != nil { 119 | return nil, errors.Wrap(err, "failed to head request") 120 | } 121 | 122 | if resp.Header.Get("Accept-Ranges") != "bytes" { 123 | return nil, errors.New("does not support range request") 124 | } 125 | 126 | if resp.ContentLength <= 0 { 127 | return nil, errors.New("invalid content length") 128 | } 129 | 130 | filename := "" 131 | _, params, _ := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) 132 | if len(params) > 0 && params["filename"] != "" { 133 | filename = params["filename"] 134 | } 135 | 136 | // To perform with the correct "range access" 137 | // get the last url in the redirect 138 | _url := resp.Request.URL.String() 139 | if isNotLastURL(_url, url) { 140 | return &mirrorInfo{ 141 | RetrievedURL: _url, 142 | ContentLength: resp.ContentLength, 143 | Filename: filename, 144 | }, nil 145 | } 146 | 147 | return &mirrorInfo{ 148 | RetrievedURL: url, 149 | ContentLength: resp.ContentLength, 150 | Filename: filename, 151 | }, nil 152 | } 153 | 154 | // check contents are the same on each mirrors 155 | func checkEachContent(infos []*mirrorInfo) (string, error) { 156 | var ( 157 | filename string 158 | contentLength int64 159 | ) 160 | for _, info := range infos { 161 | if info.Filename != "" { 162 | filename = info.Filename 163 | } 164 | if contentLength == 0 { 165 | contentLength = info.ContentLength 166 | continue 167 | } 168 | if contentLength != info.ContentLength { 169 | return "", errors.New("does not match content length on each mirrors") 170 | } 171 | } 172 | return filename, nil 173 | } 174 | -------------------------------------------------------------------------------- /run_test.go: -------------------------------------------------------------------------------- 1 | package pget 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "net/http/httptest" 11 | "os" 12 | "path/filepath" 13 | "testing" 14 | "time" 15 | 16 | "github.com/mholt/archiver" 17 | ) 18 | 19 | func TestRunResume(t *testing.T) { 20 | // listening file server 21 | mux := http.NewServeMux() 22 | 23 | mux.HandleFunc("/file.name", func(w http.ResponseWriter, r *http.Request) { 24 | fp := filepath.Join("_testdata", "test.tar.gz") 25 | data, err := ioutil.ReadFile(fp) 26 | if err != nil { 27 | t.Errorf("failed to readfile: %s", err) 28 | } 29 | http.ServeContent(w, r, fp, time.Now(), bytes.NewReader(data)) 30 | }) 31 | 32 | ts := httptest.NewServer(mux) 33 | t.Cleanup(ts.Close) 34 | 35 | url := ts.URL 36 | targetURL := fmt.Sprintf("%s/%s", url, "file.name") 37 | tmpDir := t.TempDir() 38 | 39 | // resume.tar.gz is included resumable file structures. 40 | // _file.name.3 41 | // ├── file.name.3.0 42 | // ├── file.name.3.1 43 | // └── file.name.3.2 44 | resumeFilePath := filepath.Join(tmpDir, "resume.tar.gz") 45 | if err := copy( 46 | filepath.Join("_testdata", "resume.tar.gz"), 47 | resumeFilePath, 48 | ); err != nil { 49 | t.Fatalf("failed to copy: %s", err) 50 | } 51 | 52 | if err := archiver.NewTarGz().Unarchive(resumeFilePath, tmpDir); err != nil { 53 | t.Fatalf("failed to untargz: %s", err) 54 | } 55 | 56 | p := New() 57 | if err := p.Run(context.Background(), version, []string{ 58 | "pget", 59 | "-p", 60 | "3", 61 | targetURL, 62 | "--timeout", 63 | "5", 64 | "--output", 65 | tmpDir, 66 | }); err != nil { 67 | t.Errorf("failed to Run: %s", err) 68 | } 69 | 70 | cmpFileChecksum(t, 71 | filepath.Join("_testdata", "test.tar.gz"), 72 | filepath.Join(tmpDir, "file.name"), 73 | ) 74 | } 75 | 76 | func copy(src, dest string) error { 77 | srcp, err := os.Open(src) 78 | if err != nil { 79 | return err 80 | } 81 | defer srcp.Close() 82 | 83 | dst, err := os.Create(dest) 84 | if err != nil { 85 | return err 86 | } 87 | defer dst.Close() 88 | 89 | if _, err = io.Copy(dst, srcp); err != nil { 90 | return err 91 | } 92 | 93 | return nil 94 | } 95 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package pget 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | ) 8 | 9 | func getPartialDirname(targetDir, filename string, procs int) string { 10 | if targetDir == "" { 11 | return fmt.Sprintf("_%s.%d", filename, procs) 12 | } 13 | return filepath.Join(targetDir, fmt.Sprintf("_%s.%d", filename, procs)) 14 | } 15 | 16 | // getPartialFilePath returns the path of the partial file 17 | func getPartialFilePath(targetDir, filename string, id, procs int) string { 18 | return filepath.Join( 19 | targetDir, 20 | fmt.Sprintf("%s.%d.%d", filename, procs, id), 21 | ) 22 | } 23 | 24 | // checkProgress In order to confirm the degree of progress 25 | func checkProgress(dirname string) (int64, error) { 26 | return subDirsize(dirname) 27 | } 28 | 29 | func subDirsize(dirname string) (int64, error) { 30 | var size int64 31 | err := filepath.Walk(dirname, func(_ string, info os.FileInfo, err error) error { 32 | if !info.IsDir() { 33 | size += info.Size() 34 | } 35 | return err 36 | }) 37 | 38 | return size, err 39 | } 40 | 41 | func makeRange(i, procs int, rangeSize, contentLength int64) Range { 42 | low := rangeSize * int64(i) 43 | if i == procs-1 { 44 | return Range{ 45 | low: low, 46 | high: contentLength, 47 | } 48 | } 49 | return Range{ 50 | low: low, 51 | high: low + rangeSize - 1, 52 | } 53 | } 54 | 55 | func (r Range) BytesRange() string { 56 | return fmt.Sprintf("bytes=%d-%d", r.low, r.high) 57 | } 58 | --------------------------------------------------------------------------------