├── .gitignore ├── LICENSE ├── README.md ├── downloader.go ├── go.mod ├── go.sum └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 polarisxu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # downloader 2 | 并发下载的示例程序 3 | -------------------------------------------------------------------------------- /downloader.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "net/http" 8 | "os" 9 | "path" 10 | "strings" 11 | "sync" 12 | 13 | "github.com/k0kubun/go-ansi" 14 | "github.com/schollz/progressbar/v3" 15 | ) 16 | 17 | type Downloader struct { 18 | concurrency int 19 | resume bool 20 | 21 | bar *progressbar.ProgressBar 22 | } 23 | 24 | func NewDownloader(concurrency int, resume bool) *Downloader { 25 | return &Downloader{concurrency: concurrency, resume: resume} 26 | } 27 | 28 | func (d *Downloader) Download(strURL, filename string) error { 29 | if filename == "" { 30 | filename = path.Base(strURL) 31 | } 32 | 33 | resp, err := http.Head(strURL) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | if resp.StatusCode == http.StatusOK && resp.Header.Get("Accept-Ranges") == "bytes" { 39 | return d.multiDownload(strURL, filename, int(resp.ContentLength)) 40 | } 41 | 42 | return d.singleDownload(strURL, filename) 43 | } 44 | 45 | func (d *Downloader) multiDownload(strURL, filename string, contentLen int) error { 46 | d.setBar(contentLen) 47 | 48 | partSize := contentLen / d.concurrency 49 | 50 | // 创建部分文件的存放目录 51 | partDir := d.getPartDir(filename) 52 | os.Mkdir(partDir, 0777) 53 | defer os.RemoveAll(partDir) 54 | 55 | var wg sync.WaitGroup 56 | wg.Add(d.concurrency) 57 | 58 | rangeStart := 0 59 | 60 | for i := 0; i < d.concurrency; i++ { 61 | go func(i, rangeStart int) { 62 | defer wg.Done() 63 | 64 | rangeEnd := rangeStart + partSize 65 | // 最后一部分，总长度不能超过 ContentLength 66 | if i == d.concurrency-1 { 67 | rangeEnd = contentLen 68 | } 69 | 70 | downloaded := 0 71 | if d.resume { 72 | partFileName := d.getPartFilename(filename, i) 73 | content, err := os.ReadFile(partFileName) 74 | if err == nil { 75 | downloaded = len(content) 76 | } 77 | d.bar.Add(downloaded) 78 | } 79 | 80 | d.downloadPartial(strURL, filename, rangeStart+downloaded, rangeEnd, i) 81 | 82 | }(i, rangeStart) 83 | 84 | rangeStart += partSize + 1 85 | } 86 | 87 | wg.Wait() 88 | 89 | d.merge(filename) 90 | 91 | return nil 92 | } 93 | 94 | func (d *Downloader) downloadPartial(strURL, filename string, rangeStart, rangeEnd, i int) { 95 | if rangeStart >= rangeEnd { 96 | return 97 | } 98 | 99 | req, err := http.NewRequest("GET", strURL, nil) 100 | if err != nil { 101 | log.Fatal(err) 102 | } 103 | 104 | req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", rangeStart, rangeEnd)) 105 | resp, err := http.DefaultClient.Do(req) 106 | if err != nil { 107 | log.Fatal(err) 108 | } 109 | defer resp.Body.Close() 110 | 111 | flags := os.O_CREATE | os.O_WRONLY 112 | if d.resume { 113 | flags |= os.O_APPEND 114 | } 115 | 116 | partFile, err := os.OpenFile(d.getPartFilename(filename, i), flags, 0666) 117 | if err != nil { 118 | log.Fatal(err) 119 | } 120 | defer partFile.Close() 121 | 122 | buf := make([]byte, 32*1024) 123 | _, err = io.CopyBuffer(io.MultiWriter(partFile, d.bar), resp.Body, buf) 124 | if err != nil { 125 | if err == io.EOF { 126 | return 127 | } 128 | log.Fatal(err) 129 | } 130 | } 131 | 132 | func (d *Downloader) merge(filename string) error { 133 | destFile, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0666) 134 | if err != nil { 135 | return err 136 | } 137 | defer destFile.Close() 138 | 139 | for i := 0; i < d.concurrency; i++ { 140 | partFileName := d.getPartFilename(filename, i) 141 | partFile, err := os.Open(partFileName) 142 | if err != nil { 143 | return err 144 | } 145 | io.Copy(destFile, partFile) 146 | partFile.Close() 147 | os.Remove(partFileName) 148 | } 149 | 150 | return nil 151 | } 152 | 153 | // getPartDir 部分文件存放的目录 154 | func (d *Downloader) getPartDir(filename string) string { 155 | return strings.SplitN(filename, ".", 2)[0] 156 | } 157 | 158 | // getPartFilename 构造部分文件的名字 159 | func (d *Downloader) getPartFilename(filename string, partNum int) string { 160 | partDir := d.getPartDir(filename) 161 | return fmt.Sprintf("%s/%s-%d", partDir, filename, partNum) 162 | } 163 | 164 | func (d *Downloader) singleDownload(strURL, filename string) error { 165 | resp, err := http.Get(strURL) 166 | if err != nil { 167 | return err 168 | } 169 | defer resp.Body.Close() 170 | 171 | d.setBar(int(resp.ContentLength)) 172 | 173 | f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0666) 174 | if err != nil { 175 | return err 176 | } 177 | defer f.Close() 178 | 179 | buf := make([]byte, 32*1024) 180 | _, err = io.CopyBuffer(io.MultiWriter(f, d.bar), resp.Body, buf) 181 | return err 182 | } 183 | 184 | func (d *Downloader) setBar(length int) { 185 | d.bar = progressbar.NewOptions( 186 | length, 187 | progressbar.OptionSetWriter(ansi.NewAnsiStdout()), 188 | progressbar.OptionEnableColorCodes(true), 189 | progressbar.OptionShowBytes(true), 190 | progressbar.OptionSetWidth(50), 191 | progressbar.OptionSetDescription("downloading..."), 192 | progressbar.OptionSetTheme(progressbar.Theme{ 193 | Saucer: "[green]=[reset]", 194 | SaucerHead: "[green]>[reset]", 195 | SaucerPadding: " ", 196 | BarStart: "[", 197 | BarEnd: "]", 198 | }), 199 | ) 200 | } 201 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/polaris1119/downloader 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 7 | github.com/schollz/progressbar/v3 v3.8.1 8 | github.com/urfave/cli/v2 v2.3.0 9 | ) 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 3 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 6 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= 8 | github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= 9 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 10 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 11 | github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow= 12 | github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= 13 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= 14 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= 15 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 16 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 17 | github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 18 | github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= 19 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 20 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= 21 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 22 | github.com/schollz/progressbar/v3 v3.8.1 h1:maiA95sku3mMHbERvCwzn/Tj6258Fm5NQf0E4L/a+5o= 23 | github.com/schollz/progressbar/v3 v3.8.1/go.mod h1:rS3+CgxcNODZywN7C/z/7XH8gxCBLwuW5UmOUiNpOgs= 24 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 25 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 26 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 27 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 28 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 29 | github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= 30 | github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= 31 | golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o= 32 | golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= 33 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 34 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 35 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 36 | golang.org/x/sys v0.0.0-20210511113859-b0526f3d8744 h1:yhBbb4IRs2HS9PPlAg6DMC6mUOKexJBNsLf4Z+6En1Q= 37 | golang.org/x/sys v0.0.0-20210511113859-b0526f3d8744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 38 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 39 | golang.org/x/term v0.0.0-20210503060354-a79de5458b56 h1:b8jxX3zqjpqb2LklXPzKSGJhzyxCOZSz8ncv8Nv+y7w= 40 | golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= 41 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 42 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 43 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 44 | gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 45 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "runtime" 7 | 8 | "github.com/urfave/cli/v2" 9 | ) 10 | 11 | func main() { 12 | // 默认并发数 13 | concurrencyN := runtime.NumCPU() 14 | 15 | app := &cli.App{ 16 | Name: "downloader", 17 | Usage: "File concurrency downloader", 18 | Flags: []cli.Flag{ 19 | &cli.StringFlag{ 20 | Name: "url", 21 | Aliases: []string{"u"}, 22 | Usage: "`URL` to download", 23 | Required: true, 24 | }, 25 | &cli.StringFlag{ 26 | Name: "output", 27 | Aliases: []string{"o"}, 28 | Usage: "Output `filename`", 29 | }, 30 | &cli.IntFlag{ 31 | Name: "concurrency", 32 | Aliases: []string{"n"}, 33 | Value: concurrencyN, 34 | Usage: "Concurrency `number`", 35 | }, 36 | &cli.BoolFlag{ 37 | Name: "resume", 38 | Aliases: []string{"r"}, 39 | Value: true, 40 | Usage: "Resume download", 41 | }, 42 | }, 43 | Action: func(c *cli.Context) error { 44 | strURL := c.String("url") 45 | filename := c.String("output") 46 | concurrency := c.Int("concurrency") 47 | resume := c.Bool("resume") 48 | return NewDownloader(concurrency, resume).Download(strURL, filename) 49 | }, 50 | } 51 | 52 | err := app.Run(os.Args) 53 | if err != nil { 54 | log.Fatal(err) 55 | } 56 | } 57 | --------------------------------------------------------------------------------