├── .gitignore ├── screenshot_grozilla.jpg ├── main.go ├── LICENSE ├── util.go ├── README.md ├── download_helper.go ├── download.go ├── fileutil.go └── log.go /.gitignore: -------------------------------------------------------------------------------- 1 | grozilla 2 | -------------------------------------------------------------------------------- /screenshot_grozilla.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prashant-agarwala/grozilla/HEAD/screenshot_grozilla.jpg -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | ) 7 | 8 | var ( 9 | noOfFiles = flag.Int("n", 10, "number of parallel connection") 10 | resume = flag.Bool("r", false, "resume pending download") 11 | maxTryCount = flag.Int("m", 1, "maximum attempts to establish a connection") 12 | timeout = flag.Int("t", 900, "maximum time in seconds it will wait to establish a connection") 13 | ovrdConnLimit = flag.Bool("N", false, "maximum connection is restricted to 20, to force more connection") 14 | ) 15 | 16 | func main() { 17 | flag.Parse() 18 | args := flag.Args() 19 | if len(args) < 1 { 20 | log.Fatal("Specify a file url to download") 21 | } 22 | validateFlags() 23 | url := args[0] 24 | url, resHeader := getFinalurl(url) 25 | if *resume { 26 | if acceptRanges(resHeader) { 27 | Resume(url, getContentLength(resHeader)) 28 | } 29 | } else { 30 | if acceptRanges(resHeader) { 31 | Download(url, getContentLength(resHeader)) 32 | } else { 33 | DownloadSingle(url) 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Prashant Agarwala 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | "net/http" 7 | "os" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | func acceptRanges(m http.Header) bool { 13 | for _, v := range m["Accept-Ranges"] { 14 | if v == "bytes" { 15 | return true 16 | } 17 | } 18 | return false 19 | } 20 | 21 | func getFilenameFromURL(url string) string { 22 | file := url[strings.LastIndex(url, "/")+1:] 23 | if strings.Index(file, "?") != -1 { 24 | return file[:strings.Index(file, "?")] 25 | } 26 | return file 27 | } 28 | 29 | func getContentLength(m http.Header) int { 30 | length, _ := strconv.Atoi(m["Content-Length"][0]) 31 | return length 32 | } 33 | 34 | func getFinalurl(url string) (string, http.Header) { 35 | client := &http.Client{} 36 | res, err := client.Head(url) 37 | if err != nil { 38 | log.Fatal(err) 39 | } 40 | responseURL := res.Request.URL.String() 41 | if responseURL != url { 42 | return getFinalurl(responseURL) 43 | } 44 | return responseURL, res.Header 45 | } 46 | 47 | func validateFlags() { 48 | if *noOfFiles <= 0 || *maxTryCount <= 0 || *timeout <= 0 { 49 | log.Println("Give a value greater than 0") 50 | flag.Usage() 51 | os.Exit(1) 52 | } 53 | if !(*ovrdConnLimit) { 54 | if *noOfFiles > 20 { 55 | log.Println("Connection limit restricted to 20, either use lower value or override using -N") 56 | flag.Usage() 57 | os.Exit(1) 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Grozilla 2 | The Grozilla is a simple implementation that allows downloading of video,audio,package or zip files parallely and 3 | efficiently using light weight go routines. 4 | 5 | ### Usage: 6 | 7 | ``` 8 | grozilla [-m] [-n] [-r] [-t] [-N] [download link] 9 | ``` 10 | 11 | ### Installation: 12 | ``` 13 | $ go get github.com/gophergala2016/grozilla 14 | ``` 15 | This will download grozilla to $GOPATH/src/github.com/gophergala2016/grozilla. In this directory run go build to create the grozilla binary. 16 | 17 | ### Description: 18 | 19 | The utility allows to parallely download any downloadble file from the download link. Following are the customization flags which a client can 20 | give 21 | 22 | ``` -n routines ``` 23 | Used to specify number of go routines (default 10). 24 | 25 | ``` -r ``` 26 | Used to resume pending download (which was stopped due to sudden exit). 27 | 28 | ``` -t time ``` 29 | Used to specify maximum time in seconds it will wait to establish a connection (default 900). 30 | 31 | ``` -m attempts ``` 32 | Used to specify maximum attempt to establish a connection (default 1). 33 | 34 | ``` -N nolimit ``` 35 | Used to override maximum connection limit of 20 36 | 37 | 38 | ### Example: 39 | 40 | ![Grozilla-image](https://github.com/gophergala2016/grozilla/blob/master/screenshot_grozilla.jpg "grozilla") 41 | 42 | ### Coming Soon 43 | 44 | - Smooth UI 45 | - Additional Flags 46 | - for output filename 47 | - additional log messages 48 | - header info 49 | - retry delay 50 | - and some more 51 | 52 | ### References 53 | 54 | - https://github.com/cheggaaa/pb 55 | -------------------------------------------------------------------------------- /download_helper.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "io/ioutil" 6 | "net/http" 7 | "strconv" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | type httpResponse struct { 13 | resp *http.Response 14 | err error 15 | } 16 | 17 | //PACKETLENGTH is size of each packet in bytes 18 | const PACKETLENGTH = 32000 19 | 20 | var wg sync.WaitGroup 21 | var errorGoRoutine bool 22 | 23 | func downloadPacket(client *http.Client, req *http.Request, partFilename string, byteStart, byteEnd int) error { 24 | c := make(chan httpResponse, 1) 25 | go func() { 26 | resp, err := client.Do(req) 27 | httpResponse := httpResponse{resp, err} 28 | c <- httpResponse 29 | }() 30 | select { 31 | case httpResponse := <-c: 32 | if err := handleResponse(httpResponse, partFilename, byteStart, byteEnd); err != nil { 33 | return err 34 | } 35 | case <-time.After(time.Second * time.Duration(*timeout)): 36 | err := errors.New("Manual time out as response not recieved") 37 | return err 38 | } 39 | return nil 40 | } 41 | 42 | func handleResponse(httpResponse httpResponse, partFilename string, byteStart, byteEnd int) error { 43 | if httpResponse.err != nil { 44 | return httpResponse.err 45 | } 46 | defer httpResponse.resp.Body.Close() 47 | reader, err := ioutil.ReadAll(httpResponse.resp.Body) 48 | if err != nil { 49 | return err 50 | } 51 | err = writeBytes(partFilename, reader, byteStart, byteEnd) 52 | if err != nil { 53 | return err 54 | } 55 | return nil 56 | } 57 | 58 | func downloadPacketWithRetry(client *http.Client, req *http.Request, partFilename string, byteStart, byteEnd int) error { 59 | var err error 60 | for i := 0; i < *maxTryCount; i++ { 61 | err = downloadPacket(client, req, partFilename, byteStart, byteEnd) 62 | if err == nil { 63 | return nil 64 | } else if err.Error() == "Manual time out as response not recieved" { 65 | continue 66 | } else { 67 | return err 68 | } 69 | } 70 | return err 71 | } 72 | 73 | func downloadPart(url, filename string, index, byteStart, byteEnd int) { 74 | client := &http.Client{} 75 | partFilename := filename + "_" + strconv.Itoa(index) 76 | noofpacket := (byteEnd-byteStart+1)/PACKETLENGTH + 1 77 | for i := 0; i < noofpacket; i++ { 78 | packetStart := byteStart + i*PACKETLENGTH 79 | packetEnd := packetStart + PACKETLENGTH 80 | if i == noofpacket-1 { 81 | packetEnd = byteEnd 82 | } 83 | rangeHeader := "bytes=" + strconv.Itoa(packetStart) + "-" + strconv.Itoa(packetEnd-1) 84 | req, _ := http.NewRequest("GET", url, nil) 85 | req.Header.Add("Range", rangeHeader) 86 | err := downloadPacketWithRetry(client, req, partFilename, byteStart, byteEnd) 87 | if err != nil { 88 | handleErrorInGoRoutine(index, err) 89 | return 90 | } 91 | UpdateStat(index, packetStart, packetEnd) 92 | } 93 | wg.Done() 94 | } 95 | 96 | func handleErrorInGoRoutine(index int, err error) { 97 | ReportErrorStat(index, err, *noOfFiles) 98 | errorGoRoutine = true 99 | wg.Done() 100 | } 101 | -------------------------------------------------------------------------------- /download.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "log" 6 | "net/http" 7 | "os" 8 | "strconv" 9 | ) 10 | 11 | //Download downloads a file from a given url by creating parallel connection 12 | func Download(url string, length int) { 13 | partLength := length / *noOfFiles 14 | filename := getFilenameFromURL(url) 15 | filename = getFilename(filename) 16 | if _, err := os.Stat("temp/" + filename + "_0"); err == nil { 17 | log.Fatal("Downloading has already started, resume downloading.") 18 | } 19 | if err := SetupLog(length, *noOfFiles); err != nil { 20 | log.Fatal(err) 21 | } 22 | for i := 0; i < *noOfFiles; i++ { 23 | byteStart := partLength * (i) 24 | byteEnd := byteStart + partLength 25 | if i == *noOfFiles-1 { 26 | byteEnd = length 27 | } 28 | os.MkdirAll("temp/", 0777) 29 | createTempFile("temp/"+filename+"_"+strconv.Itoa(i), byteStart, byteEnd) 30 | wg.Add(1) 31 | go downloadPart(url, filename, i, byteStart, byteEnd) 32 | } 33 | wg.Wait() 34 | FinishLog() 35 | if !errorGoRoutine { 36 | mergeFiles(filename, *noOfFiles) 37 | clearFiles(filename, *noOfFiles) 38 | log.Println("download successful") 39 | } else { 40 | log.Println("download unsuccessful") 41 | } 42 | } 43 | 44 | //Resume resumes a interrupted download by creating same number of connection 45 | func Resume(url string, length int) { 46 | filename := getFilenameFromURL(url) 47 | filename = getFilename(filename) 48 | *noOfFiles = noOfExistingConnection(filename, length) 49 | partLength := length / *noOfFiles 50 | if err := SetupResumeLog(filename, length, *noOfFiles); err != nil { 51 | log.Fatal(err) 52 | } 53 | for i := 0; i < *noOfFiles; i++ { 54 | partFilename := "temp/" + filename + "_" + strconv.Itoa(i) 55 | if _, err := os.Stat(partFilename); err != nil { 56 | byteStart := partLength * (i) 57 | byteEnd := byteStart + partLength 58 | if i == *noOfFiles-1 { 59 | byteEnd = length 60 | } 61 | wg.Add(1) 62 | go downloadPart(url, filename, i, byteStart, byteEnd) 63 | } else { 64 | byteStart, byteEnd := readHeader(partFilename) 65 | if byteStart < byteEnd { 66 | wg.Add(1) 67 | go downloadPart(url, filename, i, byteStart, byteEnd) 68 | } 69 | } 70 | } 71 | wg.Wait() 72 | FinishLog() 73 | if !errorGoRoutine { 74 | mergeFiles(filename, *noOfFiles) 75 | clearFiles(filename, *noOfFiles) 76 | log.Println("download successful") 77 | } else { 78 | log.Println("download unsuccessful") 79 | } 80 | } 81 | 82 | //DownloadSingle downloads a file from a given url by creating single connection 83 | func DownloadSingle(url string) { 84 | filename := getFilenameFromURL(url) 85 | client := &http.Client{} 86 | req, _ := http.NewRequest("GET", url, nil) 87 | resp, err := client.Do(req) 88 | if err != nil { 89 | log.Fatal(err) 90 | } 91 | reader, err := ioutil.ReadAll(resp.Body) 92 | if err != nil { 93 | log.Fatal(err) 94 | } 95 | err = ioutil.WriteFile(filename, reader, 0666) 96 | if err != nil { 97 | log.Fatal(err) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /fileutil.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "io" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "strconv" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | func createTempFile(partFilename string, fileBegin, fileEnd int) { 16 | buf := new(bytes.Buffer) 17 | if err := binary.Write(buf, binary.LittleEndian, int64(fileBegin)); err != nil { 18 | log.Fatal(err) 19 | } 20 | if err := binary.Write(buf, binary.LittleEndian, int64(fileEnd)); err != nil { 21 | log.Fatal(err) 22 | } 23 | if err := ioutil.WriteFile(partFilename, buf.Bytes(), 0666); err != nil { 24 | log.Fatal(err) 25 | } 26 | } 27 | 28 | func writeBytes(partFilename string, reader []byte, byteStart, byteEnd int) error { 29 | if err := os.MkdirAll("temp/", 0777); err != nil { 30 | return err 31 | } 32 | if _, err := os.Stat("temp/" + partFilename); err != nil { 33 | createTempFile("temp/"+partFilename, byteStart, byteEnd) 34 | } 35 | file, err := os.OpenFile("temp/"+partFilename, os.O_WRONLY|os.O_APPEND, 0666) 36 | if err != nil { 37 | return err 38 | } 39 | defer file.Close() 40 | if _, err = file.WriteString(string(reader)); err != nil { 41 | return err 42 | } 43 | return nil 44 | } 45 | 46 | func readHeader(partFilename string) (int, int) { 47 | reader, err := ioutil.ReadFile(partFilename) 48 | if err != nil { 49 | log.Fatal(err) 50 | } 51 | header := reader[:16] 52 | byteStart := int(binary.LittleEndian.Uint64(header[0:8])) + len(reader) - 16 53 | byteEnd := int(binary.LittleEndian.Uint64(header[8:16])) 54 | return byteStart, byteEnd 55 | } 56 | 57 | func mergeFiles(filename string, count int) { 58 | tempFilename := strconv.Itoa(time.Now().Nanosecond()) + "_" + filename 59 | for i := 0; i < count; i++ { 60 | partFilename := "temp/" + filename + "_" + strconv.Itoa(i) 61 | file, err := os.OpenFile(tempFilename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) 62 | if err != nil { 63 | log.Fatal(err) 64 | } 65 | defer file.Close() 66 | reader, err := ioutil.ReadFile(partFilename) 67 | reader = reader[16:] 68 | if err != nil { 69 | log.Fatal(err) 70 | } 71 | if _, err = file.WriteString(string(reader)); err != nil { 72 | log.Fatal(err) 73 | } 74 | } 75 | os.Rename(tempFilename, filename) 76 | } 77 | func isDirEmpty(name string) (bool, error) { 78 | f, err := os.Open(name) 79 | if err != nil { 80 | return false, err 81 | } 82 | defer f.Close() 83 | _, err = f.Readdir(1) 84 | if err == io.EOF { 85 | return true, nil 86 | } 87 | return false, err 88 | } 89 | 90 | func clearFiles(filename string, count int) { 91 | for i := 0; i < count; i++ { 92 | partFilename := "temp/" + filename + "_" + strconv.Itoa(i) 93 | os.Remove(partFilename) 94 | } 95 | empty, err := isDirEmpty("temp/") 96 | if err != nil { 97 | log.Fatal(err) 98 | } 99 | if empty { 100 | os.Remove("temp/") 101 | } 102 | } 103 | 104 | func noOfExistingConnection(filename string, length int) int { 105 | existingFilename := "temp/" + filename + "_0" 106 | if _, err := os.Stat(existingFilename); err != nil { 107 | log.Fatal("No file to resume downloading") 108 | } 109 | if _, err := os.Stat(existingFilename); err == nil { 110 | reader, err := ioutil.ReadFile(existingFilename) 111 | if err != nil { 112 | log.Fatal(err) 113 | } 114 | if len(reader) < 16 { 115 | log.Fatal("No file to resume downloading") 116 | } 117 | header := reader[:16] 118 | interval := int(binary.LittleEndian.Uint64(header[8:16])) - int(binary.LittleEndian.Uint64(header[0:8])) 119 | if interval == 0 { 120 | log.Fatal("No file to resume downloading") 121 | } 122 | return (length / interval) 123 | } 124 | return 0 125 | } 126 | 127 | func getFilename(filename string) string { 128 | j := 0 129 | for j = 0; ; j++ { 130 | if j == 1 { 131 | filename += "(1)" 132 | } 133 | if (j != 0) && (j != 1) { 134 | filename = strings.Replace(filename, "("+strconv.Itoa(j-1)+")", "("+strconv.Itoa(j)+")", 1) 135 | } 136 | if _, err := os.Stat(filename); os.IsNotExist(err) { 137 | break 138 | } 139 | } 140 | if j != 0 && j != 1 { 141 | filename = strings.Replace(filename, "("+strconv.Itoa(j-1)+")", "("+strconv.Itoa(j)+")", 1) 142 | } 143 | return filename 144 | } 145 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/binary" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "strconv" 9 | 10 | "github.com/cheggaaa/pb" 11 | ) 12 | 13 | //ConnectionLog keeps log of all connection throgh progressbar 14 | type ConnectionLog struct { 15 | stats []ConnectionStat 16 | pool *pb.Pool 17 | totalbar *pb.ProgressBar 18 | } 19 | 20 | //ConnectionStat keeps statistic of each connection 21 | type ConnectionStat struct { 22 | connectionIndex int 23 | pbar *pb.ProgressBar 24 | Err error 25 | } 26 | 27 | var connLog ConnectionLog 28 | 29 | //SetupLog sets up initial ConnectionLog 30 | func SetupLog(length, noOfConn int) error { 31 | connLog.stats = make([]ConnectionStat, noOfConn) 32 | barArray := make([]*pb.ProgressBar, noOfConn+1) 33 | lenSub := length / noOfConn 34 | for i := 0; i < noOfConn; i++ { 35 | fileBegin := lenSub * i 36 | fileEnd := lenSub * (i + 1) 37 | if i == noOfConn-1 { 38 | fileEnd = length 39 | } 40 | bar := pb.New(fileEnd - fileBegin).Prefix("Connection " + strconv.Itoa(i+1) + " ") 41 | customizeBar(bar) 42 | connLog.stats[i] = ConnectionStat{connectionIndex: i, pbar: bar} 43 | barArray[i] = bar 44 | } 45 | bar := pb.New(length).Prefix("Total ") 46 | customizeBar(bar) 47 | connLog.totalbar = bar 48 | barArray[noOfConn] = bar 49 | var err error 50 | connLog.pool, err = pb.StartPool(barArray...) 51 | if err != nil { 52 | return err 53 | } 54 | return nil 55 | } 56 | 57 | func customizeBar(bar *pb.ProgressBar) { 58 | bar.ShowCounters = true 59 | bar.ShowTimeLeft = false 60 | bar.ShowSpeed = true 61 | bar.SetMaxWidth(80) 62 | bar.SetUnits(pb.U_BYTES) 63 | } 64 | 65 | //SetupResumeLog sets up ConnectionLog for a resumed download 66 | func SetupResumeLog(filename string, length, noOfConn int) error { 67 | connLog.stats = make([]ConnectionStat, noOfConn) 68 | barArray := make([]*pb.ProgressBar, noOfConn+1) 69 | totalbar := pb.New(length).Prefix("Total ") 70 | lenSub := length / noOfConn 71 | for i := 0; i < noOfConn; i++ { 72 | partFilename := "temp/" + filename + "_" + strconv.Itoa(i) 73 | if _, err := os.Stat(partFilename); err == nil { 74 | reader, err := ioutil.ReadFile(partFilename) 75 | if err != nil { 76 | return err 77 | } 78 | header := reader[:16] 79 | fileBegin := int(binary.LittleEndian.Uint64(header[0:8])) 80 | fileEnd := int(binary.LittleEndian.Uint64(header[8:16])) 81 | bar := pb.New(fileEnd - fileBegin).Prefix("Connection " + strconv.Itoa(i+1) + " ") 82 | for j := 0; j < len(reader)-16; j++ { 83 | bar.Increment() 84 | totalbar.Increment() 85 | } 86 | customizeBar(bar) 87 | connLog.stats[i] = ConnectionStat{connectionIndex: i, pbar: bar} 88 | barArray[i] = bar 89 | } else { 90 | fileBegin := lenSub * i 91 | fileEnd := lenSub * (i + 1) 92 | if i == noOfConn-1 { 93 | fileEnd = length 94 | } 95 | bar := pb.New(fileEnd - fileBegin).Prefix("Connection " + strconv.Itoa(i+1) + " ") 96 | customizeBar(bar) 97 | connLog.stats[i] = ConnectionStat{connectionIndex: i, pbar: bar} 98 | barArray[i] = bar 99 | } 100 | } 101 | customizeBar(totalbar) 102 | connLog.totalbar = totalbar 103 | barArray[noOfConn] = totalbar 104 | var err error 105 | connLog.pool, err = pb.StartPool(barArray...) 106 | if err != nil { 107 | return err 108 | } 109 | return nil 110 | } 111 | 112 | //UpdateStat updates statistic of a connection 113 | func UpdateStat(i int, fileBegin int, fileEnd int) { 114 | for j := fileBegin; j < fileEnd; j++ { 115 | connLog.stats[i].pbar.Increment() 116 | connLog.totalbar.Increment() 117 | } 118 | } 119 | 120 | //FinishLog stops ConnectionLog pool 121 | func FinishLog() { 122 | connLog.pool.Stop() 123 | } 124 | 125 | //ReportErrorStat reports a log if an error occurs in a connection 126 | func ReportErrorStat(i int, err error, noOfConn int) { 127 | connLog.stats[i].Err = err 128 | connLog.pool.Stop() 129 | log.Println() 130 | log.Println("Error in connection " + strconv.Itoa(i+1) + " : " + err.Error()) 131 | log.Println() 132 | barArray := make([]*pb.ProgressBar, noOfConn+1) 133 | for i := 0; i < noOfConn; i++ { 134 | barArray[i] = connLog.stats[i].pbar 135 | } 136 | barArray[noOfConn] = connLog.totalbar 137 | connLog.pool, _ = pb.StartPool(barArray...) 138 | } 139 | --------------------------------------------------------------------------------