├── .gitignore
├── README.md
├── lib
    └── commons-io-2.6.jar
└── src
    ├── META-INF
        └── MANIFEST.MF
    └── com
        └── zanvent
            └── downloader
                ├── DownloadListener.java
                ├── DownloadManager.java
                ├── Downloader.java
                └── Util.java


/.gitignore:
--------------------------------------------------------------------------------
1 | *.iml
2 | /.idea/
3 | /out
4 | /src/com/zanvent/main


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Introduction 
 2 | 
 3 | File downloader written in Java. Uses thread to download files in multiple connections to fully utilize internet speed.
 4 | 
 5 | ## Features
 6 | 
 7 | * **Multi-threaded. Download files in multiple connections if server accepts range header.**
 8 | * **Retries downloading file on failure.**
 9 | 
10 | ## How-To
11 | 
12 | Create an instance of `DownloadManager` using the constructor `DownloadManager(int concurrentDownload)`. Set `DownloadListener` to the instance of `DownloadManager` using `setDownloadListener`. 
13 | Set path and max connection per downloads, `DownloadManager.Config.path = "D:\\downloads"` and ` DownloadManager.Config.connections = 8`
14 | Create an instance of `URL` using the file url and call `downloadManager.download(url)`.
15 | 
16 | ## Example
17 | 
18 | ```java
19 | public static void main(String[] args) {
20 |         try {
21 |             DownloadManager downloadManager = new DownloadManager(100); // 100 concurrent downloads
22 |             DownloadManager.Config.path = "H:\\transfer";
23 |             DownloadManager.Config.connections = 8;
24 |             downloadManager.setDownloadListener(new DownloadListener() {
25 |                 @Override
26 |                 public void onDownloadStarted(Downloader.DownloadStatus downloadStatus) {
27 |                     System.out.println(downloadStatus.getFileName() + " download started.");
28 |                 }
29 | 
30 |                 @Override
31 |                 public void onDownloadFinished(Downloader.DownloadStatus downloadStatus) {
32 |                     System.out.println(downloadStatus.getFileName() + " download finished. File size " + downloadStatus.getFileSize() + ", downloaded " + downloadStatus.getDownloadedSize());
33 |                 }
34 | 
35 |                 @Override
36 |                 public void onDownloadFailed(Downloader.DownloadStatus downloadStatus) {
37 |                     System.out.println(downloadStatus.getFileName() + " download failed. File size " + downloadStatus.getFileSize() + ", downloaded " + downloadStatus.getDownloadedSize());
38 |                 }
39 |             });
40 | 
41 |             downloadManager.download(new URL("https://pop-iso.sfo2.cdn.digitaloceanspaces.com/19.04/amd64/intel/3/pop-os_19.04_amd64_intel_3.iso".replaceAll(" ", "%20")));
42 |         } catch (IOException e) {
43 |             e.printStackTrace();
44 |         }
45 | }
46 | ```
47 | ## Methods
48 | * `download(URL url)` - Downloads the file from the URL and returns a String containing the process Id.
49 | * `cancelDownload(String processId)` - Cancels a download according to processId.
50 | * `getDownloadStatus(String processId)` - Returns an instance of `DownloadStatus` containing download information according to the process Id.
51 | * `getDownloadList()` - Returns an ArrayList<Downloader> containing all the download processes.
52 | 
53 | ## TODO
54 | 
55 | * Add pause/resume feature
56 | 
57 | ## Contributions
58 | 
59 | Please feel free to contribute!!
60 | 
61 | License
62 | =======
63 | 
64 |      Copyright 2019 Farhan Farooqui
65 |      
66 |      Licensed under the Apache License, Version 2.0 (the "License");
67 |      you may not use this file except in compliance with the License.
68 |      You may obtain a copy of the License at
69 | 
70 |      http://www.apache.org/licenses/LICENSE-2.0
71 | 
72 |      Unless required by applicable law or agreed to in writing, software
73 |      distributed under the License is distributed on an "AS IS" BASIS,
74 |      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
75 |      See the License for the specific language governing permissions and
76 |      limitations under the License.


--------------------------------------------------------------------------------
/lib/commons-io-2.6.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/frhnfrq/Java-Multi-Threaded-File-Downloader/a048b30a19e2b247fc4943ca795851284f291edb/lib/commons-io-2.6.jar


--------------------------------------------------------------------------------
/src/META-INF/MANIFEST.MF:
--------------------------------------------------------------------------------
1 | Manifest-Version: 1.0
2 | Main-Class: com.zanvent.main.Main
3 | 
4 | 


--------------------------------------------------------------------------------
/src/com/zanvent/downloader/DownloadListener.java:
--------------------------------------------------------------------------------
1 | package com.zanvent.downloader;
2 | 
3 | public interface DownloadListener {
4 |     void onDownloadStarted(Downloader.DownloadStatus downloadStatus);
5 |     void onDownloadFinished(Downloader.DownloadStatus downloadStatus);
6 |     void onDownloadFailed(Downloader.DownloadStatus downloadStatus);
7 | }
8 | 


--------------------------------------------------------------------------------
/src/com/zanvent/downloader/DownloadManager.java:
--------------------------------------------------------------------------------
 1 | package com.zanvent.downloader;
 2 | 
 3 | import java.net.URL;
 4 | import java.util.ArrayList;
 5 | import java.util.HashMap;
 6 | import java.util.concurrent.ExecutorService;
 7 | import java.util.concurrent.Executors;
 8 | 
 9 | public class DownloadManager {
10 |     private HashMap<String, Downloader> mDownloadProcesses;
11 |     private int mConcurrentDownload;
12 |     private ExecutorService mExecutorService;
13 |     private DownloadListener mDownloadListener;
14 | 
15 |     public DownloadManager(int concurrentDownload) {
16 |         mConcurrentDownload = concurrentDownload;
17 |         mExecutorService = Executors.newFixedThreadPool(concurrentDownload);
18 |         mDownloadProcesses = new HashMap<>();
19 |     }
20 | 
21 |     public String download(URL url) {
22 |         Downloader downloader = new Downloader(this, url);
23 |         mDownloadProcesses.put(downloader.getProcessID(), downloader);
24 |         mExecutorService.execute(downloader);
25 |         return downloader.getProcessID();
26 |     }
27 | 
28 |     public Downloader.DownloadStatus getDownloadStatus(String processID) {
29 |         Downloader downloader = mDownloadProcesses.get(processID);
30 |         if (downloader == null) {
31 |             return null;
32 |         }
33 | 
34 |         return downloader.getDownloadStatus();
35 |     }
36 | 
37 |     public void cancelDownload(String processId) {
38 |         Downloader downloader = mDownloadProcesses.get(processId);
39 |         downloader.interrupt();
40 |     }
41 | 
42 |     public void setDownloadListener(DownloadListener downloadListener) {
43 |         mDownloadListener = downloadListener;
44 |     }
45 | 
46 |     public DownloadListener getDownloadListener() {
47 |         return mDownloadListener;
48 |     }
49 | 
50 |     public ArrayList<Downloader> getDownloadList() {
51 |         return new ArrayList<>(mDownloadProcesses.values());
52 |     }
53 | 
54 |     public static class Config {
55 |         public static String path = "H:\\transfer";
56 |         public static int connections = 8;
57 |     }
58 | 
59 | }
60 | 


--------------------------------------------------------------------------------
/src/com/zanvent/downloader/Downloader.java:
--------------------------------------------------------------------------------
  1 | package com.zanvent.downloader;
  2 | 
  3 | import org.apache.commons.io.FilenameUtils;
  4 | import org.apache.commons.io.IOUtils;
  5 | 
  6 | import java.io.*;
  7 | import java.net.HttpURLConnection;
  8 | import java.net.URL;
  9 | import java.util.ArrayList;
 10 | import java.util.concurrent.*;
 11 | import java.util.regex.Matcher;
 12 | import java.util.regex.Pattern;
 13 | 
 14 | public class Downloader implements Runnable {
 15 |     private DownloadManager mDownloadManager;
 16 |     private URL mUrl;
 17 |     private HttpURLConnection mConnection;
 18 |     private long mFileSize;
 19 |     private String mFileName;
 20 |     private boolean mAcceptsRange;
 21 |     private boolean mRan = false;
 22 |     private boolean mFinished = false;
 23 |     private boolean mSuccess;
 24 |     private volatile boolean mInterrupt = false;
 25 |     private String mProcessID;
 26 |     private DownloadStatus mDownloadStatus;
 27 |     private ExecutorService mExecutorService;
 28 |     private ArrayList<Future<Boolean>> mDownloadTaskResults;
 29 |     private ArrayList<Range> mRanges;
 30 |     private ArrayList<File> mFileParts;
 31 |     private Util.LockBasedLong mDownloadedSize;
 32 | 
 33 |     public Downloader(DownloadManager downloadManager, URL url) {
 34 |         mDownloadManager = downloadManager;
 35 |         mUrl = url;
 36 |         mProcessID = Util.createProcessID();
 37 |         mDownloadTaskResults = new ArrayList<>();
 38 |         mRanges = new ArrayList<>();
 39 |         mFileParts = new ArrayList<>();
 40 |         mDownloadedSize = new Util.LockBasedLong();
 41 |         mDownloadStatus = new DownloadStatus();
 42 |         mDownloadStatus.setProcessID(mProcessID);
 43 |     }
 44 | 
 45 |     @Override
 46 |     public void run() {
 47 |         mRan = true; // keeps track of download status, if the download ever started
 48 |         try {
 49 |             mConnection = (HttpURLConnection) mUrl.openConnection();
 50 |             mFileSize = getFileSize(mConnection);
 51 |             mFileName = getFilename(mConnection);
 52 | //            System.out.println("Filename is " + mFileName);
 53 |             mAcceptsRange = acceptsRange(mConnection);
 54 | 
 55 |             mDownloadStatus.setFileName(mFileName);
 56 |             mDownloadStatus.setFileSize(mFileSize);
 57 | 
 58 |             mDownloadManager.getDownloadListener().onDownloadStarted(mDownloadStatus);
 59 | 
 60 |             if (mFileName != null && mFileSize > 0) {  // valid URL
 61 |                 if (mAcceptsRange) // "Range: bytes min-max" in the header, download it in multiple connections
 62 |                     mSuccess = initiateDownloader(mUrl, mFileName, DownloadManager.Config.connections);
 63 |                 else    // download in single connection
 64 |                     mSuccess = initiateDownloader(mUrl, mFileName, 1);
 65 |             } else { // invalid URL
 66 |                 mSuccess = false;
 67 |             }
 68 |         } catch (IOException e) {
 69 |             e.printStackTrace();
 70 |             mSuccess = false;
 71 |         }
 72 |         mFinished = true;
 73 |         mConnection.disconnect();
 74 | 
 75 |         if (mSuccess) {
 76 | //            System.out.println(Thread.currentThread().getName() + " download completed. Download size " + mDownloadedSize.get() + " and file size " + mFileSize);
 77 |             mDownloadManager.getDownloadListener().onDownloadFinished(mDownloadStatus);
 78 |         } else {
 79 | //            System.out.println(Thread.currentThread().getName() + " download failed");
 80 |             mDownloadManager.getDownloadListener().onDownloadFailed(mDownloadStatus);
 81 |         }
 82 |     }
 83 | 
 84 | 
 85 |     private boolean initiateDownloader(URL url, String filename, int connections) {
 86 | 
 87 |         File file = new File(DownloadManager.Config.path, filename); // which will be created after download finishes
 88 |         file.getParentFile().mkdirs();
 89 | 
 90 |         if (connections > 1) { // download in multiple connections
 91 |             mExecutorService = Executors.newFixedThreadPool(connections);
 92 |             mRanges = Util.range(mFileSize, connections); // mRanges of bytes for each connection eg. min and max
 93 | 
 94 |             for (int i = 1; i <= connections; i++) {
 95 |                 File f = new File(DownloadManager.Config.path, filename + "_part" + i); // temp file containing parts of the file from each connection
 96 |                 mFileParts.add(f); // keep track of the temp files
 97 |                 DownloaderTask downloaderTask = new DownloaderTask(url, f, i, mRanges.get(i - 1));
 98 |                 Future<Boolean> future = mExecutorService.submit(downloaderTask);
 99 |                 mDownloadTaskResults.add(future); // list of Future<Boolean> is needed to check if the file downloaded successfully or not
100 |             }
101 | 
102 |             boolean success = true;
103 |             for (Future<Boolean> future : mDownloadTaskResults) {
104 |                 try {
105 |                     success &= future.get(); // this is a blocking method. if even one connection failed to download successfully boolean mSuccess will be false
106 |                 } catch (InterruptedException | ExecutionException e) {
107 |                     e.printStackTrace();
108 |                 }
109 |             }
110 | 
111 |             if (success) { // if the file was downloaded successfully, merge it into a single file
112 |                 try {
113 |                     OutputStream outputStream = new FileOutputStream(file, true);
114 |                     for (File fp : mFileParts) {
115 |                         InputStream inputStream = new FileInputStream(fp);
116 |                         IOUtils.copyLarge(inputStream, outputStream);
117 |                         inputStream.close();
118 |                         fp.delete(); // delete the temp file
119 |                     }
120 |                     outputStream.close();
121 |                 } catch (IOException e) {
122 |                     e.printStackTrace();
123 |                     success = false;
124 |                 }
125 |             } else { // delete the temp files if the download was failed
126 |                 for (File fp : mFileParts) {
127 |                     fp.delete();
128 |                 }
129 |             }
130 | 
131 |             mExecutorService.shutdown();
132 | 
133 |             return success;
134 | 
135 |         } else { // download in single connection and the download will be done on the current thread
136 |             DownloaderTask downloaderTask = new DownloaderTask(url, file, 1, new Range(0, mFileSize));
137 |             boolean success;
138 |             try {
139 |                 success = downloaderTask.call();
140 |             } catch (Exception e) {
141 |                 e.printStackTrace();
142 |                 success = false;
143 |             }
144 |             if (!success) // if the download failed, delete the file
145 |                 if (file.exists())
146 |                     file.delete();
147 | 
148 |             return success;
149 |         }
150 |     }
151 | 
152 | 
153 |     private class DownloaderTask implements Callable<Boolean> {
154 | 
155 |         private URL url;
156 |         private File file;
157 |         private int part;
158 |         private Range range;
159 | 
160 |         private int retry = 0;
161 | 
162 |         public DownloaderTask(URL url, File file, int part, Range range) {
163 |             this.url = url;
164 |             this.file = file;
165 |             this.part = part;
166 |             this.range = range;
167 |         }
168 | 
169 |         @Override
170 |         public Boolean call() throws Exception {
171 | 
172 |             HttpURLConnection conn = (HttpURLConnection) url.openConnection();
173 | 
174 |             if (file.exists()) {
175 | //                System.out.println("File existed for " + Thread.currentThread().getName() + " with a size of " + file.length());
176 |                 if (file.length() >= (range.max - range.min)) {
177 |                     return true;
178 |                 } else if (retry == 0) { // Downloader was restarted, add the downloaded bytes count to the mDownloadedSize field
179 |                     mDownloadedSize.add(file.length());
180 |                 }
181 |                 range.min += file.length(); // update the min range
182 |             } else {
183 | //                System.out.println("File didn't exist for " + Thread.currentThread().getName());
184 |             }
185 | 
186 | //            System.out.println("Range for " + Thread.currentThread().getName() + " " + range.min + " " + range.max);
187 | 
188 |             conn.setRequestProperty("Range", "bytes=" + range.min + "-" + range.max);
189 | 
190 |             try (BufferedInputStream in = new BufferedInputStream(conn.getInputStream());
191 |                  FileOutputStream out = new FileOutputStream(file, true)) {
192 | 
193 | 
194 |                 int byteSize = 1024 * 50;
195 |                 byte[] dataBuffer = new byte[byteSize];
196 |                 int bytesRead;
197 |                 long bytesReadSize = 0; // TEMP
198 |                 long lastTime = System.currentTimeMillis();
199 |                 while ((bytesRead = in.read(dataBuffer, 0, byteSize)) != -1) {
200 |                     if (isInterrupted()) {
201 |                         return false;
202 |                     }
203 |                     mDownloadedSize.add(bytesRead);
204 |                     bytesReadSize += bytesRead;
205 |                     out.write(dataBuffer, 0, bytesRead);
206 |                     if ((System.currentTimeMillis() - lastTime) > 2000) {
207 |                         lastTime = System.currentTimeMillis();
208 | //                        System.out.println("Thread " + Thread.currentThread().getName() + " " + ((((float) bytesReadSize) / 1024) / 1024) / 2);
209 |                         bytesReadSize = 0;
210 |                     }
211 |                 }
212 |                 return true;
213 |             } catch (IOException e) {
214 |                 e.printStackTrace();
215 |                 if (retry < 3) {
216 |                     retry++;
217 | //                    System.out.println("Retrying for " + Thread.currentThread().getName());
218 |                     return call();
219 |                 } else {
220 | //                    System.out.println("Failed to retry for " + Thread.currentThread().getName());
221 |                     interrupt();
222 |                 }
223 |                 return false;
224 |             } finally {
225 |                 conn.disconnect();
226 |             }
227 |         }
228 |     }
229 | 
230 |     private boolean isInterrupted() {
231 |         return mInterrupt;
232 |     }
233 | 
234 |     public void interrupt() {
235 |         mInterrupt = true;
236 |     }
237 | 
238 |     private long getFileSize(HttpURLConnection conn) {
239 |         return conn.getContentLengthLong();
240 |     }
241 | 
242 |     private String getFilename(HttpURLConnection conn) {
243 | 
244 |         String filename = FilenameUtils.getName(conn.getURL().getPath());
245 | 
246 |         if (conn.getHeaderField("Content-Disposition") != null) {
247 |             String contentDisposition = conn.getHeaderField("Content-Disposition");
248 |             String reg = "(?<=filename=)([^&]*)\"";
249 | 
250 |             Pattern p = Pattern.compile(reg);
251 | 
252 |             Matcher m = p.matcher(contentDisposition);
253 |             if (m.find()) {
254 |                 String result = m.group();
255 |                 return result.substring(1, result.length() - 1);
256 |             }
257 |         }
258 | 
259 |         if (filename != null && !filename.isEmpty()) {
260 |             return filename;
261 |         } else {
262 |             return null;
263 |         }
264 |     }
265 | 
266 |     private boolean acceptsRange(HttpURLConnection conn) {
267 |         if (conn.getHeaderField("Accept-Ranges") != null) {
268 |             if (conn.getHeaderField("Accept-Ranges").equals("bytes")) {
269 |                 return true;
270 |             } else {
271 |                 return false;
272 |             }
273 |         } else {
274 |             return false;
275 |         }
276 |     }
277 | 
278 |     public static class Range {
279 |         long min;
280 |         long max;
281 | 
282 |         Range() {
283 | 
284 |         }
285 | 
286 |         Range(long min, long max) {
287 |             this.min = min;
288 |             this.max = max;
289 |         }
290 |     }
291 | 
292 |     public long getFileSize() {
293 |         return mFileSize;
294 |     }
295 | 
296 |     public String getFileName() {
297 |         return mFileName;
298 |     }
299 | 
300 |     public boolean hasRan() {
301 |         return mRan;
302 |     }
303 | 
304 |     public boolean isFinished() {
305 |         return mFinished;
306 |     }
307 | 
308 |     public boolean isSuccess() {
309 |         return mSuccess;
310 |     }
311 | 
312 |     public long getDownloadedSize() {
313 |         return mDownloadedSize.get();
314 |     }
315 | 
316 |     public String getProcessID() {
317 |         return mProcessID;
318 |     }
319 | 
320 |     public DownloadStatus getDownloadStatus() {
321 |         mDownloadStatus.setDownloadedSize(getDownloadedSize());
322 |         mDownloadStatus.setFinished(isFinished());
323 |         mDownloadStatus.setRan(hasRan());
324 |         mDownloadStatus.setSuccess(isSuccess());
325 |         return mDownloadStatus;
326 |     }
327 | 
328 |     public static class DownloadStatus {
329 |         private String processID;
330 |         private String fileName;
331 |         private long fileSize;
332 |         private long downloadedSize;
333 |         private boolean ran, finished, success;
334 | 
335 |         public DownloadStatus() {
336 | 
337 |         }
338 | 
339 |         public DownloadStatus(String processID, String fileName, long fileSize, long downloadedSize, boolean ran, boolean finished, boolean success) {
340 |             this.processID = processID;
341 |             this.fileName = fileName;
342 |             this.fileSize = fileSize;
343 |             this.downloadedSize = downloadedSize;
344 |             this.ran = ran;
345 |             this.finished = finished;
346 |             this.success = success;
347 |         }
348 | 
349 |         public void setProcessID(String processID) {
350 |             this.processID = processID;
351 |         }
352 | 
353 |         public void setFileName(String fileName) {
354 |             this.fileName = fileName;
355 |         }
356 | 
357 |         public void setFileSize(long fileSize) {
358 |             this.fileSize = fileSize;
359 |         }
360 | 
361 |         public void setDownloadedSize(long downloadedSize) {
362 |             this.downloadedSize = downloadedSize;
363 |         }
364 | 
365 |         public void setRan(boolean ran) {
366 |             this.ran = ran;
367 |         }
368 | 
369 |         public void setFinished(boolean finished) {
370 |             this.finished = finished;
371 |         }
372 | 
373 |         public void setSuccess(boolean success) {
374 |             this.success = success;
375 |         }
376 | 
377 |         public String getProcessID() {
378 |             return processID;
379 |         }
380 | 
381 |         public String getFileName() {
382 |             return fileName;
383 |         }
384 | 
385 |         public long getFileSize() {
386 |             return fileSize;
387 |         }
388 | 
389 |         public long getDownloadedSize() {
390 |             return downloadedSize;
391 |         }
392 | 
393 |         public boolean hasRan() {
394 |             return ran;
395 |         }
396 | 
397 |         public boolean isFinished() {
398 |             return finished;
399 |         }
400 | 
401 |         public boolean isSuccess() {
402 |             return success;
403 |         }
404 |     }
405 | }
406 | 


--------------------------------------------------------------------------------
/src/com/zanvent/downloader/Util.java:
--------------------------------------------------------------------------------
 1 | package com.zanvent.downloader;
 2 | 
 3 | import java.util.ArrayList;
 4 | import java.util.concurrent.atomic.AtomicLong;
 5 | 
 6 | public class Util {
 7 | 
 8 |     private static AtomicLong processCounter = new AtomicLong(1);
 9 | 
10 |     public static String createProcessID() {
11 |         return String.valueOf(processCounter.getAndIncrement());
12 |     }
13 | 
14 |     public static ArrayList<Downloader.Range> range(long length, long gap) {
15 |         ArrayList<Downloader.Range> ranges = new ArrayList<>();
16 |         long chunkSize = length / gap;
17 |         for (int i = 1; i <= gap; i++) {
18 |             Downloader.Range range = new Downloader.Range();
19 |             range.min = (chunkSize * (i - 1));
20 |             if (i != gap)
21 |                 range.max = (chunkSize * i) - 1;
22 |             else
23 |                 range.max = length;
24 | 
25 |             ranges.add(range);
26 |         }
27 |         return ranges;
28 |     }
29 | 
30 |     public static class LockBasedLong {
31 |         private long i = 0;
32 | 
33 |         public synchronized void add(long n) {
34 |             i += n;
35 |         }
36 | 
37 |         public synchronized long get() {
38 |             return i;
39 |         }
40 |     }
41 | 
42 | 
43 | }
44 | 


--------------------------------------------------------------------------------