├── DownLoadManager.java ├── DownLoadThread ├── DownloadThreadTask.java ├── README.md └── tuling ├── fileServer ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── tuling │ ├── Start.java │ ├── client │ └── DownloadClient.java │ └── controller │ └── DownLoadController.java └── pom.xml /DownLoadManager.java: -------------------------------------------------------------------------------- 1 | package com.fengye.download; 2 | 3 | import org.apache.commons.lang.exception.ExceptionUtils; 4 | import org.apache.http.impl.client.CloseableHttpClient; 5 | import org.apache.http.impl.client.HttpClients; 6 | import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import java.io.File; 10 | import java.io.IOException; 11 | import java.io.RandomAccessFile; 12 | import java.net.HttpURLConnection; 13 | import java.net.URL; 14 | import java.util.concurrent.CountDownLatch; 15 | import java.util.concurrent.ExecutorService; 16 | import java.util.concurrent.Executors; 17 | 18 | /** 19 | * @Description: 文件下载管理类 20 | * @Author: huang 21 | * @Date: 2021/10/20 23:06 22 | */ 23 | public class DownLoadManager { 24 | private static final Logger LOGGER = LoggerFactory.getLogger(DownLoadManager.class); 25 | /** 26 | * 每个线程下载的字节数 27 | */ 28 | private long unitSize = 1000 * 1024; 29 | private ExecutorService taskExecutor = Executors.newFixedThreadPool(10); 30 | 31 | private CloseableHttpClient httpClient; 32 | 33 | public DownLoadManager() { 34 | PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); 35 | cm.setMaxTotal(100); 36 | httpClient = HttpClients.custom().setConnectionManager(cm).build(); 37 | } 38 | public static void main(String[] args) throws IOException { 39 | new DownLoadManager().doDownload(); 40 | } 41 | /** 42 | * 启动多个线程下载文件 43 | */ 44 | public void doDownload() throws IOException { 45 | //要下载的url 46 | String remoteFileUrl = "https://dldir1.qq.com/qqyy/pc/QQPlayerSetup4.6.3.1104.exe"; 47 | String localPath = "E://MyTestMp3//"; 48 | String fileName = new URL(remoteFileUrl).getFile(); 49 | System.out.println("远程文件名称：" + fileName); 50 | fileName = fileName.substring(fileName.lastIndexOf("/") + 1, fileName.length()).replace("%20", " "); 51 | System.out.println("本地文件名称：" + fileName); 52 | long fileSize = this.getRemoteFileSize(remoteFileUrl); 53 | this.createFile(localPath, localPath + fileName, fileSize); 54 | Long threadCount = (fileSize / unitSize) + (fileSize % unitSize != 0 ? 1 : 0); 55 | long offset = 0; 56 | 57 | CountDownLatch end = new CountDownLatch(threadCount.intValue()); 58 | if (fileSize <= unitSize) {// 如果远程文件尺寸小于等于unitSize 59 | DownloadThreadTask downloadThread = new DownloadThreadTask(remoteFileUrl, localPath + fileName, offset, fileSize, end, httpClient); 60 | taskExecutor.execute(downloadThread); 61 | } else {// 如果远程文件尺寸大于unitSize 62 | for (int i = 1; i < threadCount; i++) { 63 | DownloadThreadTask downloadThread = new DownloadThreadTask(remoteFileUrl, localPath + fileName, offset, unitSize, end, httpClient); 64 | taskExecutor.execute(downloadThread); 65 | offset = offset + unitSize; 66 | } 67 | if (fileSize % unitSize != 0) {// 如果不能整除，则需要再创建一个线程下载剩余字节 68 | DownloadThreadTask downloadThread = new DownloadThreadTask(remoteFileUrl, localPath + fileName, offset, fileSize - unitSize * (threadCount - 1), end, httpClient); 69 | taskExecutor.execute(downloadThread); 70 | } 71 | } 72 | try { 73 | end.await(); 74 | } catch (InterruptedException e) { 75 | LOGGER.error("DownLoadManager exception msg:{}", ExceptionUtils.getFullStackTrace(e)); 76 | e.printStackTrace(); 77 | } 78 | taskExecutor.shutdown(); 79 | LOGGER.debug("下载完成！{} ", localPath + fileName); 80 | } 81 | 82 | /** 83 | * 获取远程文件尺寸 84 | */ 85 | private long getRemoteFileSize(String remoteFileUrl) throws IOException { 86 | long fileSize = 0; 87 | HttpURLConnection httpConnection = (HttpURLConnection) new URL(remoteFileUrl).openConnection(); 88 | //使用HEAD方法 89 | httpConnection.setRequestMethod("HEAD"); 90 | int responseCode = httpConnection.getResponseCode(); 91 | if (responseCode >= 400) { 92 | LOGGER.debug("Web服务器响应错误!"); 93 | return 0; 94 | } 95 | String sHeader; 96 | for (int i = 1;; i++) { 97 | sHeader = httpConnection.getHeaderFieldKey(i); 98 | if (sHeader != null && sHeader.equals("Content-Length")) { 99 | System.out.println("文件大小ContentLength:" + httpConnection.getContentLength()); 100 | fileSize = Long.parseLong(httpConnection.getHeaderField(sHeader)); 101 | break; 102 | } 103 | } 104 | return fileSize; 105 | } 106 | 107 | /** 108 | * 创建指定大小的文件 109 | */ 110 | private void createFile(String localPath, String fileName, long fileSize) throws IOException { 111 | File file = new File(localPath); 112 | if (!file.exists()) { 113 | file.mkdirs(); 114 | } 115 | File newFile = new File(fileName); 116 | RandomAccessFile raf = new RandomAccessFile(newFile, "rw"); 117 | raf.setLength(fileSize); 118 | raf.close(); 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /DownLoadThread: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /DownloadThreadTask.java: -------------------------------------------------------------------------------- 1 | package com.fengye.download; 2 | 3 | import org.apache.commons.lang.exception.ExceptionUtils; 4 | import org.apache.http.client.ClientProtocolException; 5 | import org.apache.http.client.methods.CloseableHttpResponse; 6 | import org.apache.http.client.methods.HttpGet; 7 | import org.apache.http.impl.client.CloseableHttpClient; 8 | import org.apache.http.protocol.BasicHttpContext; 9 | import org.apache.http.protocol.HttpContext; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import java.io.BufferedInputStream; 13 | import java.io.File; 14 | import java.io.IOException; 15 | import java.io.RandomAccessFile; 16 | import java.util.concurrent.CountDownLatch; 17 | 18 | /** 19 | * @Description: 20 | * @Author: huang 21 | * @Date: 2021/10/20 23:06 22 | */ 23 | public class DownloadThreadTask implements Runnable{ 24 | private static final Logger LOGGER = LoggerFactory.getLogger(DownloadThreadTask.class); 25 | 26 | /** 27 | * 待下载的文件 28 | */ 29 | private String url = null; 30 | 31 | /** 32 | * 本地文件名 33 | */ 34 | private String fileName = null; 35 | 36 | /** 37 | * 偏移量 38 | */ 39 | private long offset = 0; 40 | 41 | /** 42 | * 分配给本线程的下载字节数 43 | */ 44 | private long length = 0; 45 | 46 | private CountDownLatch end; 47 | private CloseableHttpClient httpClient; 48 | private HttpContext context; 49 | 50 | /** 51 | * @param url 52 | * 下载文件地址 53 | * @param file 另存文件名 54 | * 55 | * @param offset 56 | * 本线程下载偏移量 57 | * @param length 58 | * 本线程下载长度 59 | */ 60 | public DownloadThreadTask(String url, String file, long offset, long length, CountDownLatch end, CloseableHttpClient httpClient) { 61 | this.url = url; 62 | this.fileName = file; 63 | this.offset = offset; 64 | this.length = length; 65 | this.end = end; 66 | this.httpClient = httpClient; 67 | this.context = new BasicHttpContext(); 68 | LOGGER.debug("偏移量=" + offset + ";字节数=" + length); 69 | } 70 | 71 | 72 | @Override 73 | public void run() { 74 | try { 75 | HttpGet httpGet = new HttpGet(this.url); 76 | httpGet.addHeader("Range", "bytes=" + this.offset + "-" + (this.offset + this.length - 1)); 77 | //httpGet.addHeader("Referer", "http://api.bilibili.com"); 78 | CloseableHttpResponse response = httpClient.execute(httpGet, context); 79 | BufferedInputStream bis = new BufferedInputStream(response.getEntity().getContent()); 80 | byte[] buff = new byte[1024]; 81 | int bytesRead; 82 | File newFile = new File(fileName); 83 | RandomAccessFile raf = new RandomAccessFile(newFile, "rw"); 84 | while ((bytesRead = bis.read(buff, 0, buff.length)) != -1) { 85 | raf.seek(this.offset); 86 | raf.write(buff, 0, bytesRead); 87 | this.offset = this.offset + bytesRead; 88 | } 89 | raf.close(); 90 | bis.close(); 91 | } catch (ClientProtocolException e) { 92 | LOGGER.error("DownloadThread exception msg:{}", e.getMessage()); 93 | } catch (IOException e) { 94 | LOGGER.error("DownloadThread exception msg:{}", ExceptionUtils.getFullStackTrace(e)); 95 | } finally { 96 | end.countDown(); 97 | LOGGER.info(end.getCount() + " is go on!"); 98 | System.out.println(end.getCount() + " is go on!"); 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## BurstDownload 2 | ### 大文件(1G以上)基于SpringBoot分片下载解决方案 3 | 近期基于项目上使用到的RestTemplate下载文件流，遇到1G以上的大文件，下载需要3-4分钟，因为调用API接口没有做分片与多线程， 4 | 文件流全部采用同步方式加载，性能很慢。最近结合网上案例及自己总结，写了一个分片下载tuling/fileServer项目： 5 | 1.包含同步下载文件流在浏览器加载输出相关代码； 6 | 2.包含分片多线程下载分片文件及合并文件相关代码； 7 | 8 | 另外在DownloadThread项目中使用代码完成了一个远程RestUrl请求去获取一个远端资源大文件进行多线程分片下载 9 | 到本地的一个案例，可以下载一些诸如.mp4/.avi等视频类大文件。相关代码也一并打包上传。 10 | 11 | 代码都在本地亲测（已修复Bug）可用，目前比较欠缺的是没有实现在分片下载时对应浏览器进行下载展示，需要暂存在本地磁盘目录。 12 | 目前将代码开源，希望能有更好解决方案的Coder Fork支持！也欢迎Star捧场。 13 | 14 | 代码同步在博客园开放，欢迎点赞关注： 15 | https://www.cnblogs.com/yif0118/ 16 | -------------------------------------------------------------------------------- /tuling/fileServer/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | tuling 7 | org.example 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | fileServer 13 | 14 | 15 | 8 16 | 8 17 | 18 | 19 | 20 | 21 | org.springframework.boot 22 | spring-boot-starter-web 23 | 2.3.1.RELEASE 24 | 25 | 26 | org.springframework.boot 27 | spring-boot-actuator-autoconfigure 28 | 2.3.1.RELEASE 29 | 30 | 31 | commons-fileupload 32 | commons-fileupload 33 | 1.3.1 34 | 35 | 36 | commons-io 37 | commons-io 38 | 2.4 39 | 40 | 41 | org.apache.httpcomponents 42 | httpcore 43 | 4.4.10 44 | 45 | 46 | org.apache.httpcomponents 47 | httpclient 48 | 4.5.6 49 | 50 | 51 | -------------------------------------------------------------------------------- /tuling/fileServer/src/main/java/com/tuling/Start.java: -------------------------------------------------------------------------------- 1 | package com.tuling; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | /** 7 | * @Description: 8 | * @Author: huang 9 | * @Date: 2021/10/22 0:26 10 | */ 11 | @SpringBootApplication 12 | public class Start { 13 | public static void main(String[] args) { 14 | SpringApplication.run(Start.class, args); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /tuling/fileServer/src/main/java/com/tuling/client/DownloadClient.java: -------------------------------------------------------------------------------- 1 | package com.tuling.client; 2 | 3 | import org.apache.commons.io.FileUtils; 4 | import org.apache.http.HttpEntity; 5 | import org.apache.http.HttpResponse; 6 | import org.apache.http.client.HttpClient; 7 | import org.apache.http.client.methods.HttpGet; 8 | import org.apache.http.impl.client.HttpClients; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import org.springframework.web.bind.annotation.RequestMapping; 12 | import org.springframework.web.bind.annotation.RestController; 13 | 14 | import javax.servlet.http.HttpServletRequest; 15 | import javax.servlet.http.HttpServletResponse; 16 | import java.io.*; 17 | import java.net.HttpURLConnection; 18 | import java.net.URL; 19 | import java.net.URLDecoder; 20 | import java.util.concurrent.*; 21 | 22 | /** 23 | * @Description: 24 | * @Author: huang 25 | * @Date: 2021/10/23 12:52 26 | */ 27 | @RestController 28 | public class DownloadClient { 29 | private static final Logger LOGGER = LoggerFactory.getLogger(DownloadClient.class); 30 | private final static long PER_PAGE = 1024L * 1024L * 50L; 31 | private final static String DOWN_PATH = "F:\\fileItem"; 32 | ExecutorService taskExecutor = Executors.newFixedThreadPool(10); 33 | 34 | @RequestMapping("/downloadFile") 35 | public String downloadFile() { 36 | // 探测下载 37 | FileInfo fileInfo = download(0, 10, -1, null); 38 | if (fileInfo != null) { 39 | long pages = fileInfo.fSize / PER_PAGE; 40 | for (long i = 0; i <= pages; i++) { 41 | Future future = taskExecutor.submit(new DownloadThread(i * PER_PAGE, (i + 1) * PER_PAGE - 1, i, fileInfo.fName)); 42 | if (!future.isCancelled()) { 43 | try { 44 | fileInfo = future.get(); 45 | } catch (InterruptedException | ExecutionException e) { 46 | e.printStackTrace(); 47 | } 48 | } 49 | } 50 | return System.getProperty("user.home") + "\\Downloads\\" + fileInfo.fName; 51 | } 52 | return null; 53 | } 54 | 55 | class FileInfo { 56 | long fSize; 57 | String fName; 58 | 59 | public FileInfo(long fSize, String fName) { 60 | this.fSize = fSize; 61 | this.fName = fName; 62 | } 63 | } 64 | 65 | /** 66 | * 根据开始位置/结束位置 67 | * 分片下载文件，临时存储文件分片 68 | * 文件大小=结束位置-开始位置 69 | * 70 | * @return 71 | */ 72 | private FileInfo download(long start, long end, long page, String fName) { 73 | File dir = new File(DOWN_PATH); 74 | if (!dir.exists()) { 75 | dir.mkdirs(); 76 | } 77 | // 断点下载 78 | File file = new File(DOWN_PATH, page + "-" + fName); 79 | if (file.exists() && page != -1 && file.length() == PER_PAGE) { 80 | return null; 81 | } 82 | try { 83 | HttpClient client = HttpClients.createDefault(); 84 | HttpGet httpGet = new HttpGet("http://127.0.0.1:8080/download"); 85 | httpGet.setHeader("Range", "bytes=" + start + "-" + end); 86 | HttpResponse response = client.execute(httpGet); 87 | String fSize = response.getFirstHeader("fSize").getValue(); 88 | fName = URLDecoder.decode(response.getFirstHeader("fName").getValue(), "UTF-8"); 89 | HttpEntity entity = response.getEntity(); 90 | InputStream is = entity.getContent(); 91 | FileOutputStream fos = new FileOutputStream(file); 92 | byte[] buffer = new byte[1024]; 93 | int ch; 94 | while ((ch = is.read(buffer)) != -1) { 95 | fos.write(buffer, 0, ch); 96 | } 97 | is.close(); 98 | fos.flush(); 99 | fos.close(); 100 | // 最后一个分片 101 | if (end - Long.parseLong(fSize) > 0) { 102 | // 开始合并文件 103 | mergeFile(fName, page); 104 | } 105 | 106 | return new FileInfo(Long.parseLong(fSize), fName); 107 | } catch (IOException e) { 108 | e.printStackTrace(); 109 | } 110 | return null; 111 | } 112 | 113 | private void mergeFile(String fName, long page) { 114 | File file = new File(DOWN_PATH, fName); 115 | try { 116 | BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(file)); 117 | for (long i = 0; i <= page; i++) { 118 | File tempFile = new File(DOWN_PATH, i + "-" + fName); 119 | while (!file.exists() || (i != page && tempFile.length() < PER_PAGE)) { 120 | try { 121 | Thread.sleep(100); 122 | } catch (InterruptedException e) { 123 | e.printStackTrace(); 124 | } 125 | } 126 | byte[] bytes = FileUtils.readFileToByteArray(tempFile); 127 | os.write(bytes); 128 | os.flush(); 129 | tempFile.delete(); 130 | } 131 | File testFile = new File(DOWN_PATH, -1 + "-null"); 132 | testFile.delete(); 133 | os.flush(); 134 | os.close(); 135 | } catch (IOException e) { 136 | e.printStackTrace(); 137 | } 138 | } 139 | 140 | /** 141 | * 获取远程文件尺寸 142 | */ 143 | private long getRemoteFileSize(String remoteFileUrl) throws IOException { 144 | long fileSize = 0; 145 | HttpURLConnection httpConnection = (HttpURLConnection) new URL(remoteFileUrl).openConnection(); 146 | //使用HEAD方法 147 | httpConnection.setRequestMethod("HEAD"); 148 | int responseCode = httpConnection.getResponseCode(); 149 | if (responseCode >= 400) { 150 | LOGGER.debug("Web服务器响应错误!"); 151 | return 0; 152 | } 153 | String sHeader; 154 | for (int i = 1;; i++) { 155 | sHeader = httpConnection.getHeaderFieldKey(i); 156 | if (sHeader != null && sHeader.equals("Content-Length")) { 157 | LOGGER.debug("文件大小ContentLength:" + httpConnection.getContentLength()); 158 | fileSize = Long.parseLong(httpConnection.getHeaderField(sHeader)); 159 | break; 160 | } 161 | } 162 | return fileSize; 163 | } 164 | 165 | class DownloadThread implements Callable { 166 | long start; 167 | long end; 168 | long page; 169 | String fName; 170 | 171 | public DownloadThread(long start, long end, long page, String fName) { 172 | this.start = start; 173 | this.end = end; 174 | this.page = page; 175 | this.fName = fName; 176 | } 177 | 178 | @Override 179 | public FileInfo call() { 180 | return download(start, end, page, fName); 181 | } 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /tuling/fileServer/src/main/java/com/tuling/controller/DownLoadController.java: -------------------------------------------------------------------------------- 1 | package com.tuling.controller; 2 | 3 | import org.springframework.stereotype.Controller; 4 | import org.springframework.web.bind.annotation.RequestMapping; 5 | import javax.servlet.http.HttpServletRequest; 6 | import javax.servlet.http.HttpServletResponse; 7 | import java.io.*; 8 | import java.net.URLEncoder; 9 | 10 | /** 11 | * @Description: 12 | * @Author: huang 13 | * @Date: 2021/10/22 7:16 14 | */ 15 | @Controller 16 | public class DownLoadController { 17 | private static final String UTF8 = "UTF-8"; 18 | @RequestMapping("/download") 19 | public void downLoadFile(HttpServletRequest request, HttpServletResponse response) throws IOException { 20 | File file = new File("D:\\DevTools\\ideaIU-2021.1.3.exe"); 21 | response.setCharacterEncoding(UTF8); 22 | InputStream is = null; 23 | OutputStream os = null; 24 | try { 25 | // 分片下载 Range表示方式 bytes=100-1000 100- 26 | long fSize = file.length(); 27 | response.setContentType("application/x-download"); 28 | String fileName = URLEncoder.encode(file.getName(), UTF8); 29 | response.addHeader("Content-Disposition", "attachment;filename=" + fileName); 30 | // 支持分片下载 31 | response.setHeader("Accept-Range", "bytes"); 32 | response.setHeader("fSize", String.valueOf(fSize)); 33 | response.setHeader("fName", fileName); 34 | 35 | long pos = 0, last = fSize - 1, sum = 0; 36 | if (null != request.getHeader("Range")) { 37 | response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); 38 | String numberRange = request.getHeader("Range").replaceAll("bytes=", ""); 39 | String[] strRange = numberRange.split("-"); 40 | if (strRange.length == 2) { 41 | pos = Long.parseLong(strRange[0].trim()); 42 | last = Long.parseLong(strRange[1].trim()); 43 | if (last > fSize-1) { 44 | last = fSize - 1; 45 | } 46 | } else { 47 | pos = Long.parseLong(numberRange.replaceAll("-", "").trim()); 48 | } 49 | } 50 | long rangeLength = last - pos + 1; 51 | String contentRange = new StringBuffer("bytes").append(pos).append("-").append(last).append("/").append(fSize).toString(); 52 | response.setHeader("Content-Range", contentRange); 53 | response.setHeader("Content-Length", String.valueOf(rangeLength)); 54 | 55 | os = new BufferedOutputStream(response.getOutputStream()); 56 | is = new BufferedInputStream(new FileInputStream(file)); 57 | is.skip(pos); 58 | byte[] buffer = new byte[1024]; 59 | int length = 0; 60 | while (sum < rangeLength) { 61 | int readLength = (int) (rangeLength - sum); 62 | length = is.read(buffer, 0, (rangeLength - sum) <= buffer.length ? readLength : buffer.length); 63 | sum += length; 64 | os.write(buffer,0, length); 65 | } 66 | System.out.println("下载完成"); 67 | }finally { 68 | if (is != null){ 69 | is.close(); 70 | } 71 | if (os != null){ 72 | os.close(); 73 | } 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /tuling/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.example 8 | tuling 9 | pom 10 | 1.0-SNAPSHOT 11 | 12 | fileServer 13 | 14 | 15 | 16 | 8 17 | 8 18 | 19 | 20 | --------------------------------------------------------------------------------