├── DownLoadManager.java
├── DownLoadThread
├── DownloadThreadTask.java
├── README.md
└── tuling
├── fileServer
├── pom.xml
└── src
│ └── main
│ └── java
│ └── com
│ └── tuling
│ ├── Start.java
│ ├── client
│ └── DownloadClient.java
│ └── controller
│ └── DownLoadController.java
└── pom.xml
/DownLoadManager.java:
--------------------------------------------------------------------------------
1 | package com.fengye.download;
2 |
3 | import org.apache.commons.lang.exception.ExceptionUtils;
4 | import org.apache.http.impl.client.CloseableHttpClient;
5 | import org.apache.http.impl.client.HttpClients;
6 | import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
7 | import org.slf4j.Logger;
8 | import org.slf4j.LoggerFactory;
9 | import java.io.File;
10 | import java.io.IOException;
11 | import java.io.RandomAccessFile;
12 | import java.net.HttpURLConnection;
13 | import java.net.URL;
14 | import java.util.concurrent.CountDownLatch;
15 | import java.util.concurrent.ExecutorService;
16 | import java.util.concurrent.Executors;
17 |
18 | /**
19 | * @Description: 文件下载管理类
20 | * @Author: huang
21 | * @Date: 2021/10/20 23:06
22 | */
23 | public class DownLoadManager {
24 | private static final Logger LOGGER = LoggerFactory.getLogger(DownLoadManager.class);
25 | /**
26 | * 每个线程下载的字节数
27 | */
28 | private long unitSize = 1000 * 1024;
29 | private ExecutorService taskExecutor = Executors.newFixedThreadPool(10);
30 |
31 | private CloseableHttpClient httpClient;
32 |
33 | public DownLoadManager() {
34 | PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
35 | cm.setMaxTotal(100);
36 | httpClient = HttpClients.custom().setConnectionManager(cm).build();
37 | }
38 | public static void main(String[] args) throws IOException {
39 | new DownLoadManager().doDownload();
40 | }
41 | /**
42 | * 启动多个线程下载文件
43 | */
44 | public void doDownload() throws IOException {
45 | //要下载的url
46 | String remoteFileUrl = "https://dldir1.qq.com/qqyy/pc/QQPlayerSetup4.6.3.1104.exe";
47 | String localPath = "E://MyTestMp3//";
48 | String fileName = new URL(remoteFileUrl).getFile();
49 | System.out.println("远程文件名称:" + fileName);
50 | fileName = fileName.substring(fileName.lastIndexOf("/") + 1, fileName.length()).replace("%20", " ");
51 | System.out.println("本地文件名称:" + fileName);
52 | long fileSize = this.getRemoteFileSize(remoteFileUrl);
53 | this.createFile(localPath, localPath + fileName, fileSize);
54 | Long threadCount = (fileSize / unitSize) + (fileSize % unitSize != 0 ? 1 : 0);
55 | long offset = 0;
56 |
57 | CountDownLatch end = new CountDownLatch(threadCount.intValue());
58 | if (fileSize <= unitSize) {// 如果远程文件尺寸小于等于unitSize
59 | DownloadThreadTask downloadThread = new DownloadThreadTask(remoteFileUrl, localPath + fileName, offset, fileSize, end, httpClient);
60 | taskExecutor.execute(downloadThread);
61 | } else {// 如果远程文件尺寸大于unitSize
62 | for (int i = 1; i < threadCount; i++) {
63 | DownloadThreadTask downloadThread = new DownloadThreadTask(remoteFileUrl, localPath + fileName, offset, unitSize, end, httpClient);
64 | taskExecutor.execute(downloadThread);
65 | offset = offset + unitSize;
66 | }
67 | if (fileSize % unitSize != 0) {// 如果不能整除,则需要再创建一个线程下载剩余字节
68 | DownloadThreadTask downloadThread = new DownloadThreadTask(remoteFileUrl, localPath + fileName, offset, fileSize - unitSize * (threadCount - 1), end, httpClient);
69 | taskExecutor.execute(downloadThread);
70 | }
71 | }
72 | try {
73 | end.await();
74 | } catch (InterruptedException e) {
75 | LOGGER.error("DownLoadManager exception msg:{}", ExceptionUtils.getFullStackTrace(e));
76 | e.printStackTrace();
77 | }
78 | taskExecutor.shutdown();
79 | LOGGER.debug("下载完成!{} ", localPath + fileName);
80 | }
81 |
82 | /**
83 | * 获取远程文件尺寸
84 | */
85 | private long getRemoteFileSize(String remoteFileUrl) throws IOException {
86 | long fileSize = 0;
87 | HttpURLConnection httpConnection = (HttpURLConnection) new URL(remoteFileUrl).openConnection();
88 | //使用HEAD方法
89 | httpConnection.setRequestMethod("HEAD");
90 | int responseCode = httpConnection.getResponseCode();
91 | if (responseCode >= 400) {
92 | LOGGER.debug("Web服务器响应错误!");
93 | return 0;
94 | }
95 | String sHeader;
96 | for (int i = 1;; i++) {
97 | sHeader = httpConnection.getHeaderFieldKey(i);
98 | if (sHeader != null && sHeader.equals("Content-Length")) {
99 | System.out.println("文件大小ContentLength:" + httpConnection.getContentLength());
100 | fileSize = Long.parseLong(httpConnection.getHeaderField(sHeader));
101 | break;
102 | }
103 | }
104 | return fileSize;
105 | }
106 |
107 | /**
108 | * 创建指定大小的文件
109 | */
110 | private void createFile(String localPath, String fileName, long fileSize) throws IOException {
111 | File file = new File(localPath);
112 | if (!file.exists()) {
113 | file.mkdirs();
114 | }
115 | File newFile = new File(fileName);
116 | RandomAccessFile raf = new RandomAccessFile(newFile, "rw");
117 | raf.setLength(fileSize);
118 | raf.close();
119 | }
120 | }
121 |
--------------------------------------------------------------------------------
/DownLoadThread:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/DownloadThreadTask.java:
--------------------------------------------------------------------------------
1 | package com.fengye.download;
2 |
3 | import org.apache.commons.lang.exception.ExceptionUtils;
4 | import org.apache.http.client.ClientProtocolException;
5 | import org.apache.http.client.methods.CloseableHttpResponse;
6 | import org.apache.http.client.methods.HttpGet;
7 | import org.apache.http.impl.client.CloseableHttpClient;
8 | import org.apache.http.protocol.BasicHttpContext;
9 | import org.apache.http.protocol.HttpContext;
10 | import org.slf4j.Logger;
11 | import org.slf4j.LoggerFactory;
12 | import java.io.BufferedInputStream;
13 | import java.io.File;
14 | import java.io.IOException;
15 | import java.io.RandomAccessFile;
16 | import java.util.concurrent.CountDownLatch;
17 |
18 | /**
19 | * @Description:
20 | * @Author: huang
21 | * @Date: 2021/10/20 23:06
22 | */
23 | public class DownloadThreadTask implements Runnable{
24 | private static final Logger LOGGER = LoggerFactory.getLogger(DownloadThreadTask.class);
25 |
26 | /**
27 | * 待下载的文件
28 | */
29 | private String url = null;
30 |
31 | /**
32 | * 本地文件名
33 | */
34 | private String fileName = null;
35 |
36 | /**
37 | * 偏移量
38 | */
39 | private long offset = 0;
40 |
41 | /**
42 | * 分配给本线程的下载字节数
43 | */
44 | private long length = 0;
45 |
46 | private CountDownLatch end;
47 | private CloseableHttpClient httpClient;
48 | private HttpContext context;
49 |
50 | /**
51 | * @param url
52 | * 下载文件地址
53 | * @param file 另存文件名
54 | *
55 | * @param offset
56 | * 本线程下载偏移量
57 | * @param length
58 | * 本线程下载长度
59 | */
60 | public DownloadThreadTask(String url, String file, long offset, long length, CountDownLatch end, CloseableHttpClient httpClient) {
61 | this.url = url;
62 | this.fileName = file;
63 | this.offset = offset;
64 | this.length = length;
65 | this.end = end;
66 | this.httpClient = httpClient;
67 | this.context = new BasicHttpContext();
68 | LOGGER.debug("偏移量=" + offset + ";字节数=" + length);
69 | }
70 |
71 |
72 | @Override
73 | public void run() {
74 | try {
75 | HttpGet httpGet = new HttpGet(this.url);
76 | httpGet.addHeader("Range", "bytes=" + this.offset + "-" + (this.offset + this.length - 1));
77 | //httpGet.addHeader("Referer", "http://api.bilibili.com");
78 | CloseableHttpResponse response = httpClient.execute(httpGet, context);
79 | BufferedInputStream bis = new BufferedInputStream(response.getEntity().getContent());
80 | byte[] buff = new byte[1024];
81 | int bytesRead;
82 | File newFile = new File(fileName);
83 | RandomAccessFile raf = new RandomAccessFile(newFile, "rw");
84 | while ((bytesRead = bis.read(buff, 0, buff.length)) != -1) {
85 | raf.seek(this.offset);
86 | raf.write(buff, 0, bytesRead);
87 | this.offset = this.offset + bytesRead;
88 | }
89 | raf.close();
90 | bis.close();
91 | } catch (ClientProtocolException e) {
92 | LOGGER.error("DownloadThread exception msg:{}", e.getMessage());
93 | } catch (IOException e) {
94 | LOGGER.error("DownloadThread exception msg:{}", ExceptionUtils.getFullStackTrace(e));
95 | } finally {
96 | end.countDown();
97 | LOGGER.info(end.getCount() + " is go on!");
98 | System.out.println(end.getCount() + " is go on!");
99 | }
100 | }
101 | }
102 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | ## BurstDownload
2 | ### 大文件(1G以上)基于SpringBoot分片下载解决方案
3 | 近期基于项目上使用到的RestTemplate下载文件流,遇到1G以上的大文件,下载需要3-4分钟,因为调用API接口没有做分片与多线程,
4 | 文件流全部采用同步方式加载,性能很慢。最近结合网上案例及自己总结,写了一个分片下载tuling/fileServer项目:
5 | 1.包含同步下载文件流在浏览器加载输出相关代码;
6 | 2.包含分片多线程下载分片文件及合并文件相关代码;
7 |
8 | 另外在DownloadThread项目中使用代码完成了一个远程RestUrl请求去获取一个远端资源大文件进行多线程分片下载
9 | 到本地的一个案例,可以下载一些诸如.mp4/.avi等视频类大文件。相关代码也一并打包上传。
10 |
11 | 代码都在本地亲测(已修复Bug)可用,目前比较欠缺的是没有实现在分片下载时对应浏览器进行下载展示,需要暂存在本地磁盘目录。
12 | 目前将代码开源,希望能有更好解决方案的Coder Fork支持!也欢迎Star捧场。
13 |
14 | 代码同步在博客园开放,欢迎点赞关注:
15 | https://www.cnblogs.com/yif0118/
16 |
--------------------------------------------------------------------------------
/tuling/fileServer/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
5 |
6 | tuling
7 | org.example
8 | 1.0-SNAPSHOT
9 |
10 | 4.0.0
11 |
12 | fileServer
13 |
14 |
15 | 8
16 | 8
17 |
18 |
19 |
20 |
21 | org.springframework.boot
22 | spring-boot-starter-web
23 | 2.3.1.RELEASE
24 |
25 |
26 | org.springframework.boot
27 | spring-boot-actuator-autoconfigure
28 | 2.3.1.RELEASE
29 |
30 |
31 | commons-fileupload
32 | commons-fileupload
33 | 1.3.1
34 |
35 |
36 | commons-io
37 | commons-io
38 | 2.4
39 |
40 |
41 | org.apache.httpcomponents
42 | httpcore
43 | 4.4.10
44 |
45 |
46 | org.apache.httpcomponents
47 | httpclient
48 | 4.5.6
49 |
50 |
51 |
--------------------------------------------------------------------------------
/tuling/fileServer/src/main/java/com/tuling/Start.java:
--------------------------------------------------------------------------------
1 | package com.tuling;
2 |
3 | import org.springframework.boot.SpringApplication;
4 | import org.springframework.boot.autoconfigure.SpringBootApplication;
5 |
6 | /**
7 | * @Description:
8 | * @Author: huang
9 | * @Date: 2021/10/22 0:26
10 | */
11 | @SpringBootApplication
12 | public class Start {
13 | public static void main(String[] args) {
14 | SpringApplication.run(Start.class, args);
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/tuling/fileServer/src/main/java/com/tuling/client/DownloadClient.java:
--------------------------------------------------------------------------------
1 | package com.tuling.client;
2 |
3 | import org.apache.commons.io.FileUtils;
4 | import org.apache.http.HttpEntity;
5 | import org.apache.http.HttpResponse;
6 | import org.apache.http.client.HttpClient;
7 | import org.apache.http.client.methods.HttpGet;
8 | import org.apache.http.impl.client.HttpClients;
9 | import org.slf4j.Logger;
10 | import org.slf4j.LoggerFactory;
11 | import org.springframework.web.bind.annotation.RequestMapping;
12 | import org.springframework.web.bind.annotation.RestController;
13 |
14 | import javax.servlet.http.HttpServletRequest;
15 | import javax.servlet.http.HttpServletResponse;
16 | import java.io.*;
17 | import java.net.HttpURLConnection;
18 | import java.net.URL;
19 | import java.net.URLDecoder;
20 | import java.util.concurrent.*;
21 |
22 | /**
23 | * @Description:
24 | * @Author: huang
25 | * @Date: 2021/10/23 12:52
26 | */
27 | @RestController
28 | public class DownloadClient {
29 | private static final Logger LOGGER = LoggerFactory.getLogger(DownloadClient.class);
30 | private final static long PER_PAGE = 1024L * 1024L * 50L;
31 | private final static String DOWN_PATH = "F:\\fileItem";
32 | ExecutorService taskExecutor = Executors.newFixedThreadPool(10);
33 |
34 | @RequestMapping("/downloadFile")
35 | public String downloadFile() {
36 | // 探测下载
37 | FileInfo fileInfo = download(0, 10, -1, null);
38 | if (fileInfo != null) {
39 | long pages = fileInfo.fSize / PER_PAGE;
40 | for (long i = 0; i <= pages; i++) {
41 | Future future = taskExecutor.submit(new DownloadThread(i * PER_PAGE, (i + 1) * PER_PAGE - 1, i, fileInfo.fName));
42 | if (!future.isCancelled()) {
43 | try {
44 | fileInfo = future.get();
45 | } catch (InterruptedException | ExecutionException e) {
46 | e.printStackTrace();
47 | }
48 | }
49 | }
50 | return System.getProperty("user.home") + "\\Downloads\\" + fileInfo.fName;
51 | }
52 | return null;
53 | }
54 |
55 | class FileInfo {
56 | long fSize;
57 | String fName;
58 |
59 | public FileInfo(long fSize, String fName) {
60 | this.fSize = fSize;
61 | this.fName = fName;
62 | }
63 | }
64 |
65 | /**
66 | * 根据开始位置/结束位置
67 | * 分片下载文件,临时存储文件分片
68 | * 文件大小=结束位置-开始位置
69 | *
70 | * @return
71 | */
72 | private FileInfo download(long start, long end, long page, String fName) {
73 | File dir = new File(DOWN_PATH);
74 | if (!dir.exists()) {
75 | dir.mkdirs();
76 | }
77 | // 断点下载
78 | File file = new File(DOWN_PATH, page + "-" + fName);
79 | if (file.exists() && page != -1 && file.length() == PER_PAGE) {
80 | return null;
81 | }
82 | try {
83 | HttpClient client = HttpClients.createDefault();
84 | HttpGet httpGet = new HttpGet("http://127.0.0.1:8080/download");
85 | httpGet.setHeader("Range", "bytes=" + start + "-" + end);
86 | HttpResponse response = client.execute(httpGet);
87 | String fSize = response.getFirstHeader("fSize").getValue();
88 | fName = URLDecoder.decode(response.getFirstHeader("fName").getValue(), "UTF-8");
89 | HttpEntity entity = response.getEntity();
90 | InputStream is = entity.getContent();
91 | FileOutputStream fos = new FileOutputStream(file);
92 | byte[] buffer = new byte[1024];
93 | int ch;
94 | while ((ch = is.read(buffer)) != -1) {
95 | fos.write(buffer, 0, ch);
96 | }
97 | is.close();
98 | fos.flush();
99 | fos.close();
100 | // 最后一个分片
101 | if (end - Long.parseLong(fSize) > 0) {
102 | // 开始合并文件
103 | mergeFile(fName, page);
104 | }
105 |
106 | return new FileInfo(Long.parseLong(fSize), fName);
107 | } catch (IOException e) {
108 | e.printStackTrace();
109 | }
110 | return null;
111 | }
112 |
113 | private void mergeFile(String fName, long page) {
114 | File file = new File(DOWN_PATH, fName);
115 | try {
116 | BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(file));
117 | for (long i = 0; i <= page; i++) {
118 | File tempFile = new File(DOWN_PATH, i + "-" + fName);
119 | while (!file.exists() || (i != page && tempFile.length() < PER_PAGE)) {
120 | try {
121 | Thread.sleep(100);
122 | } catch (InterruptedException e) {
123 | e.printStackTrace();
124 | }
125 | }
126 | byte[] bytes = FileUtils.readFileToByteArray(tempFile);
127 | os.write(bytes);
128 | os.flush();
129 | tempFile.delete();
130 | }
131 | File testFile = new File(DOWN_PATH, -1 + "-null");
132 | testFile.delete();
133 | os.flush();
134 | os.close();
135 | } catch (IOException e) {
136 | e.printStackTrace();
137 | }
138 | }
139 |
140 | /**
141 | * 获取远程文件尺寸
142 | */
143 | private long getRemoteFileSize(String remoteFileUrl) throws IOException {
144 | long fileSize = 0;
145 | HttpURLConnection httpConnection = (HttpURLConnection) new URL(remoteFileUrl).openConnection();
146 | //使用HEAD方法
147 | httpConnection.setRequestMethod("HEAD");
148 | int responseCode = httpConnection.getResponseCode();
149 | if (responseCode >= 400) {
150 | LOGGER.debug("Web服务器响应错误!");
151 | return 0;
152 | }
153 | String sHeader;
154 | for (int i = 1;; i++) {
155 | sHeader = httpConnection.getHeaderFieldKey(i);
156 | if (sHeader != null && sHeader.equals("Content-Length")) {
157 | LOGGER.debug("文件大小ContentLength:" + httpConnection.getContentLength());
158 | fileSize = Long.parseLong(httpConnection.getHeaderField(sHeader));
159 | break;
160 | }
161 | }
162 | return fileSize;
163 | }
164 |
165 | class DownloadThread implements Callable {
166 | long start;
167 | long end;
168 | long page;
169 | String fName;
170 |
171 | public DownloadThread(long start, long end, long page, String fName) {
172 | this.start = start;
173 | this.end = end;
174 | this.page = page;
175 | this.fName = fName;
176 | }
177 |
178 | @Override
179 | public FileInfo call() {
180 | return download(start, end, page, fName);
181 | }
182 | }
183 | }
184 |
--------------------------------------------------------------------------------
/tuling/fileServer/src/main/java/com/tuling/controller/DownLoadController.java:
--------------------------------------------------------------------------------
1 | package com.tuling.controller;
2 |
3 | import org.springframework.stereotype.Controller;
4 | import org.springframework.web.bind.annotation.RequestMapping;
5 | import javax.servlet.http.HttpServletRequest;
6 | import javax.servlet.http.HttpServletResponse;
7 | import java.io.*;
8 | import java.net.URLEncoder;
9 |
10 | /**
11 | * @Description:
12 | * @Author: huang
13 | * @Date: 2021/10/22 7:16
14 | */
15 | @Controller
16 | public class DownLoadController {
17 | private static final String UTF8 = "UTF-8";
18 | @RequestMapping("/download")
19 | public void downLoadFile(HttpServletRequest request, HttpServletResponse response) throws IOException {
20 | File file = new File("D:\\DevTools\\ideaIU-2021.1.3.exe");
21 | response.setCharacterEncoding(UTF8);
22 | InputStream is = null;
23 | OutputStream os = null;
24 | try {
25 | // 分片下载 Range表示方式 bytes=100-1000 100-
26 | long fSize = file.length();
27 | response.setContentType("application/x-download");
28 | String fileName = URLEncoder.encode(file.getName(), UTF8);
29 | response.addHeader("Content-Disposition", "attachment;filename=" + fileName);
30 | // 支持分片下载
31 | response.setHeader("Accept-Range", "bytes");
32 | response.setHeader("fSize", String.valueOf(fSize));
33 | response.setHeader("fName", fileName);
34 |
35 | long pos = 0, last = fSize - 1, sum = 0;
36 | if (null != request.getHeader("Range")) {
37 | response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
38 | String numberRange = request.getHeader("Range").replaceAll("bytes=", "");
39 | String[] strRange = numberRange.split("-");
40 | if (strRange.length == 2) {
41 | pos = Long.parseLong(strRange[0].trim());
42 | last = Long.parseLong(strRange[1].trim());
43 | if (last > fSize-1) {
44 | last = fSize - 1;
45 | }
46 | } else {
47 | pos = Long.parseLong(numberRange.replaceAll("-", "").trim());
48 | }
49 | }
50 | long rangeLength = last - pos + 1;
51 | String contentRange = new StringBuffer("bytes").append(pos).append("-").append(last).append("/").append(fSize).toString();
52 | response.setHeader("Content-Range", contentRange);
53 | response.setHeader("Content-Length", String.valueOf(rangeLength));
54 |
55 | os = new BufferedOutputStream(response.getOutputStream());
56 | is = new BufferedInputStream(new FileInputStream(file));
57 | is.skip(pos);
58 | byte[] buffer = new byte[1024];
59 | int length = 0;
60 | while (sum < rangeLength) {
61 | int readLength = (int) (rangeLength - sum);
62 | length = is.read(buffer, 0, (rangeLength - sum) <= buffer.length ? readLength : buffer.length);
63 | sum += length;
64 | os.write(buffer,0, length);
65 | }
66 | System.out.println("下载完成");
67 | }finally {
68 | if (is != null){
69 | is.close();
70 | }
71 | if (os != null){
72 | os.close();
73 | }
74 | }
75 | }
76 | }
77 |
--------------------------------------------------------------------------------
/tuling/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
5 | 4.0.0
6 |
7 | org.example
8 | tuling
9 | pom
10 | 1.0-SNAPSHOT
11 |
12 | fileServer
13 |
14 |
15 |
16 | 8
17 | 8
18 |
19 |
20 |
--------------------------------------------------------------------------------