大文件上傳下載實現思路，分片、斷點續傳代碼實現
VSole2023-05-15 09:43:09
大文件上傳
實現思路：
- 分片：按照自定義緩沖區大小，將大文件分成多個小文件片段。
- 斷點續傳：根據分片數量，給每個小文件通過循環起對應名稱，當文件下載中斷在續傳時，判斷小文件名稱若存在則不存了，此時還需要判斷文件若不是最后一個分片則大小為緩沖區固定大小，若沒達到則證明小文件沒傳完需要重新傳輸。
- 合并：下載時通過線程池創建任務進行下載或上傳、當判斷最后一個分片傳完時，調用合并方法，根據之前定義的文件名稱順序進行合并，肯能出現最后一個分片傳完，之前分片未傳完的情況，需要使用while循環進行判斷，多文件未傳輸完，則等待一會繼續判斷。
- 大文件秒傳：實際上是根據文件名稱區一個唯一的md5值存儲，傳文件時進行判斷，若存在則不傳。
創建springboot項目，添加依賴
<dependencies> <dependency> <groupId>org.springframework.bootgroupId> <artifactId>spring-boot-starterartifactId> dependency> <dependency> <groupId>org.springframework.bootgroupId> <artifactId>spring-boot-starter-testartifactId> <scope>testscope> dependency> <dependency> <groupId>org.springframework.bootgroupId> <artifactId>spring-boot-autoconfigureartifactId> dependency> <dependency> <groupId>commons-fileuploadgroupId> <artifactId>commons-fileuploadartifactId> <version>1.3.1version> dependency> <dependency> <groupId>commons-iogroupId> <artifactId>commons-ioartifactId> <version>2.4version> dependency> <dependency> <groupId>org.apache.httpcomponentsgroupId> <artifactId>httpcoreartifactId> dependency> <dependency> <groupId>org.apache.httpcomponentsgroupId> <artifactId>httpclientartifactId> dependency> <dependency> <groupId>org.springframework.bootgroupId> <artifactId>spring-boot-starter-webartifactId> dependency> dependencies>
編寫測試環境看環境有沒有搭建成功
@Controller
public class UploadController {
@RequestMapping("/up")
@ResponseBody
public String upload(HttpServletRequest request, HttpServletResponse response){
return "搭建成功";
}
}
頁面主要代碼
<body>
<div id="upload-container">
<span>上傳span>
div>
<div id="upload-list">div>
<button id="picker">點擊上傳button>
body>
<script>
$('#upload-container').click(function (event){
$("#picker").find('input').click();
});
var uploader = WebUploader.create({
auto: true,
swf : 'Uploader.swf', //swf文件路徑
server: 'http://localhost:8080/upload',
dnd: '#upload-container',
pick: '#picker', //內部根據當前運行創建
multiple: true, //選擇多個
chunked: true, //開啟分片
threads: 20, //并發數
method: 'POST',
fileSizeLimit: 1024*1024*1024*10, //單個文件大小限制
fileSingleSizeLimit: 1024*1024*1024, //總文件大小
fileVal: 'upload'
});
uploader.on("beforeFileQueued",function (file){
console.log(file); //獲取文件后綴
});
uploader.on('fileQueued',function (file){
//選中文件要做的事
console.log(file.ext);
console.log(file.size);
console.log(file.name);
var html = '文件名：'+file.name+''"class="btn-delete">刪除'"class="btn-retry">重試+file.id+'" style="width: 0%;">'
$('#upload-list').append(html);
uploader.md5File(file) //給文件定義唯一的md5值，當再次上傳相同文件時，就不用傳了 大文件秒傳實際上是沒傳，直接拷貝之前文件地址
//顯示進度
.progress(function (percentage){
console.log('Percentage:',percentage);
})
//完成
.then(function (val){
console.log('md5 result',val);
});
});
webUpload組件支持分片上傳：利用多進程并發上傳，將大文件拆分成一個一個的小文件，每一個小文件屬于大文件的一個分片
斷點續傳實現：后端代碼
@Controller
public class UploadController {
private final static String utf8 = "utf-8";
@RequestMapping("/up")
@ResponseBody
public void upload(HttpServletRequest request, HttpServletResponse response) throws Exception {
response.setCharacterEncoding(utf8);
//長傳時候會有多個分片，需要記錄當前為那個分片
Integer schunk = null;
//總分片數
Integer schunks = null;
//名字
String name = null;
//文件目錄
String path = "D:\\file";
BufferedOutputStream os = null;
try {
//設置緩沖區大小 先讀到內存里在從內存寫
DiskFileItemFactory factory = new DiskFileItemFactory();
factory.setSizeThreshold(1024);
factory.setRepository(new File(path));
//解析
ServletFileUpload upload = new ServletFileUpload(factory);
//設置單個大小與最大大小
upload.setFileSizeMax(5l*1024l*1024l*1024l);
upload.setSizeMax(10l*1024l*1024l*1024l);
List items = upload.parseRequest(request);
for (FileItem item : items){
if (item.isFormField()){
//獲取分片數賦值給遍量
if ("chunk".equals(item.getFieldName())){
schunk = Integer.parseInt(item.getString(utf8));
}
if ("chunks".equals(item.getFieldName())){
schunks = Integer.parseInt(item.getString(utf8));
}
if ("name".equals(item.getFieldName())){
name = item.getString(utf8);
}
}
}
//取出文件基本信息后
for (FileItem item : items){
if (!item.isFormField()){
//有分片需要臨時目錄
String temFileName = name;
if (name != null){
if (schunk != null){
temFileName = schunk+"_"+name;
}
//判斷文件是否存在
File temfile = new File(path, temFileName);
//斷點續傳 判斷文件是否存在，若存在則不傳
if (!temfile.exists()){
item.write(temfile);
}
}
}
}
//文件合并 當前分片為最后一個就合并
if (schunk != null && schunk.intValue()== schunks.intValue()-1){
File tempFile = new File(path, name);
os = new BufferedOutputStream(new FileOutputStream(tempFile));
//根據之前命名規則找到所有分片
for (int i = 0; i < schunks; i++) {
File file = new File(path, i + "_" + name);
//并發情況 需要判斷所有 因為可能最后一個分片傳完，之前有的還沒傳完
while (!file.exists()){
//不存在休眠100毫秒后在從新判斷
Thread.sleep(100);
}
//分片存在 讀入數組中
byte[] bytes = FileUtils.readFileToByteArray(file);
os.write(bytes);
os.flush();
file.delete();
}
os.flush();
}
response.getWriter().write("上傳成功");
}finally {
try {
if (os != null){
os.close();
}
}catch (IOException e){
e.printStackTrace();
}
}
}
}
文件分片下載服務端
@Controller
public class DownLoadController {
private final static String utf8 = "utf-8";
@RequestMapping("/down")
public void downLoadFile(HttpServletRequest request, HttpServletResponse response) throws IOException {
response.setCharacterEncoding(utf8);
//定義文件路徑
File file = new File("D:\\File\\a.mp4");
InputStream is = null;
OutputStream os = null;
try {
//分片下載
long fSize = file.length();//獲取長度
response.setContentType("application/x-download");
String fileName = URLEncoder.encode(file.getName(),utf8);
response.addHeader("Content-Disposition","attachment;filename="+fileName);
//根據前端傳來的Range 判斷支不支持分片下載
response.setHeader("Accept-Range","bytes");
//獲取文件大小
response.setHeader("fSize",String.valueOf(fSize));
response.setHeader("fName",fileName);
//定義斷點
long pos = 0,last = fSize-1,sum = 0;
//判斷前端需不需要分片下載
if (null != request.getHeader("Range")){
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
String numRange = request.getHeader("Range").replaceAll("bytes=","");
String[] strRange = numRange.split("-");
if (strRange.length == 2){
pos = Long.parseLong(strRange[0].trim());
last = Long.parseLong(strRange[1].trim());
//若結束字節超出文件大小 取文件大小
if (last>fSize-1){
last = fSize-1;
}
}else {
//若只給一個長度 開始位置一直到結束
pos = Long.parseLong(numRange.replaceAll("-","").trim());
}
}
long rangeLenght = last-pos+1;
String contentRange = new StringBuffer("bytes").append(pos).append("-").append(last).append("/").append(fSize).toString();
response.setHeader("Content-Range",contentRange);
response.setHeader("Content-Lenght",String.valueOf(rangeLenght));
os = new BufferedOutputStream(response.getOutputStream());
is = new BufferedInputStream(new FileInputStream(file));
is.skip(pos);//跳過已讀的文件
byte[] buffer = new byte[1024];
int lenght = 0;
//相等證明讀完
while (sum < rangeLenght){
lenght = is.read(buffer,0, (rangeLenght-sum)<=buffer.length? (int) (rangeLenght - sum) :buffer.length);
sum = sum+lenght;
os.write(buffer,0,lenght);
}
System.out.println("下載完成");
}finally {
if (is!= null){
is.close();
}
if (os!=null){
os.close();
}
}
}
}
客戶端分片下載，指定固定文件
@RestController
public class DownloadClient {
private final static long per_page = 1024l*1024l*50l;
//分片存儲臨時目錄 當分片下載完后在目錄中找到文件合并
private final static String down_path="D:\\File";
//多線程下載
ExecutorService pool = Executors.newFixedThreadPool(10);
//文件大小 分片數量 文件名稱
//使用探測 獲取變量
//使用多線程分片下載
//最后一個分片下載完 開始合并
@RequestMapping("/downloadFile")
public String downloadFile() throws IOException {
FileInfo fileInfo = download(0,10,-1,null);
if (fileInfo!= null){
long pages = fileInfo.fSize/per_page;
for (int i = 0; i <= pages; i++) {
pool.submit(new Download(i*per_page,(i+1)*per_page-1,i,fileInfo.fName));
}
}
return "成功";
}
class Download implements Runnable{
long start;
long end;
long page;
String fName;
public Download(long start, long end, long page, String fName) {
this.start = start;
this.end = end;
this.page = page;
this.fName = fName;
}
@Override
public void run() {
try {
FileInfo fileInfo = download(start,end,page,fName);
} catch (IOException e) {
e.printStackTrace();
}
}
}
//返回文件名 跟大小
private FileInfo download(long start,long end,long page,String fName) throws IOException {
//斷點下載 文件存在不需要下載
File file = new File(down_path, page + "-" + fName);
//探測必須放行 若下載分片只下載一半就鍛煉需要重新下載所以需要判斷文件是否完整
if (file.exists()&&page != -1&&file.length()==per_page){
return null;
}
//需要知道 開始-結束 = 分片大小
HttpClient client = HttpClients.createDefault();
//httpclient進行請求
HttpGet httpGet = new HttpGet("http://127.0.0.1:8080/down");
//告訴服務端做分片下載
httpGet.setHeader("Range","bytes="+start+"-"+end);
HttpResponse response = client.execute(httpGet);
String fSize = response.getFirstHeader("fSize").getValue();
fName= URLDecoder.decode(response.getFirstHeader("fName").getValue(),"utf-8");
HttpEntity entity = response.getEntity();//獲取文件流對象
InputStream is = entity.getContent();
//臨時存儲分片文件
FileOutputStream fos = new FileOutputStream(file);
byte[] buffer = new byte[1024];//定義緩沖區
int ch;
while ((ch = is.read(buffer)) != -1){
fos.write(buffer,0,ch);
}
is.close();
fos.flush();
fos.close();
//判斷是不是最后一個分片
if (end-Long.valueOf(fSize)>0){
//合并
try {
mergeFile(fName,page);
} catch (Exception e) {
e.printStackTrace();
}
}
return new FileInfo(Long.valueOf(fSize),fName);
}
private void mergeFile(String fName, long page) throws Exception {
//歸并文件位置
File file = new File(down_path, fName);
BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(file));
for (int i = 0; i <= page; i++) {
File tempFile = new File(down_path, i + "-" + fName);
//分片沒下載或者沒下載完需要等待
while (!file.exists()||(i!=page&&tempFile.length() Thread.sleep(100);
}
byte[] bytes = FileUtils.readFileToByteArray(tempFile);
os.write(bytes);
os.flush();
tempFile.delete();
}
File file1 = new File(down_path, -1 + "-null");
file1.delete();
os.flush();
os.close();
}
//使用內部類實現
class FileInfo{
long fSize;
String fName;
public FileInfo(long fSize, String fName) {
this.fSize = fSize;
this.fName = fName;
}
}
}
VSole
網絡安全專家