大文件分片断点上传下载
2019-08-16 本文已影响0人
忧郁的小码仔
当上传或者下载的文件比较大的时候，如果中途网络断掉或者后台前端出现问题，导致下载中断，作为用户来说，我肯定希望下次登陆的时候接着上次下载上传的进度继续上传或者下载，而不是从头开始。这就需要前端和后台支持断点续传。
断点分片上传
大体思路是这样的：
前端：假定我们每个分片的大小是1Mb，每次前端上传的时候，先读取一个分片大小的数据，然后询问后台该分片是否已经上传，如果没有上传，则将该分片上传，如果上传过了，那么就seek 一个分片大小的offset，读取下一个分片继续同样的操作，直到所有分片上传完成。
后端：我们将接收到的一个一个的分片文件单独放到一个以文件md5命名的文件夹下，当前端询问分片是否存在的时候，只要到该文件夹下查找对应的临时文件是否存在即可。当所有的分片文件都上传后，再合并所有的临时文件即可。
先看前端的实现：
@interface LFUpload : NSObject
-(void)uploadFile:(NSString *)fileName withType:(NSString*)fileType progress:(Progress)progress;
@end
这里，我们将上传单独放到一个类里面
static int offset = 1024*1024; // 每片的大小是1Mb
@interface LFUpload()
@property (nonatomic, assign) NSInteger truncks;
@property (nonatomic, copy) NSString *fileMD5;
@property (nonatomic, copy) Progress progress;
@end
@implementation LFUpload
-(void)uploadFile:(NSString *)fileName withType:(NSString *)fileType progress:(Progress)progress {
self.progress = progress;
NSString *filePath = [[NSBundle mainBundle] pathForResource:fileName ofType:fileType];
NSData *fileData = [NSData dataWithContentsOfFile:filePath];
self.truncks = fileData.length % offset == 0 ? fileData.length/offset : fileData.length/offset + 1; // 计算该文件的总分片数
self.fileMD5 = [FileUtil getMD5ofFile:filePath]; // 获取文件md5值并传给后台
[self checkTrunck:1]; // 检查第一个分片是否已经上传
}
// 检查分片是否已经上传
-(void)checkTrunck:(NSInteger)currentTrunck {
if (currentTrunck > self.truncks) {
self.progress(100, NO); // 标示上传完成
return;
}
__weak typeof(self) weakSelf = self;
NSMutableDictionary *params = [[NSMutableDictionary alloc] init];
[params setValue:self.fileMD5 forKey:@"md5file"];
[params setValue:@(currentTrunck) forKey:@"chunk"];
[[LFNetManager defaultManager] POST:@"https://192.168.1.57:443/checkChunk" parameters:params progress:nil success:^(NSURLSessionDataTask * _Nonnull task, id _Nullable responseObject) {
NSString *code = [responseObject objectForKey:@"code"];
if ([code isEqualToString:@"0002"]) { //分片未上传，下面开始上传该分片数据
[weakSelf uploadTrunck:currentTrunck];
} else { // 分片已经上传过了，直接更新上传进度
CGFloat progressFinished = currentTrunck * 1.0/self.truncks; self.progress(progressFinished, NO);
[weakSelf checkTrunck:currentTrunck + 1];
}
} failure:^(NSURLSessionDataTask * _Nullable task, NSError * _Nonnull error) {
self.progress(0, YES);
}];
}
// 上传分片
-(void)uploadTrunck:(NSInteger)currentTrunck {
__weak typeof(self) weakSelf = self;
[[LFNetManager defaultManager] POST:@"https://192.168.1.57:443/upload"
parameters:nil
constructingBodyWithBlock:^(id<AFMultipartFormData> _Nonnull formData) {
NSString *filePath = [[NSBundle mainBundle] pathForResource:@"myFilm" ofType:@"mp4"];
NSData *data;
NSFileHandle *readHandler = [NSFileHandle fileHandleForReadingAtPath:filePath];
// 偏移到这个分片开始的位置，并读取数据
[readHandler seekToFileOffset:offset * (currentTrunck - 1)];
data = [readHandler readDataOfLength:offset];
[formData appendPartWithFileData:data name:@"file" fileName:@"myFilm.mp4" mimeType:@"application/mp4"];
// md5File， 后台需要的参数
NSData *md5FileData = [self.fileMD5 dataUsingEncoding:NSUTF8StringEncoding];
[formData appendPartWithFormData:md5FileData name:@"md5File"];
// truncks， 后台需要直到分片总数，来判断上传是否完成
NSData *truncksData = [[NSString stringWithFormat:@"%ld", (long)self.truncks] dataUsingEncoding:NSUTF8StringEncoding];
[formData appendPartWithFormData:truncksData name:@"truncks"];
// currentTrunck，告知后台当前上传的分片索引
NSData *trunckData = [[NSString stringWithFormat:@"%ld", (long)currentTrunck] dataUsingEncoding:NSUTF8StringEncoding];
[formData appendPartWithFormData:trunckData name:@"currentTrunck"];
} progress:^(NSProgress * _Nonnull uploadProgress) {
CGFloat progressInThisTrunck = (1.0 * uploadProgress.completedUnitCount) / (uploadProgress.totalUnitCount * self.truncks); // 当前分片中已上传数据占文件总数据的百分比
CGFloat progressFinished = (currentTrunck - 1) * 1.0/self.truncks; // 已经完成的进度
self.progress(progressInThisTrunck + progressFinished, NO);
} success:^(NSURLSessionDataTask * _Nonnull task, id _Nullable responseObject) {
[weakSelf checkTrunck:currentTrunck + 1];
} failure:^(NSURLSessionDataTask * _Nullable task, NSError * _Nonnull error) {
self.progress(0, YES);
}];
}
@end
再来看后台的实现：
- 检查分片是否已经上传
@PostMapping("checkChunk")
@ResponseBody
public ResponseCommon checkChunk(@RequestParam(value = "md5file") String md5file, @RequestParam(value = "chunk") Integer chunk) {
ResponseCommon responseCommon = new ResponseCommon();
// 这里通过判断分片对应的文件存不存在来判断分片有没有上传
try {
File path = new File(ResourceUtils.getURL("classpath:").getPath());
File fileUpload = new File(path.getAbsolutePath(), "static/uploaded/" + md5file + "/" + chunk + ".tmp");
if (fileUpload.exists()) {
responseCommon.setCode("0001");
responseCommon.setMsg("分片已上传");
} else {
responseCommon.setCode("0002");
responseCommon.setMsg("分片未上传");
}
} catch (FileNotFoundException e) {
e.printStackTrace();
responseCommon.setCode("1111");
responseCommon.setMsg(e.getLocalizedMessage());
}
return responseCommon;
}
- 分片上传
@PostMapping("upload")
@ResponseBody
public ResponseCommon upload(@RequestParam(value = "file") MultipartFile file,
@RequestParam(value = "md5File") String md5File,
@RequestParam(value = "truncks") Integer truncks,
@RequestParam(value = "currentTrunck") Integer currentTrunck) {
ResponseCommon responseCommon = new ResponseCommon();
try {
File path = new File(ResourceUtils.getURL("classpath:").getPath());
File fileUpload;
if (truncks == 1) {
fileUpload = new File(path.getAbsolutePath(), "static/uploaded/" + md5File + "/1.tmp");
} else {
fileUpload = new File(path.getAbsolutePath(), "static/uploaded/" + md5File + "/" + currentTrunck + ".tmp");
}
if (!fileUpload.exists()) {
fileUpload.mkdirs();
}
file.transferTo(fileUpload);
if (currentTrunck == truncks) {
boolean result = this.merge(truncks, md5File, file.getOriginalFilename());
if (result) {
responseCommon.setCode("0000");
responseCommon.setMsg("上传成功");
} else {
responseCommon.setCode("1111");
responseCommon.setMsg("文件合并失败");
}
} else {
responseCommon.setCode("0000");
responseCommon.setMsg("上传成功");
}
} catch (FileNotFoundException e) {
e.printStackTrace();
responseCommon.setCode("1111");
responseCommon.setMsg(e.getLocalizedMessage());
} catch (IOException e) {
e.printStackTrace();
responseCommon.setCode("1111");
responseCommon.setMsg(e.getLocalizedMessage());
}
return responseCommon;
}
- 合并所有分片临时文件
public boolean merge(@RequestParam(value = "truncks") Integer truncks,
@RequestParam(value = "md5File") String md5File,
@RequestParam(value = "fileName") String fileName) {
ResponseCommon responseCommon = new ResponseCommon();
FileOutputStream outputStream = null;
try {
File path = new File(ResourceUtils.getURL("classpath:").getPath());
File md5FilePath = new File(path.getAbsolutePath(), "static/uploaded/" + md5File);
File finalFile = new File(path.getAbsolutePath(), "static/uploaded/" + fileName);
outputStream = new FileOutputStream(finalFile);
byte[] buffer = new byte[1024];
for (int i=1; i<=truncks; i++) {
String chunckFile = i + ".tmp";
File tmpFile = new File(md5FilePath + "/" + chunckFile);
InputStream inputStream = new FileInputStream(tmpFile);
int len = 0;
while ((len = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, len);
}
inputStream.close();
}
} catch (FileNotFoundException e) {
e.printStackTrace();
return false;
} catch (IOException e) {
e.printStackTrace();
return false;
} finally {
try {
outputStream.close();
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
return true;
}
这里，我们只是大体实现一下，不必在意细节和规范。
断点分片下载
这里的逻辑和上传的逻辑正好是反过来的
前端先到后台获取总分片数，然后循环遍历，检查本地对应文件夹下是否有对应的分片临时文件，如果有，表示之前下载过，如果没有，就到后台请求下载对应的分片文件，当所有分片下载完成后，再合并所有的分片临时文件即可
后台，这里要提供文件的总分片数，以及和之前前端一样的可以seek到某个分片，单独提供给前端下载，另外，考虑到gzip的问题，最好提供一个单独的接口告知前端整个文件的大小，当然，这里，因为我们知道每个分片的大小是1Mb，以及所有的分片数，为了简单起见，就简单粗糙的计算下下载进度。
先看后端实现：
- 获取该文件的所有分片数
@GetMapping("getTruncks")
@ResponseBody
public ResponseCommon getTruncks() {
ResponseCommon responseCommon = new ResponseCommon();
responseCommon.setCode("0000");
try {
File file = new File("/Users/archerlj/Desktop/Untitled/myFilm.mp4");
Long fileLength = file.length();
Long trunck = fileLength/(1024 * 1024);
if (fileLength%(1024*1024) == 0) {
responseCommon.setTruncks(trunck.intValue());
} else {
responseCommon.setTruncks(trunck.intValue() + 1);
}
String fileMD5 = DigestUtils.md5DigestAsHex(new FileInputStream(file));
responseCommon.setFileMD5(fileMD5);
responseCommon.setCode("0000");
} catch (IOException e) {
e.printStackTrace();
responseCommon.setCode("1111");
responseCommon.setMsg(e.getLocalizedMessage());
}
return responseCommon;
}
- 分片下载接口
@GetMapping("downloadFile")
@ResponseBody
public void downloadFile(Integer trunck, HttpServletRequest request, HttpServletResponse response) {
try {
RandomAccessFile file = new RandomAccessFile("/Users/archerlj/Desktop/Untitled/myFilm.mp4", "r");
long offset = (trunck - 1) * 1024 * 1024;
file.seek(offset);
byte[] buffer = new byte[1024];
int len = 0;
int allLen = 0;
for (int i=0; i<1024; i++) {
len = file.read(buffer);
if (len == -1) {
break;
}
allLen += len;
response.getOutputStream().write(buffer, 0, len);
file.seek(offset + (1024 * (i + 1)));
}
file.close();
response.setContentLength(allLen);
response.flushBuffer();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
前端实现:
同样，这里我们单独提供一个类：
typedef void(^DownloadProgress)(CGFloat progress, Boolean error);
@interface LFDownload : NSObject
-(void)downloadWithCallBack:(DownloadProgress)callback;
@end
@interface LFDownload()
@property (nonatomic, copy) DownloadProgress progress;
@property (nonatomic, assign) NSInteger allTruncks;
@property (nonatomic, assign) NSInteger currentTrunck;
@property (nonatomic, copy) NSString *fileMD5;
@end
@implementation LFDownload
-(void)downloadWithCallBack:(DownloadProgress)callback {
self.progress = callback;
[self getTruncks];
[self getProgress]; // 这里我们单独来计算下载进度
}
-(void)getTruncks {
__weak typeof(self) weakSelf = self;
[[LFNetManager defaultManager] GET:@"https://192.168.1.57:443/getTruncks" parameters:nil progress:nil success:^(NSURLSessionDataTask * _Nonnull task, id _Nullable responseObject) {
if ([[responseObject valueForKey:@"code"] isEqualToString:@"0000"]) {
weakSelf.allTruncks = [[responseObject valueForKey:@"truncks"] integerValue];
weakSelf.fileMD5 = [responseObject valueForKey:@"fileMD5"];
if ([self checkMD5FilePath]) {
[self downloadWithTrunck:1];
} else {
weakSelf.progress(0, YES);
}
} else {
weakSelf.progress(0, YES);
}
} failure:^(NSURLSessionDataTask * _Nullable task, NSError * _Nonnull error) {
weakSelf.progress(0, YES);
}];
}
// 检查存放temp文件的文件夹是否存在
-(BOOL)checkMD5FilePath {
NSString *path = [[NSHomeDirectory() stringByAppendingPathComponent:@"Documents"] stringByAppendingPathComponent:self.fileMD5];
NSFileManager *fileManager = [NSFileManager defaultManager];
BOOL isDir = YES;
BOOL fileMD5PathExists = [fileManager fileExistsAtPath:path isDirectory:&isDir];
if (!fileMD5PathExists) {
// 如果不存在，就创建文件夹
NSError *error;
[fileManager createDirectoryAtPath:path withIntermediateDirectories:YES attributes:nil error:&error];
if (error) {
NSLog(@"下载失败");
return NO;
}
}
return YES;
}
// 下载完成，合并所有temp文件
-(void)mergeTempFiles {
NSFileManager *fileManager = [NSFileManager defaultManager];
NSString *finalFilePath = [[NSHomeDirectory() stringByAppendingPathComponent:@"Documents"] stringByAppendingPathComponent:@"myFilm.mp4"];
BOOL isDir = NO;
// 检查合并的最终文件是否存在，不存在就新建一个文件
if (![fileManager fileExistsAtPath:finalFilePath isDirectory:&isDir]) {
BOOL result = [fileManager createFileAtPath:finalFilePath contents:nil attributes:nil];
if (!result) {
self.progress(0, YES);
NSLog(@"文件合并失败");
return;
}
}
NSFileHandle *outFileHandler = [NSFileHandle fileHandleForUpdatingAtPath:finalFilePath];
NSString *path = [[NSHomeDirectory() stringByAppendingPathComponent:@"Documents"] stringByAppendingPathComponent:self.fileMD5];
// 开始循环读取每个分片临时文件
for (int i=1; i<= self.allTruncks; i++) {
NSString *fileName = [NSString stringWithFormat:@"%d.temp", i];
NSString *tempFilePath = [path stringByAppendingPathComponent:fileName];
NSFileHandle *inFileHandler = [NSFileHandle fileHandleForReadingAtPath:tempFilePath];
int offsetIndex = 0;
BOOL end = NO;
// 这里每个分片文件，我们每次读取 1024 byte， 循环读取，直到读取的data长度为0
while (!end) {
[inFileHandler seekToFileOffset:1024 * offsetIndex];
NSData *data = [inFileHandler readDataOfLength:1024];
if (data.length == 0) {
end = YES;
} else {
// 将最终文件seek到最后，并将数据写入
[outFileHandler seekToEndOfFile];
[outFileHandler writeData:data];
}
offsetIndex += 1;
}
[inFileHandler closeFile];
}
[outFileHandler closeFile];
NSLog(@"文件合并成功");
}
-(void)downloadWithTrunck:(NSInteger)currentTrunck {
if (currentTrunck > self.allTruncks) {
NSLog(@"下载完成");
self.progress(1.0, NO);
[self mergeTempFiles];
return;
}
__weak typeof(self) weakSelf = self;
NSString *urlStr = [NSString stringWithFormat:@"https://192.168.1.57:443/downloadFile?trunck=%ld", (long)currentTrunck];
NSURL *url = [NSURL URLWithString:urlStr];
NSURLRequest *request = [NSURLRequest requestWithURL:url];
NSString *fileName = [NSString stringWithFormat:@"%ld.temp", (long)currentTrunck];
NSString *path = [[NSHomeDirectory() stringByAppendingPathComponent:@"Documents"] stringByAppendingPathComponent:self.fileMD5];
NSString *tempFilePath = [path stringByAppendingPathComponent:fileName];
NSFileManager *fileManager = [NSFileManager defaultManager];
BOOL isDir = NO;
BOOL tempFileExists = [fileManager fileExistsAtPath:tempFilePath isDirectory:&isDir];
if (tempFileExists) { // 文件已经下载了
NSLog(@"%ld.temp 已经下载过了", currentTrunck);
self.progress(currentTrunck * 1.9 /self.allTruncks, NO);
[weakSelf downloadWithTrunck:currentTrunck + 1];
} else { // 开始下载文件
self.currentTrunck = currentTrunck;
NSURLSessionDownloadTask *task = [[LFNetManager defaultSessionManager] downloadTaskWithRequest:request progress:^(NSProgress * _Nonnull downloadProgress) {
/* 由于后台使用gzip压缩的原因，导致http header中没有Content-Length字段，所以这里不能获取到下载进度 */
} destination:^NSURL * _Nonnull(NSURL * _Nonnull targetPath, NSURLResponse * _Nonnull response) {
NSLog(@"%@", tempFilePath);
return [NSURL fileURLWithPath:tempFilePath];
} completionHandler:^(NSURLResponse * _Nonnull response, NSURL * _Nullable filePath, NSError * _Nullable error) {
if (error) {
weakSelf.progress(0, YES);
} else {
[weakSelf downloadWithTrunck:currentTrunck + 1];
}
}];
[task resume];
}
}
// 我们自己在回调中计算进度
-(void)getProgress {
__weak typeof(self) weakSelf = self;
[[LFNetManager defaultSessionManager] setDownloadTaskDidWriteDataBlock:^(NSURLSession * _Nonnull session, NSURLSessionDownloadTask * _Nonnull downloadTask, int64_t bytesWritten, int64_t totalBytesWritten, int64_t totalBytesExpectedToWrite) {
// 这里总长度应该从后台单独获取，这里取一个大致的值，假设所有的trunk都有1Mb，其实，只有最后一个没有1Mb
long allBytes = 1024 * 1024 * self.allTruncks;
long downloadedBytes = bytesWritten + 1024 * 1024 * (self.currentTrunck - 1); // 当前trunk已经下载的大小 + 前面已经完成的trunk的大小
weakSelf.progress(downloadedBytes * 1.0 / allBytes, NO);
}];
}
@end
文件下载完成以后，我们可以在XCode中，到Window->Devices and Simulators中，将app的container文件下载到电脑上，查看下载的文件：
屏幕快照 2019-08-16 上午10.44.38.png
屏幕快照 2019-08-16 上午10.55.35.png
这里只考虑了分片断点续传，其实文件比较大的时候，考虑多线程上传应该会更快一些，可以用NSOperationQueue来简单的将文件分片分成多个任务上传下载，这里就要控制下多线程下临界区的问题了，这里主要是分片索引和进度的多线程访问问题。当然，如果是多个文件上传下载，将每个文件单独分配任务的话就简单多了。
详细的代码请参考下面链接：
