java多线程下载器
2018-09-18 本文已影响0人
IFLEU
最近在看爬虫，其中的文件下载就找了下网上的资料，写了一个test，随便记下。
Header
import java.util.HashMap;
/**
* @author xbl
* @date 2018-09-18
*/
public class Header {
public static final HashMap<String ,String> header = new HashMap<>();
private Header() {
}
static {
header.put("Accept-Encoding", "gzip, deflate");
header.put("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
header.put("Cache-Control", "max-age=0");
header.put("Connection", "keep-alive");
header.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0");
header.put("Accept",
"image/gif,image/jpeg,image/pjpeg,image/pjpeg, "
+ "application/x-shockwave-flash, application/xaml+xml, "
+ "application/vnd.ms-xpsdocument, application/x-ms-xbap"
+ "application/x-ms-application,application/vnd.ms-excel"
+ "application/vnd.ms-powerpoint, application/msword,*/*");
}
}
DownUtil
import entity.Header;
import java.io.RandomAccessFile;
import java.math.BigDecimal;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.concurrent.CountDownLatch;
/**
* @author xbl
* @date 2018-09-17
*/
public class DownUtil {
//定义下载路径
private String path;
//指定所下载的文件的保存位置
private String targetFile;
//定义下载线程的数量
private int threadNum;
//定义下载线程的对象
private DownloadThread[] threads;
//下载文件的总大小
private int fileSize;
//线程计数器
private CountDownLatch latch;
public DownUtil(String path,String targetFile,int threadNum){
this.path = path;
this.targetFile = targetFile;
this.threadNum = threadNum;
threads = new DownloadThread[threadNum];
latch = new CountDownLatch(threadNum);
}
public void downLoad() throws Exception{
long t1 = System.currentTimeMillis();
URL url = new URL(path);
HttpURLConnection conn = (HttpURLConnection)url.openConnection();
conn.setConnectTimeout(5 * 1000);
//设置请求方法
conn.setRequestMethod("GET");
//设置请求属性
Header.header.forEach((key, value) -> conn.setRequestProperty(key,value));
//得到文件大小
fileSize = conn.getContentLength();
conn.disconnect();
int currentPartSize = fileSize / threadNum + 1;
RandomAccessFile file = new RandomAccessFile(targetFile,"rw");
//设置本地文件大小
file.setLength(fileSize);
file.close();
for(int i = 0;i < threadNum;i++){
//计算每个线程的下载位置
int startPos = i * currentPartSize;
//每个线程使用一个RandomAccessFile进行下载
RandomAccessFile currentPart = new RandomAccessFile(targetFile,"rw");
//定位该线程的下载位置
currentPart.seek(startPos);
//创建下载线程
threads[i] = new DownloadThread(startPos, currentPartSize, currentPart , path , latch);
Thread t = new Thread(threads[i]);
t.start();
}
BigDecimal n = BigDecimal.ZERO;
while (true){
BigDecimal length = BigDecimal.ZERO;
if (latch.getCount()==0 && BigDecimal.ONE.compareTo(n)<=0){break;}
for (DownloadThread downloadThread:threads) {
length = length.add(new BigDecimal(downloadThread.getLength()));
}
n = length.divide(new BigDecimal(fileSize), 4, BigDecimal.ROUND_HALF_UP);
System.out.println("已下载"+n.multiply(new BigDecimal("100"))+"%");
Thread.sleep(1000);
}
long t2 = System.currentTimeMillis();
System.out.println("下载完成，文件大小为"+fileSize*1.0/(1024*1024)+"M，共用时"+(t2-t1)*1.0/1000+"秒,平均"+fileSize*1.0/(1024*1024)/((t2-t1)/1000)+"MB/s");
//线程等待,一个文件下载成功后继续下一个
// latch.await();
}
}
DownloadThread
import entity.Header;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.concurrent.CountDownLatch;
/**
* @author xbl
* @date 2018-09-17
*/
public class DownloadThread implements Runnable {
//当前线程的下载位置
private int startPos;
//定义当前线程负责下载的文件大小
private int currentPartSize;
//当前线程需要下载的文件块,此类的实例支持对随机访问文件的读取和写入。
private RandomAccessFile currentPart;
//定义该线程已下载的字节数
private int length;
//线程计数器
private CountDownLatch latch;
private String path;
public DownloadThread(int startPos,int currentPartSize,RandomAccessFile currentPart, String path ,CountDownLatch latch){
this.startPos = startPos;
this.currentPartSize = currentPartSize;
this.currentPart = currentPart;
this.path = path;
this.latch =latch;
}
@Override
public void run(){
InputStream inputStream = null;
try{
URL url = new URL(path);
HttpURLConnection conn = (HttpURLConnection)url.openConnection();
conn.setConnectTimeout(5 * 1000);
//设置请求方法
conn.setRequestMethod("GET");
//设置请求属性
Header.header.forEach((key, value) -> conn.setRequestProperty(key,value));
inputStream = conn.getInputStream();
//inputStream.skip(n);跳过和丢弃此输入流中数据的 n 个字节
inputStream.skip(this.startPos);
byte[] buffer = new byte[1024];
int hasRead = 0;
//读取网络数据写入本地
while(length < currentPartSize && (hasRead = inputStream.read(buffer)) != -1){
currentPart.write(buffer, 0, hasRead);
length += hasRead;
}
}
catch(Exception e){
e.printStackTrace();
}finally {
try {
currentPart.close();
inputStream.close();
}catch (Exception e){
e.printStackTrace();
}
latch.countDown();
}
}
public int getLength() {
return length;
}
}
测试
/**
* @author xbl
* @date 2018-09-14
*/
public class MyDowloadTest {
public static void main(String[] args) {
DownUtil downUtil = new DownUtil("http://122.246.9.250/67747A6C5D84B71E22A3C3CFE/03000A03005B9A34F27DF9800CB6C41096EB0F-B8A1-40F9-BE9D-77E1AD6E6E21.mp4?ccode=0502&duration=391&expire=18000&psid=12d0040bed59918644c5f644cc4ffd32&sp=&ups_client_netip=73cd74c8&ups_ts=1537256639&ups_userid=&utid=WqjDEjj3X1oCATy6JD6F%2Fbwk&vid=XMzgyMjM0MzUwNA%3D%3D&vkey=Be4882f87d72ed2b036aa6d16b12d2968&s=efbfbd135a731defbfbd&ali_redirect_domain=vali-dns.cp31.ott.cibntv.net","C:\\Users\\abc\\Desktop\\xbl\\img\\2.mp4",8);
try {
downUtil.downLoad();
} catch (Exception e) {
e.printStackTrace();
}
}
}
效果如下:
ok
