Java两表数据大数据量比对
使用FutureTask进行多线程数据读取，减少读取时间
public void checkCompany(){
//获取数据
List<CompanyVo> omsList = getOmsList();
//将要比对数据放入Map
Map<Long, CompanyVo> omsMap = new HashMap<Long, CompanyVo>();
for(CompanyVo vo:omsList){
omsMap.put(vo.getIdUuid(), vo);
}
//比较数据
compareCompany(omsMap);
}
每2000条数据比对一次，MySQL数据库
public void compareCompany(Map<Long, CompanyVo> omsMap){
//获取总数据量
int count = jdbcTemplate.queryForObject("select count(1) from oms_company", Integer.class);
//分页查询
int pages = count%2000= 0?(count/2000):(count/2000 + 1);
ExecutorService executor = Executors.newFixedThreadPool(10);
for(int i = 0; i < pages; i++){
int startIndex = i*2000;
ProTask task = new ProTask(startIndex, omsMap);
FutureTask future = new FutureTask(task,null);
executor.execute(future);
}
executor.shutdown();
//判断是否比对完成
while(true){
if(executor.isTerminated()){
break;
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
获取要比较的总数据，Oracle数据库
public List<CompanyVo> getOmsList(){
List<CompanyVo> list = new ArrayList<>();
int count = jdbcTemplate.queryForObject("select count(1) from TB_PUBHA", Integer.class);
int pages = count%2000 == 0?(count/2000):(count/2000 + 1);
ExecutorService executor = Executors.newFixedThreadPool(10);
List<FutureTask<List<CompanyVo>>> futureList = new ArrayList<>();
for(int i = 0; i < pages; i++){
int start = i*2000 + 1;
int end = (i+1)*2000;
if(end > count) end = count;
OmsTask task = new OmsTask(start, end);
FutureTask<List<CompanyVo>> future = new FutureTask<>(task);
executor.submit(future);
futureList.add(future);
}
executor.shutdown();
for(FutureTask<List<CompanyVo>> future:futureList){
try {
List<CompanyVo> listq = future.get();
if(null != listq && !listq.isEmpty())
list.addAll(listq);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ExecutionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return list;
}
OmsTask实现Callable接口，因为需要返回值
class OmsTask implements Callable<List<CompanyVo>>{
private int start;
private int end;
public OmsTask(int start,int end){
this.start = start;
this.end = end;
}
@Override
public List<CompanyVo> call() throws Exception {
String sql = "select * from (select rownum rm, PUBHA_ID id_uuid,UUID company_number,PUBHA003 company_name,PUBHA005 company_address " +
",PUBHA006 post_code,PUBHA008 corporation,PUBHA009 corporation_tel,PUBHA010 manager,PUBHA011 manager_tel,PUBHA012 contract,PUBHA013 contract_tel" +
",PUBHA015 web_url from TB_PUBHA ) where rm between "+start+" and "+end;
List<Map<String, Object>> mapList = jdbcTemplate.queryForList(sql);
return CompanyVo.getCompany(mapList);
}
}
ProTask实现Runnable接口，不需要返回结果，只进行比较
class ProTask implements Runnable{
private int startIndex;
private Map<Long, CompanyVo> omsMap;
public ProTask(int startIndex,Map<Long, CompanyVo> omsMap){
this.startIndex = startIndex;
this.omsMap = omsMap;
}
@Override
public void run() {
List<Map<String, Object>> listMap = jdbcTemplate.queryForList("select id_uuid,company_number,company_name,province_id,city_id,company_address,"
+ "post_code,corporation,corporation_tel,manager,manager_tel,contract,contract_tel,web_url "
+ "from oms_company limit "+startIndex+","+ 2000);
List<CompanyVo> list = CompanyVo.getCompany(listMap);
if(null != list && !list.isEmpty()){
List<Object[]> obs = new ArrayList<>();
for(CompanyVo vo:list){
CompanyVo oms = omsMap.get(vo.getIdUuid());
if(null == oms){
continue;
}
if((null == vo.getCompanyNumber() && null != oms.getCompanyNumber())
|| (null != vo.getCompanyNumber() && !vo.getCompanyNumber().equals(oms.getCompanyNumber()))){
//结果不同时进行处理
}
//对已比对数据进行移除
omsMap.remove(vo.getIdUuid());
}
}
}
}
实体类，主要进行查询结果转化
package com.digital.domain;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.digital.util.StringUtil;
public class CompanyVo {
private Long idUuid;
private String companyNumber;
public Long getIdUuid() {
return idUuid;
}
public void setIdUuid(Long idUuid) {
this.idUuid = idUuid;
}
public String getCompanyNumber() {
return companyNumber;
}
public void setCompanyNumber(String companyNumber) {
this.companyNumber = companyNumber;
}
public static CompanyVo getCompany(Map<String, Object> map){
if(null == map) return null;
CompanyVo vo = new CompanyVo();
String idUuid = StringUtil.ObjectToString(map.get("ID_UUID"));
if(null != idUuid)
vo.setIdUuid(Long.parseLong(idUuid));
vo.setCompanyNumber(StringUtil.ObjectToString(map.get("COMPANY_NUMBER")));
return vo;
}
public static List<CompanyVo> getCompany(List<Map<String, Object>> mapList){
if(null == mapList) return null;
List<CompanyVo> list = new ArrayList<>();
for(Map<String, Object> map:mapList){
CompanyVo vo = getCompany(map);
if(null != vo)
list.add(vo);
}
return list;
}
}
————————————————
版权声明：本文为CSDN博主「yun0000000」的原创文章，遵循CC 4.0 by-sa版权协议，转载请附上原文出处链接及本声明。
原文链接：https://blog.csdn.net/yun0000000/article/details/52758098
