最简单实现高并发插入数万条数据(可同步可异步)
2019-03-28 本文已影响0人
jeyyband
获取数据| 整理数据
public static int count = 0;
String uniqueTimeId = IDUtils.getInstance().getUniqueTimeId();
String sqlof1 = "select vc_scode from ( "+sqlExpr+" )";
String sqlof2 = SqlExprConstant.pool_sql.replace(SqlExprConstant.POOL_TEMPLATE, poolId);
String joiner = " minus ";
//查询筛选结果
List<Map<String, Object>> reduce1 = riskruleService.executeSql(sqlof1+joiner+sqlof2);
//插入数据库 121_证券筛选执行结果
List<String> keys1 = reduce1.stream().filter(map -> !Objects.isNull(map.get("VC_SCODE")))
.map(i ->(String)i.get("VC_SCODE"))
.distinct().collect(Collectors.toList());
//并发插入数据
CurrencyAdd(keys1.size(), 100, keys1, uniqueTimeId, "1");
高并发插入上万条数据
/**
* 高并发插入上万条数据
* @param totalCount
* @param threadTotal
* @throws InterruptedException
* @author Bruce
*/
public void CurrencyAdd(int totalCount ,int threadTotal,List<String> keys,String uniqueTimeId,String cCompType) throws InterruptedException {
ExecutorService executorService = Executors.newCachedThreadPool();
// 限制同时执行的线程数
final Semaphore semaphore = new Semaphore(threadTotal);
final CountDownLatch countDownLatch = new CountDownLatch(totalCount);
for (int i = 0; i < totalCount; i++) {
executorService.execute(() -> {
try {
semaphore.acquire();
add(cCompType);
//根据下标获取取值插入
log.info("current count no is --->{}",count);
CompSecuFilterRes res = new CompSecuFilterRes();
String code = "";
code = keys.get(count-1);
res.setVcScode(code );
res.setVcFilterSeqno(uniqueTimeId);
res.setCCompType(cCompType);
res.setDMdftime(new Date());
riskruleService.addCompSecuFilterRes(res);
semaphore.release();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
countDownLatch.countDown();
});
}
//异步不等待执行过程就把这行注掉
//countDownLatch.await();
executorService.shutdown();
log.info("count{}", count);
}
private synchronized static void add(String type ) {
count++;
}
日志信息
...
2019-03-27 at 18:38:37 CST INFO com.xx.xxx.controller.xxxxController 442 lambda$4 - current count no is --->97
2019-03-27 at 18:38:37 CST DEBUG org.apache.ibatis.logging.jdbc.BaseJdbcLogger 159 debug - ==> Preparing: insert into T_COMP_SECU_FILTER_RES (VC_FILTER_SEQNO, C_REC_TYPE, VC_SCODE, C_COMP_TYPE, D_MDFTIME) values (?, ?, ?, ?, ?)
2019-03-27 at 18:38:37 CST DEBUG org.apache.ibatis.logging.jdbc.BaseJdbcLogger 159 debug - ==> Parameters: 1553683109754193(String), null, 011801593YH(String), 2(String), 2019-03-27(Date)
2019-03-27 at 18:38:37 CST DEBUG org.apache.ibatis.logging.jdbc.BaseJdbcLogger 159 debug - <== Updates: 1
2019-03-27 at 18:38:37 CST INFO com.xx.xxx.controller.xxxxController 442 lambda$4 - current count no is --->98
2019-03-27 at 18:38:37 CST DEBUG org.apache.ibatis.logging.jdbc.BaseJdbcLogger 159 debug - ==> Preparing: insert into T_COMP_SECU_FILTER_RES (VC_FILTER_SEQNO, C_REC_TYPE, VC_SCODE, C_COMP_TYPE, D_MDFTIME) values (?, ?, ?, ?, ?)
2019-03-27 at 18:38:37 CST DEBUG org.apache.ibatis.logging.jdbc.BaseJdbcLogger 159 debug - ==> Parameters: 1553683109754193(String), null, 041758030YH(String), 1(String), 2019-03-27(Date)
2019-03-27 at 18:38:37 CST DEBUG org.apache.ibatis.logging.jdbc.BaseJdbcLogger 159 debug - <== Updates: 1
2019-03-27 at 18:38:37 CST INFO com.xx.xxx.controller.xxxxController 442 lambda$4 - current count no is --->99
...
转载请注明来源。
我的博客:http://blog.northpark.cn
本文同步地址:http://blog.northpark.cn/2019/03/28/%E6%9C%80%E7%AE%80%E5%8D%95%E5%AE%9E%E7%8E%B0%E9%AB%98%E5%B9%B6%E5%8F%91%E6%8F%92%E5%85%A5%E6%95%B0%E4%B8%87%E6%9D%A1%E6%95%B0%E6%8D%AE(%E5%8F%AF%E5%90%8C%E6%AD%A5%E5%8F%AF%E5%BC%82%E6%AD%A5)/