百万级数据插入mysql

百万级数据插入

mysql版本：5.7

mysql配置: max_allowed_packet

查看max_allowed_packet
show VARIABLES like '%max_allowed_packet%'
设置大小
set global max_allowed_packet = 100*1024*1024
退出mysql，重新登录进去
show VARIABLES like '%max_allowed_packet%'

package cn.cgnb.epanda.bill.controller;

import cn.cgnb.common.utils.UUIDUtil;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Date;
import java.sql.PreparedStatement;

/**
 * @Author xianxiong
 * @Date 2020/6/29
 */
public class test1 {


    public static void main(String[] args) throws ClassNotFoundException, SQLException {
        final String url = "jdbc:mysql:loadbalance://xx.xx.xx.xx:13306/dev_epanda?roundRobinLoadBalance=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai";
        final String name = "com.mysql.cj.jdbc.Driver";
        final String user = "root";
        final String password = "123456";
        Connection conn = null;
        Class.forName(name);//指定连接类型
        conn = DriverManager.getConnection(url, user, password);//获取连接
        if (conn != null) {
            System.out.println("获取连接成功");
            insert(conn);
        } else {
            System.out.println("获取连接失败");
        }

    }

    public static void insert(Connection conn) {
        // 开始时间
        Long begin = new Date().getTime();
        // sql前缀
        String prefix = "INSERT INTO ep_owner_bill_detail(id, bill_id, bill_type, calc_date, community_id, unit_id, room_id, room_name, owner_name, rule_id, rule_name, start_value, end_value, price, rate, original_fee, actual_fee, start_date, end_date, car_number, payment_type, payment_date, ref_detail_id, delivery_id, comment, data_status, create_id, create_time, update_id, update_time) VALUES ";
        try {
            // 保存sql后缀
            StringBuffer suffix = new StringBuffer();
            // 设置事务为非自动提交
            conn.setAutoCommit(false);
            // 比起st，pst会更好些
            PreparedStatement pst = (PreparedStatement) conn.prepareStatement(" ");//准备执行语句
            // 外层循环，总提交事务次数
            for (int i = 1; i <= 100; i++) {
                suffix = new StringBuffer();
                // 第j次提交步长
                for (int j = 1; j <= 10000; j++) {
                    // 构建SQL后缀
                    long uuid = UUIDUtil.genSequenceUUID();
                    suffix.append("("+uuid+"," +"1111111"  +","+"'"+"缴费"+"'"+","+"'"+"2020-11-30"+"'"+","+"388227987483869184"+","+"388227987483869184"+","+"388227987483869184"+ "," +"'"+ "6-1-201"+"'" + ","+"'"+"张强" + "'"+"," +"388227987483869184"+"," + "'"+"垃圾清洁费" + "'" +","+"null"+","+"null" +","+"6.0"+ ","+"1.0"+ ","+"6.0"+ ","+"6.0"+ ","+"'"+"2020-11-01"+ "'"+","+"'"+"2020-11-30"+"'"+ ","+"'"+"川A-9A876"+"'"+","+"'"+"支付宝"+"'"+","+"'"+"2020-11-30"+"'"+","+"388227987483869184"+","+"388227987483869184"+ ","+"'"+"2020-02-01~2020-02-29"+"'" + ","+"'"+"0"+"'"+","+"388227987483869184"+","+"'"+"2020-03-24 17:49:27"+"'"+","+"388227987483869184"+","+ "'"+"2020-03-24 17:49:27"+"'"+")"+",");
                }
                // 构建完整SQL
                String sql = prefix + suffix.substring(0, suffix.length() - 1);
                // 添加执行SQL
                pst.addBatch(sql);
                // 执行操作
                pst.executeBatch();
                // 提交事务
                conn.commit();
                // 清空上一次添加的数据
                suffix = new StringBuffer();
            }
            // 头等连接
            pst.close();
            conn.close();
        } catch (SQLException e) {
            e.printStackTrace();
        }
        // 结束时间
        Long end = new Date().getTime();
        // 耗时
        System.out.println("1000万条数据插入花费时间 : " + (end - begin) / 1000 + " s");
        System.out.println("插入完成");
    }
}