大数据 爬虫Python AI SqlJava 杂谈技术干货

Java与Python实现最实际与高效生成数据库高级声明式SQL

2019-04-20  本文已影响0人  爱学习的蹭蹭

一、简介

1、前言与初衷

2、目的与作用

3、 什么是主脚本、副脚本、增量脚本呢?

二、实现方式与原理

2.1实现方式

2.2原理

三、技术选型体系与图解

四、Oracle自带系统表与常用的SQL语法

五、开发环境与工具

六、前端的技术选型

七、框架与技术

、八、项目结构

i项目结构

Java与Python核心代码实现

@Service
@Transactional
@SuppressWarnings("all")
public class TableServiceImpl extends  EntityDaoSupport<UserTables> implements TableService {
    
    @Autowired TableDao tableDao;
    @Autowired TableColumnsDao  tableColumnsDao;
    @Autowired UserColCommentsDao  userColCommentsDao;
    @Autowired UserTabCommentsDao  userTabCommentsDao;
    @Autowired UserConsColumnsDao  userConsColumnsDao;
    
    public  List<UserTables> queryTables(SimplePage page, UserTables object) {
        return tableDao.queryTables(page,object);
    }
    
    public List<UserTables> queryTableByName(String table_name){
        
        return tableDao.queryTableByName(table_name);
    }
 
    @Override
    public String createTableSQLScript(String tableName) {
        String result = "";
 
        //1、判断是否存在这个表
        List<UserTables> findList = queryTableByName( tableName);
        //2、组装execute immediate 'create table ...'
        if(findList!=null &&findList.size()>0){
            StringBuffer buffer =  new StringBuffer();
            //3、获取表的字段名称,并且获取约束是否为空(Null or Not null)
            List<UserTabColumns> userTabColumnList = tableColumnsDao.getUserTabColumnsByName(tableName);
            List<UserTabComments> userTabCommentList = userTabCommentsDao.getUserTabCommentsByName(tableName);
            List<UserColComments> userColCommentList = userColCommentsDao.getUserColCommentsByName(tableName);
            UserConsColumns userConsColumns = userConsColumnsDao.getUserConsColumnsByName(tableName);
            //----组装表字段列名
            int size = userTabColumnList.size()-1;//计算最后一个表列名
            buffer.append("--创建"+tableName+"表").append(ENTER);
            buffer.append("declare").append(ENTER);
            buffer.append(TAB).append("iCnt number := 0;").append(ENTER);
            buffer.append("begin ").append(ENTER);
            buffer.append("  select count(*) into iCnt from user_tables where lower(table_name) = lower('"+tableName+"');").append(ENTER); 
            buffer.append("  if iCnt = 0 then ").append(ENTER);  
            buffer.append(TAB).append("execute immediate 'create table "+tableName+"").append(ENTER);
            buffer.append(TAB).append("(").append(ENTER);
            
            for (int i = 0; i < userTabColumnList.size(); i++) {
                UserTabColumns userTabColumns = userTabColumnList.get(i);
                UserColComments userColComments = userColCommentList.get(i);
                
                String column_name1 = userColComments.getColumn_name();//列名
                
                String column_name2 = userTabColumns.getColumn_name();//列名
                String nullable = userTabColumns.getNullable();//是否允许为空
                String data_type = userTabColumns.getData_type();//数据库表的字段类型.
                BigDecimal data_length = userTabColumns.getData_length();
                BigDecimal data_precision = userTabColumns.getData_precision();
                BigDecimal data_scale = userTabColumns.getData_scale();
                
                if(ObjectUtil.isNotEmpty(column_name1) &&  ObjectUtil.isNotEmpty(column_name2) && column_name1.equalsIgnoreCase(column_name2)){
                    
                    //判断表字段类型是否为NULL或NOT NULL
                    if(ObjectUtil.isNotEmpty(nullable) && !"N".equalsIgnoreCase(nullable)){
                        if(size==i){
                            if("VARCHAR2".equalsIgnoreCase(data_type) || "NVARCHAR2".equalsIgnoreCase(data_type) ||"CHAR".equalsIgnoreCase(data_type)){
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type+"("+data_length+")" ).append(ENTER);
                            }else if("NUMBER".equalsIgnoreCase(data_type)){
                                if(data_precision!= null && data_scale!=null){
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type+"("+data_precision+","+data_scale+") ").append(ENTER);
                                }else if(data_precision!= null && data_scale==null){
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type+"("+data_precision+",0) ").append(ENTER);
                                }else{
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type).append(ENTER);
                                }
                            }else if(("DATE".equalsIgnoreCase(data_type))||(data_type!=null && data_type.contains("TIMESTAMP")) ){
                                buffer.append(TAB+TAB+column_name1+TAB+data_type).append(ENTER);
                            }else if("LONG".equalsIgnoreCase(data_type) ||"NCLOB".equalsIgnoreCase(data_type)){
                                buffer.append(TAB+TAB+column_name1+TAB+data_type).append(ENTER);
                            }
                        }else{
                            //为空但是有逗号分开
                            if("VARCHAR2".equalsIgnoreCase(data_type) || "NVARCHAR2".equalsIgnoreCase(data_type) ||"CHAR".equalsIgnoreCase(data_type)){
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type+"("+data_length+")" ).append(COMMA).append(ENTER);
                            }else if("NUMBER".equalsIgnoreCase(data_type)){
                                if(data_precision!= null && data_scale!=null){
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type+"("+data_precision+","+data_scale+") ").append(COMMA).append(ENTER);
                                }else if(data_precision!= null && data_scale==null){
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type+"("+data_precision+",0) ").append(COMMA).append(ENTER);
                                }else{
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type).append(COMMA).append(ENTER);
                                }
                            }else if(("DATE".equalsIgnoreCase(data_type))||(data_type!=null && data_type.contains("TIMESTAMP")) ){
                                buffer.append(TAB+TAB+column_name1+TAB+data_type).append(COMMA).append(ENTER);
                            }else if("LONG".equalsIgnoreCase(data_type) ||"NCLOB".equalsIgnoreCase(data_type)){
                                buffer.append(TAB+TAB+column_name1+TAB+data_type).append(COMMA).append(ENTER);
                            }
                        }
                    }else{
                        if(size==i){
                            if("VARCHAR2".equalsIgnoreCase(data_type) || "NVARCHAR2".equalsIgnoreCase(data_type) ||"CHAR".equalsIgnoreCase(data_type)){
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type+"("+data_length+")  not null "  ).append(ENTER);
                            }else if("NUMBER".equalsIgnoreCase(data_type)){
                                if(data_precision!= null && data_scale!=null){
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type+"("+data_precision+","+data_scale+")  not null").append(ENTER);
                                }else if(data_precision!= null && data_scale==null){
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type+"("+data_precision+",0)  not null ").append(ENTER);
                                }else{
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type+"").append(ENTER);
                                }
                            }else if(("DATE".equalsIgnoreCase(data_type))||(data_type!=null && data_type.contains("TIMESTAMP")) ){
                                buffer.append(TAB+TAB+column_name1+TAB+data_type).append(ENTER);
                            }else if("LONG".equalsIgnoreCase(data_type) ||"NCLOB".equalsIgnoreCase(data_type)){
                                buffer.append(TAB+TAB+column_name1+TAB+data_type).append(ENTER);
                            }
                            
                        }else{
                            if("VARCHAR2".equalsIgnoreCase(data_type) || "NVARCHAR2".equalsIgnoreCase(data_type) ||"CHAR".equalsIgnoreCase(data_type)){
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type+"("+data_length+")  not null "  ).append(COMMA).append(ENTER);
                            }else if("NUMBER".equalsIgnoreCase(data_type)){
                                if(data_precision!= null && data_scale!=null){
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type+"("+data_precision+","+data_scale+")  not null").append(COMMA).append(ENTER);
                                }else if(data_precision!= null && data_scale==null){
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type+"("+data_precision+",0)  not null ").append(COMMA).append(ENTER);
                                }else{
                                    buffer.append(TAB+TAB+column_name1+TAB+data_type+" ").append(COMMA).append(ENTER);
                                }
                            }else if(("DATE".equalsIgnoreCase(data_type))||(data_type!=null && data_type.contains("TIMESTAMP")) ){
                                buffer.append(TAB+TAB+column_name1+TAB+data_type).append(COMMA).append(ENTER);
                            }else if("LONG".equalsIgnoreCase(data_type) ||"NCLOB".equalsIgnoreCase(data_type)){
                                buffer.append(TAB+TAB+column_name1+TAB+data_type).append(COMMA).append(ENTER);
                            }
                        }
                    } 
                }
            }
            buffer.append(TAB).append(")';").append(ENTER);
            //4、获取表名给予表加备注.(有就加,无默认表描述)
            if(ObjectUtil.isNotEmpty(userTabCommentList) && ObjectUtil.isNotEmpty(userTabCommentList.get(0))){
                String comments = userTabCommentList.get(0).getComments();
                comments = (comments==null?" this is "+tableName+" tableName ":comments);
                buffer.append(TAB).append("execute immediate 'comment on table "+tableName +" is ''"+comments+"'''").append(";").append(ENTER);
            }
            //5、添加主键执行SQL脚本
            if(ObjectUtil.isNotEmpty(userConsColumns) && ObjectUtil.isNotEmpty(userConsColumns.getConstraint_name())){
                buffer.append(TAB).append("execute immediate 'alter table "+tableName).append(" add constraint "+userConsColumns.getConstraint_name()+" ");
                buffer.append("primary key ("+userConsColumns.getColumn_name()+")' ;").append(ENTER);
            }
            //6、获取表字段名给予字段加备注(有就加,无默认字段描述)
            for(UserColComments col: userColCommentList){
                String comments = col.getComments();
                String column_name = col.getColumn_name();
                comments = (comments==null?" this is "+column_name+" columnName ":comments);
                buffer.append(TAB).append("execute immediate 'comment on column "+tableName+"."+col.getColumn_name()+" is ''"+comments+"'''").append(";").append(ENTER);
            }
            
            //---end if
            buffer.append(" end if;").append(ENTER);
            buffer.append("end;").append(ENTER);
            buffer.append("/").append(ENTER);
            result = buffer.toString();
        }
        return result;
    
    }
 
    
    @Override
    public String dropTableSQLScript(String table_name) {
        
        List<UserTables> list = queryTableByName(table_name);
        StringBuffer buffer = new StringBuffer(); 
        if(list!=null && list.size()>0){
            UserTables sequences = list.get(0);
            buffer.append("--"+table_name).append(ENTER);
            buffer.append("declare").append(ENTER);
            buffer.append(TAB+TAB).append("iCnt number := 0;").append(ENTER);
            buffer.append(TAB).append("begin").append(ENTER);
            buffer.append(TAB).append(" select count(*) into iCnt from user_tables  where  lower(table_name) = '"+table_name+"'; --根据表进行查询").append(ENTER);
            buffer.append(TAB).append(" if iCnt > 0 then ").append(ENTER);
            buffer.append(TAB).append("     execute immediate 'drop table "+table_name+"'; --删除表,表结构和数据一起清空 ").append(ENTER);
            buffer.append(TAB).append(" end if;").append(ENTER);
            buffer.append(TAB).append("end;").append(ENTER);
            buffer.append("/").append(ENTER);
            return buffer.toString();
        }
        
        return null;
    }

}
# -*- coding:utf-8 -*- 处理乱码
import xlrd
#import xlwt
'''
  @Author jilongliang
  @Date   2017-04-12
  @Copyright (c) All Right Reserved jilong, 2017.
  @Description 
      xlwt和xlwt安装步骤,请参考博客  http://www.cnblogs.com/sincoolvip/p/5967010.html
                            http://www.cnblogs.com/lhj588/archive/2012/01/06/2314181.html
'''
#===============全局变量====================
TAB   = "    ";   #空格置位常量
COMMA = ",";      #SQL逗号常量
ENTER = "\n";     #换行常量
#===================================
 
#--------------定义一个读取excel的方法---------
def read_excel():
    resultSQL = "";#返回SQL
    #打开文件,Python中以r或R开头的的字符串表示(非转义的)原始字符串.说明字符串r"XXX"中的XXX是普通字符。 
    workbook = xlrd.open_workbook(r'D:\table_all.xls') 
    #获取所有sheet 
    sheet_names = workbook.sheet_names() # [u'sheet1', u'sheet2'] 
    #遍历读取excel的每个sheet
    #for sheet_name in sheet_names:
    #print(sheet_name) 
    
    #遍历读取excel的每个sheet并且读取有多少个sheet的tab
    for sheet_i in range(len(sheet_names)):
        sheetName =sheet_names[sheet_i];#sheetName就是表名,在创建excel的时候规定好即可
        sheet = workbook.sheet_by_index(sheet_i) #获取excel里面有哪些sheet
        nrows = sheet.nrows #读取每个sheet有多少行(sheet里面一个table表格有多少行)
        
        sqlStr  = "--创建"+sheetName+"表SQL脚本"+ENTER;#表描述
        sqlStr += "declare"+ENTER;#declare关键字
        sqlStr += TAB+"iCnt number := 0; "+ENTER;#查询是否存在此表的记录数变量
        sqlStr +="begin "+ENTER;#begin关键字
        sqlStr +=TAB+"select count(*) into iCnt from user_tables  where  lower(table_name) = lower('"+sheetName+"'); "+ENTER;
        sqlStr +=TAB+"if iCnt = 0 then -- 如果查询不到这个表就创建这个表"+ENTER;
        sqlStr +=TAB+"execute immediate  'create table "+sheetName+" "+ENTER;#组装创建表SQL脚本
        sqlStr +=TAB+"("+ENTER;
        #遍历每个sheet里面的table有多少行
        for row_i in range(nrows):
            if row_i==2:
                #cell(row_i,1)表示从shee的表格的第三行获取第二个列单元信息
                fieldName = sheet.cell(row_i,1).value;#获取字段名称
                fieldType = sheet.cell(row_i,2).value;#获取字段类型
                fieldLength = sheet.cell(row_i, 3).value;#获取字段长度
                isNull = sheet.cell(row_i, 4).value;#获取字段是否为空     
                if fieldLength!='':
                    fieldLength = '%s' %int(fieldLength) #把数字转换成String类型
                else:
                    fieldLength = fieldLength
                fieldLength = '%s' %fieldLength #把数字转换成String类型
                fieldType = fieldType.upper();#转换成大写
                #判断是否为Y/N 
                if isNull!='' and isNull== 'N':#判断字段为N
                    if("VARCHAR2"==fieldType or "NVARCHAR2"==fieldType or "CHAR"==fieldType):
                        sqlStr += TAB+TAB+fieldName+TAB+fieldType+"("+fieldLength+") NOT NULL "+COMMA+ENTER;
                    elif("NUMBER"==fieldType):
                        sqlStr += TAB+TAB+fieldName+TAB+fieldType+"("+fieldLength+") NOT NULL "+COMMA+ENTER;
                    elif("DATE"==fieldType or fieldType.find("TIMESTAMP")!=-1):
                        sqlStr += TAB+TAB+fieldName+TAB+fieldType+COMMA+ENTER;
                    elif("LONG"==fieldType or "NCLOB"==fieldType):
                        sqlStr += TAB+TAB+fieldName+TAB+fieldType+COMMA+ENTER;
                else:   #判断字段为Y
                    if("VARCHAR2"==fieldType or "NVARCHAR2"==fieldType or "CHAR"==fieldType):
                        sqlStr += TAB+TAB+fieldName+TAB+fieldType+"("+fieldLength+") " +COMMA + ENTER;
                    elif("NUMBER"==fieldType):
                        sqlStr += TAB+TAB+fieldName+TAB+fieldType+"("+fieldLength+")" +COMMA + ENTER;
                    elif("DATE"==fieldType or fieldType.find("TIMESTAMP")>-1 ):
                        sqlStr += TAB+TAB+fieldName+TAB+fieldType + COMMA + ENTER;
                    elif("LONG"==fieldType or "NCLOB"==fieldType):
                        sqlStr += TAB+TAB+fieldName+TAB+fieldType+COMMA+ENTER;
            elif row_i>2:
                fieldName = sheet.cell(row_i,1).value;#获取字段名称
                fieldType = sheet.cell(row_i,2).value;#获取字段类型
                fieldLength = sheet.cell(row_i, 3).value;#获取字段长度
                isNull = sheet.cell(row_i, 4).value;#获取字段是否为空           
                if fieldLength!='':
                    #从excel读取出来的数字是有小数点,先转换成整型,由于Python在拼接String的时候,int或double当数字相加了,故要转换成String进行拼接
                    #为了学习阶段目前这里使用spit分割出来,加强进一步学习,double和int是没有split方法
                    newFieldLength = '%s'%fieldLength;#double和int是没有split方法所以转换成一个string类型
                    if newFieldLength.find(".")>-1:  #找到.之后进行处理获取第0个数字进行处理转换
                        fieldLengths = newFieldLength.split(".");#split进行分割
                        fieldLength = fieldLengths[0];#获取第0个数字
                        fieldLength = '%s' %int(fieldLength) #把数字转换成String类型
                else:
                    fieldLength = fieldLength #处理为空的数字长度
                    
                fieldLength = '%s' %fieldLength #把数字转换成String类型
                
                fieldType = fieldType.upper();#转换成大写
                #判断是否为Y/N 
                if isNull!='' and isNull== 'N':#判断字段为N
                    #计算是否最后一个字段
                    if nrows-1 ==row_i :
                        if("VARCHAR2"==fieldType or "NVARCHAR2"==fieldType or "CHAR"==fieldType):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+"("+fieldLength+") NOT NULL "+ ENTER;
                        elif("NUMBER"==fieldType):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+"("+fieldLength+") NOT NULL "+ ENTER;
                        elif("DATE"==fieldType or fieldType.find("TIMESTAMP")!=-1):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+ENTER;
                        elif("LONG"==fieldType or "NCLOB"==fieldType):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+ENTER;
                    else :
                        if("VARCHAR2"==fieldType or "NVARCHAR2"==fieldType or "CHAR"==fieldType):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+"("+fieldLength+") NOT NULL "+COMMA+ ENTER;
                        elif("NUMBER"==fieldType):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+"("+fieldLength+") NOT NULL "+COMMA+ ENTER;
                        elif("DATE"==fieldType or fieldType.find("TIMESTAMP")!=-1):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+COMMA+ENTER;
                        elif("LONG"==fieldType or "NCLOB"==fieldType):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+COMMA+ENTER;
                else : #判断字段为Y
                    #计算是否最后一个字段
                    if nrows-1 ==row_i :
                        if("VARCHAR2"==fieldType or "NVARCHAR2"==fieldType or "CHAR"==fieldType):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+"("+fieldLength+")  "+ ENTER;
                        elif("NUMBER"==fieldType):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+"("+fieldLength+")  "+ ENTER;
                        elif("DATE"==fieldType or fieldType.find("TIMESTAMP")!=-1):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+ENTER;
                        elif("LONG"==fieldType or "NCLOB"==fieldType):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+ENTER;
                    else :
                        if("VARCHAR2"==fieldType or "NVARCHAR2"==fieldType or "CHAR"==fieldType):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+"("+fieldLength+")  "+COMMA+ ENTER;
                        elif("NUMBER"==fieldType):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+"("+fieldLength+")  "+COMMA+ ENTER;
                        elif("DATE"==fieldType or fieldType.find("TIMESTAMP")!=-1):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+COMMA+ENTER;
                        elif("LONG"==fieldType or "NCLOB"==fieldType):
                            sqlStr += TAB+TAB+fieldName+TAB+fieldType+COMMA+ENTER;
        #---nrows for循环结束
        sqlStr += TAB+")'; "+ENTER;#创建表脚本结束
        comentSQL = addTableAndColumnComent(sheet, nrows);  #对表与字段进行添加备注描述
        sqlStr += comentSQL;
        sqlStr += TAB+"end if; "+ENTER;#if脚本结束
        sqlStr += "end; "+ENTER#sql脚本结束
        sqlStr += "/"+ENTER;#sql脚本结束
        #sqlStr=sqlStr.encode('utf8')
        resultSQL += sqlStr; #累计SQL
    #------sheet_names for循环结束
    
    print(resultSQL);#打印结果
    
#--------------添加表名与列的注释方法---------    
def addTableAndColumnComent(sheet,nrows):
    resultSQL ="";
    for row_i in range(nrows):
        if(row_i==0):
            tableName = sheet.cell(row_i,1).value;#获取表名称
            tableNameDesc = sheet.cell(row_i,2).value;#获取表描述
            if tableNameDesc=='' :
                tableNameDesc = " this is "+tableName+" tableName "
            else :
                tableNameDesc = tableNameDesc
        elif (row_i==2): 
            isPrimaryKey = sheet.cell(row_i,0).value;#获取表主键
            tableName = sheet.cell(0,1).value;#获取表名称
            fieldName = sheet.cell(row_i,1).value;#获取字段
            if isPrimaryKey!='' :
                resultSQL += TAB+"execute immediate 'alter table "+tableName+" add constraint PK_"+fieldName+" ";
                resultSQL += " primary  key ("+fieldName+")' ;"+ENTER;
        elif(row_i>2):
            tableName = sheet.cell(0,1).value;#获取表名称   
            fieldName = sheet.cell(row_i,1).value;#获取字段
            fieldDescription = sheet.cell(row_i,6).value;#获取字段描述
            if fieldName!='':
                resultSQL += TAB+"execute immediate 'comment on column "+tableName+"."+fieldName+" is ''"+fieldDescription+"'''"+";"+ENTER;
    #----for end        
    return resultSQL;
 
 
#在很多python脚本中在最后的部分会执行一个判断语句if __name__ == "__main__:",之后还可能会有一些执行语句。那添加这个判断的目的何在?
#在python编译器读取源文件的时候会执行它找到的所有代码,而在执行之前会根据当前运行的模块是否为主程序而定义变量__name__的值为__main__还是模块名。
#因此,该判断语句为真的时候,说明当前运行的脚本为主程序,而非主程序所引用的一个模块。这在当你想要运行一些只有在将模块当做程序运行时而非当做模块引用时才执行
#的命令,只要将它们放到if __name__ == "__main__:"判断语句之后就可以了
if __name__ == '__main__': 
    read_excel()
  
    

十、运行结果

十一、总结

上一篇 下一篇

猜你喜欢

热点阅读