1-导库

# 常见库
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
import os
# 数据预处理，sts是为了adf检验，acf是用来看滞后几期比较合适
import statsmodels.tsa.stattools as sts
from sklearn import preprocessing
from sklearn.metrics import mean_squared_error
from statsmodels.tsa.stattools import acf

# 构建LSTM 所需库
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.utils import to_categorical
from tensorflow import keras
from sklearn.metrics import classification_report

2-数据导入

df_hs : 沪深300的相关数据：收盘价，换手率，成交量，股息率，最大振幅
df_marco : 各宏观指标
考虑披露时效，指标滞后一期(季度滞后3个月，月度滞后一个月)

def read_data(open_path):
    df_hs = pd.read_excel(open_path+'hs300_zhenfu.xlsx',index_col=0)
    df_hs.columns = ['g','close','vol','ts','high','low']
    df_hs['zhenfu'] = 100*(df_hs['high']-df_hs['low'])/df_hs['close']
    df_hs['chg'] = 100*df_hs['close'].pct_change(1)
    df_hs.loc[:,df_hs.columns != 'chg'] = df_hs.loc[:,df_hs.columns != 'chg'].shift(1)
    df_hs.dropna(inplace=True,axis=0)

    df_marco = pd.read_excel(open_path+'宏观数据.xls',index_col=0)
    df_marco.columns = ['gdp','ppi','cpi','pmi','industry_add','investment','current_deposit',\
                        'fixed_deposit_3','fixed_deposit_6','mlf','slf','shibor','rf_1','rf_10',\
                        'm1','m2','inter_borrow','dollar','dollar_rmb','yen_rmb','social_financing',\
                        'foreign_reserve','au','rare','aa+','aa']
    df_marco['m1_m2'] = df_marco['m1'] - df_marco['m2']
    df_marco['qixian'] = df_marco['rf_10'] - df_marco['rf_1']
    df_marco['xinyong_+'] = df_marco['aa+'] - df_marco['rf_1']
    df_marco['xinyong'] = df_marco['aa'] - df_marco['rf_1']

    # 滞后一期，先resample到月，向后推一个月，在resample到天
    day_ = df_marco[['shibor','rf_1','rf_10','dollar','dollar_rmb','yen_rmb','au','aa+','aa','qixian','xinyong_+','xinyong']]
    month_ = df_marco[['ppi','cpi','pmi','industry_add','investment','current_deposit','fixed_deposit_3','fixed_deposit_6',\
                       'slf','m1','m2','m1_m2','inter_borrow','social_financing','rare']]
    quarter_ = df_marco[['gdp','foreign_reserve']]

    day_[['dollar','dollar_rmb','yen_rmb','au']] = 100*day_[['dollar','dollar_rmb','yen_rmb','au']].pct_change(1)
    day_ = day_.shift(1)

    month_ = month_.resample('M').mean()
    month_.fillna(method='ffill',inplace=True)
    month_['pmi'] = 100*month_['pmi'].pct_change(12)
    month_['social_financing'] = 100*month_['social_financing'].pct_change(1)
    month_ = month_.shift(1)
    month_ = month_.resample('d').pad()

    quarter_ = quarter_.resample('3M').mean()
    quarter_.fillna(method='ffill',inplace=True)
    quarter_['foreign_reserve'] = 100*quarter_['foreign_reserve'].pct_change(1)
    quarter_ = quarter_.shift(1)
    quarter_ = quarter_.resample('d').pad()

    df = pd.concat([day_,month_,quarter_],axis=1)
    df = df.fillna(method='ffill')
    df = df.loc[df_hs.index]
    return df_hs, df,df_marco

3- 数据整合

计算技术指标数据
定义x(宏观指标，资产配置指标，利率指标，外汇指标，量价指标，滞后项)和y（沪深300日收益率）
将y按照分位数进行6等分类

def cal_ema(df, N):
    a = 2/(N+1)
    b = pd.DataFrame(columns = ['close'], index=df.index)
    for i in range(len(df)):
        if i == 0:
            b.iloc[i] = df['close'].iloc[i]
        else:
            b.iloc[i] = a * df['close'].iloc[i] + (1-a) * b.iloc[i-1]
    return b

def cal_dea(df, short_t=12, long_t=26 ,avg_t=9):
    ema_short = cal_ema(df, short_t)
    ema_long = cal_ema(df, long_t)
    dif = ema_short - ema_long
    dea = cal_ema(dif, avg_t)
    df['macd'] = (dif-dea)*2
    return df

def cal_adx(df, N=14, M=6):
    hd = df['high'].diff().dropna()
    ld = -df['low'].diff().dropna()
    dmp = pd.DataFrame({'dmp': [0] * len(hd)}, index=hd.index)
    dmp[(hd > 0) & (ld < 0)] = hd
    dmp = dmp.rolling(N).sum().dropna()
    dmm = pd.DataFrame({'dmm': [0] * len(ld)}, index=ld.index)
    dmm[(hd < 0) & (ld > 0)] = ld
    dmm = dmm.rolling(N).sum().dropna()
    temp = pd.concat([df['high'] - df['low'], abs(df['high'] - df['close'].shift(1)), \
                      abs(df['low'] - df['close'].shift(1))], axis=1).dropna()
    tr = temp.max(axis=1).dropna()

    s_index = dmm.index & tr.index & dmp.index
    dmp = dmp.loc[s_index]
    dmm = dmm.loc[s_index]
    tr = tr.loc[s_index]
    pdi = 100 * dmp['dmp'] / tr
    mdi = dmm['dmm'] * 100 / tr

    dx = abs(pdi - mdi) / (pdi + mdi) * 100
    adx = dx.rolling(M).mean().dropna()
    adx = pd.DataFrame(adx, columns=['adx'])
    return adx

def handle_hs(df_hs):
    df_hs = cal_dea(df_hs)
    df_hs['adx'] = cal_adx(df_hs)
    df_hs.dropna(inplace=True)
    return df_hs


def handle_data(df_hs,df_marco):
    df_hs = handle_hs(df_hs)
    df_marco = df_marco.loc[df_hs.index]

    df_hs = df_hs[df_hs.index.year>=2009]
    df_marco = df_marco[df_marco.index.year>=2009]

    #重新定义y 和 x
    y = df_hs.loc[:,'chg']
    # 找分位数
    for i in range(1,6):
        print( y.quantile(i/6))
        print(i/6)
        print("")
    y_new = y.copy()
    y_new[1.24 <= y_new] = 5
    y_new[(0.46 <= y_new) & (y_new < 1.24)] = 4
    y_new[(0.06 <= y_new) & (y_new < 0.46)] = 3
    y_new[(-0.37 <= y_new) & (y_new < 0.06)] = 2
    y_new[(-1.04 <= y_new) & (y_new < -0.37)] = 1
    y_new[y_new < -1.04] = 0

    y_new.columns = ['label']
    y_new_count = y_new.groupby(y_new).count()
    y_new_count =pd.concat([ pd.Series(['<-1.04','-1.04~-0.37','-0.37~-0.06',\
                                   '0.06~0.46','0.46~1.24','>1.24']),y_new_count],axis=1)
    y_new_count.columns = ['分位数定义','num']
    y_new_count.to_excel(save_path+'lstm的y分类.xls')

    #去除相关性不高的因子
    x = pd.concat([df_hs[['g','vol','ts','zhenfu','macd','adx']],df_marco],axis=1)
    x = x.astype('float')
    dic = {}
    for i in x.columns:
        temp = x[i]
        dic[i] = temp.corr(y)
        print(i)
    df = pd.DataFrame.from_dict(dic,orient='index')
    df.to_excel(save_path+'LSTM因子相关性.xls')
    df_temp = df[abs(df)>0.01]
    df_temp.dropna(inplace=True)

    target = ['g','vol','ts','zhenfu','macd','adx','shibor','rf_10','dollar','aa','xinyong','cpi','m1_m2']
    df_temp2 = df_temp.loc[target]
    df_temp2.to_excel(save_path+'lstm筛选后因子相关性.xls')
    x = x[target]
    return x,y_new

4-检验数据

相关性过滤--单位根检验--PCA取主成因子(不存在共线性)

def again_handle_data(x,y,df_backup):
    def adf_test(x,y):
        dic = {}
        dic['y'] = sts.adfuller(y)[1]
        for i in x.columns:
            dic[i] = sts.adfuller(x[i])[1]
        df = pd.DataFrame.from_dict(dic,orient='index')
        target = list(df[df>0.01].dropna().index)
        if len(target)>0:
            print(target)
        else:
            print('所有因子通过单位根检验')
        return df,target

    def adf_data(x, y, df_backup):
        df1, target1 = adf_test(x, y)
        for i in target1:
            x[i] = x[i].diff(1)
        x.dropna(inplace=True)
        y = y[x.index]

        # 重新diff cpi 和 m1-m2，然后赋值给x
        month_ = df_backup[['cpi','m1_m2']]
        month_.dropna(inplace=True)
        month_ = month_.shift(1)
        month_[['cpi','m1_m2']] = month_[['cpi','m1_m2']].diff(1)
        month_.dropna(inplace=True)
        for i in month_.columns:
            print(sts.adfuller(month_[i])[1])

        a = list(x.columns)
        a.remove('cpi')
        a.remove('m1_m2')
        b = x[a]
        c = pd.concat([b,month_],axis=1)
        x = c.fillna(method='bfill')
        x = x[x.index.year>=2009]
        x.dropna(inplace=True)
        s_index = x.index & y.index
        x = x.loc[s_index]
        y = y[s_index]
        # df2, target2 = adf_test(x, y)
        # df1.to_excel(save_path + 'lstm单位根检验.xls')
        # df2.to_excel(save_path + 'lstm差分后单位根检验.xls')
        return x, y
    x,y = adf_data(x,y,df_backup)
    x_new = preprocessing.scale(x,axis=0)
    x_new = pd.DataFrame(x_new,index=x.index,columns=x.columns)
    mat = np.dot(x_new.T, x_new)
    l,eig,r = np.linalg.svd(mat)
    eig = eig / eig.sum()
    eig = eig.cumsum()

    i = 0
    for e in eig:
        i += 1
        if e > 0.99:
            n_pca = i
            break
    print('最大特征数目：' + str(n_pca))

    # b = pd.DataFrame(y)
    y_new = to_categorical(y)
    y_new = pd.DataFrame(y_new,index=y.index)
    return x_new,y_new

5-数据准备

lstm要求数据输入有特定格式，n_steps=2是根据acf得到

def pre_data(x,y,n_steps=2):
    data = np.hstack([np.array(x),np.array(y)])

    n_feature = x.shape[1]
    train_pos = 0.9

    result = []
    for s in range(len(data)-n_steps):
        temp = data[s:s+n_steps]
        result.append(temp)
    result = np.array(result)

    row = round(train_pos * result.shape[0])
    x_train = result[:row, :, :-6]
    x_test = result[row:, :, :-6]
    y_train = result[:row, -1, -6:]
    y_test = result[row:, -1, -6:]

    x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], n_feature))
    x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], n_feature))
    return x_train, x_test, y_train, y_test

6-模型建立

由于对数据有结构要求，要建立两层，需要return_sequences=True

def bulid_model(x_train, neurons=[128,128,6], dropout=0.2):
    model = Sequential()
    model.add(LSTM(neurons[0], input_shape=(x_train.shape[1], x_train.shape[2]), return_sequences=False))
    model.add(Dropout(dropout))
    model.add(Dense(neurons[2], activation='softmax'))
    adam = keras.optimizers.Adam(decay=0.2)
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    model.summary()

  # 建立两层模型
   # model = Sequential()
   # model.add(LSTM(neurons[0], input_shape=(x_train.shape[1], x_train.shape[2]), return_sequences=True))
   # model.add(Dropout(dropout))
   # model .add(LSTM(neurons[1]))
   # model.add(Dropout(dropout))
   # model.add(Dense(neurons[2], activation='softmax'))
   # adam = keras.optimizers.Adam(decay=0.2)
   # model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
   # model.summary()

    return model

7-运行模型

def run_model(x_train,y_train,save_path, neurons=[128,128,9], dropout=0.2):
    model = bulid_model(x_train, neurons=[128,128,6], dropout=0.2)
    history = model.fit(x_train,y_train,epochs=32,batch_size=64,verbose=2)

    fig = plt.figure()
    ax = fig.add_subplot(111)
    pic1 = ax.plot(history.history['loss'], label='loss',color='b')
    ax2 = ax.twinx()
    pic2 = ax2.plot(history.history['accuracy'],label='acc',color='r')
    pic = pic1+pic2
    labs = [l.get_label() for l in pic]
    ax.legend(pic,labs,loc=2)
    plt.savefig(save_path+'LSTM_loss.jpg',dpi=300)
    plt.show()
    plt.close()


def model_score(model, x_train, y_train, x_test, y_test):
    trainScore = model.evaluate(x_train, y_train, verbose=0)
    print('Train Score: %.5f MSE (%.2f RMSE)' % (trainScore[0], np.sqrt(trainScore[0])))

    testScore = model.evaluate(x_test, y_test, verbose=0)
    print('Test Score: %.5f MSE (%.2f RMSE)' % (testScore[0], np.sqrt(testScore[0])))
    return trainScore, testScore        

8- 预测模型

def predict_model(model,x_test,y_test,save_path,x,train_pos=0.9):

    y_pred_prob = model.predict(x_test)
    #y_pred = tran_porb(y_pred_prob)
    y_class = model.predict_classes(x_test)
    y_pred = to_categorical(y_class)
    target = ['<-1.04', '-1.04~-0.37', '-0.37~-0.06', '0.06~0.46', '0.46~1.24', '>1.24']
    print(classification_report(y_test, y_pred, target_names=target))    

9-保存模型

读取模型中 直接
from keras.models 有可能会报错，查询了一下貌似是keras版本太高的问题
from tensorflow.keras.models 既可以了

model.save(save_path+'lstm.h5')

# 读取模型
#from tensorflow.keras.models import load_model

#model = load_model(save_path+'lstm.h5'

完整代码

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
import os
import statsmodels.tsa.stattools as sts
from sklearn import preprocessing
from statsmodels.stats.outliers_influence import variance_inflation_factor
from sklearn import linear_model
from sklearn.metrics import mean_squared_error
from statsmodels.graphics.tsaplots import plot_acf, plot_pacf
import statsmodels.api as sm
# from statsmodels.tsa.api import VAR
from statsmodels.tsa.vector_ar.var_model import VAR
from statsmodels.tsa.stattools import acf
from arch import arch_model

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.utils import to_categorical
from tensorflow import keras
from sklearn.metrics import classification_report



import ReadData as RD
import PreHandle as PH
import warnings

warnings.filterwarnings('ignore')

#%%
# 提取原始信息
def read_data(open_path):
    df_hs = pd.read_excel(open_path+'hs300_zhenfu.xlsx',index_col=0)
    df_hs.columns = ['g','close','vol','ts','high','low']
    df_hs['zhenfu'] = 100*(df_hs['high']-df_hs['low'])/df_hs['close']
    df_hs['chg'] = 100*df_hs['close'].pct_change(1)
    df_hs.loc[:,df_hs.columns != 'chg'] = df_hs.loc[:,df_hs.columns != 'chg'].shift(1)
    df_hs.dropna(inplace=True,axis=0)

    df_marco = pd.read_excel(open_path+'宏观数据.xls',index_col=0)
    df_marco.columns = ['gdp','ppi','cpi','pmi','industry_add','investment','current_deposit',\
                        'fixed_deposit_3','fixed_deposit_6','mlf','slf','shibor','rf_1','rf_10',\
                        'm1','m2','inter_borrow','dollar','dollar_rmb','yen_rmb','social_financing',\
                        'foreign_reserve','au','rare','aa+','aa']
    df_marco['m1_m2'] = df_marco['m1'] - df_marco['m2']
    df_marco['qixian'] = df_marco['rf_10'] - df_marco['rf_1']
    df_marco['xinyong_+'] = df_marco['aa+'] - df_marco['rf_1']
    df_marco['xinyong'] = df_marco['aa'] - df_marco['rf_1']

    # 滞后一期，先resample到月，向后推一个月，在resample到天
    day_ = df_marco[['shibor','rf_1','rf_10','dollar','dollar_rmb','yen_rmb','au','aa+','aa','qixian','xinyong_+','xinyong']]
    month_ = df_marco[['ppi','cpi','pmi','industry_add','investment','current_deposit','fixed_deposit_3','fixed_deposit_6',\
                       'slf','m1','m2','m1_m2','inter_borrow','social_financing','rare']]
    quarter_ = df_marco[['gdp','foreign_reserve']]

    #day_.dropna(inplace=True)
    # day_['dollar'] = day_['dollar'].pct_change(1)
    # day_['dollar_rmb'] = day_['dollar_rmb'].pct_change(1)
    # day_['yen_rmb'] = day_['yen_rmb'].pct_change(1)
    day_[['dollar','dollar_rmb','yen_rmb','au']] = 100*day_[['dollar','dollar_rmb','yen_rmb','au']].pct_change(1)
    day_ = day_.shift(1)
    #day_.dropna(inplace=True)

    month_ = month_.resample('M').mean()
    month_.fillna(method='ffill',inplace=True)
    month_['pmi'] = 100*month_['pmi'].pct_change(12)
    month_['social_financing'] = 100*month_['social_financing'].pct_change(1)
    month_ = month_.shift(1)
    month_ = month_.resample('d').pad()

    quarter_ = quarter_.resample('3M').mean()
    quarter_.fillna(method='ffill',inplace=True)
    quarter_['foreign_reserve'] = 100*quarter_['foreign_reserve'].pct_change(1)
    quarter_ = quarter_.shift(1)
    quarter_ = quarter_.resample('d').pad()

    df = pd.concat([day_,month_,quarter_],axis=1)
    df = df.fillna(method='ffill')
    df = df.loc[df_hs.index]
    return df_hs, df,df_marco


# 提取技术性指标
def cal_ema(df, N):
    a = 2/(N+1)
    b = pd.DataFrame(columns = ['close'], index=df.index)
    for i in range(len(df)):
        if i == 0:
            b.iloc[i] = df['close'].iloc[i]
        else:
            b.iloc[i] = a * df['close'].iloc[i] + (1-a) * b.iloc[i-1]
    return b

def cal_dea(df, short_t=12, long_t=26 ,avg_t=9):
    ema_short = cal_ema(df, short_t)
    ema_long = cal_ema(df, long_t)
    dif = ema_short - ema_long
    dea = cal_ema(dif, avg_t)
    df['macd'] = (dif-dea)*2
    return df

def cal_adx(df, N=14, M=6):
    hd = df['high'].diff().dropna()
    ld = -df['low'].diff().dropna()
    dmp = pd.DataFrame({'dmp': [0] * len(hd)}, index=hd.index)
    dmp[(hd > 0) & (ld < 0)] = hd
    dmp = dmp.rolling(N).sum().dropna()
    dmm = pd.DataFrame({'dmm': [0] * len(ld)}, index=ld.index)
    dmm[(hd < 0) & (ld > 0)] = ld
    dmm = dmm.rolling(N).sum().dropna()
    temp = pd.concat([df['high'] - df['low'], abs(df['high'] - df['close'].shift(1)), \
                      abs(df['low'] - df['close'].shift(1))], axis=1).dropna()
    tr = temp.max(axis=1).dropna()

    s_index = dmm.index & tr.index & dmp.index
    dmp = dmp.loc[s_index]
    dmm = dmm.loc[s_index]
    tr = tr.loc[s_index]
    pdi = 100 * dmp['dmp'] / tr
    mdi = dmm['dmm'] * 100 / tr

    dx = abs(pdi - mdi) / (pdi + mdi) * 100
    adx = dx.rolling(M).mean().dropna()
    adx = pd.DataFrame(adx, columns=['adx'])
    return adx

def handle_hs(df_hs):
    df_hs = cal_dea(df_hs)
    df_hs['adx'] = cal_adx(df_hs)
    df_hs.dropna(inplace=True)
    return df_hs



def handle_data(df_hs,df_marco):
    df_hs = handle_hs(df_hs)
    df_marco = df_marco.loc[df_hs.index]

    df_hs = df_hs[df_hs.index.year>=2009]
    df_marco = df_marco[df_marco.index.year>=2009]

    #重新定义y 和 x
    y = df_hs.loc[:,'chg']
    # 找分位数
    for i in range(1,6):
        print( y.quantile(i/6))
        print(i/6)
        print("")
    y_new = y.copy()
    y_new[1.24 <= y_new] = 5
    y_new[(0.46 <= y_new) & (y_new < 1.24)] = 4
    y_new[(0.06 <= y_new) & (y_new < 0.46)] = 3
    y_new[(-0.37 <= y_new) & (y_new < 0.06)] = 2
    y_new[(-1.04 <= y_new) & (y_new < -0.37)] = 1
    y_new[y_new < -1.04] = 0

    y_new.columns = ['label']
    y_new_count = y_new.groupby(y_new).count()
    y_new_count =pd.concat([ pd.Series(['<-1.04','-1.04~-0.37','-0.37~-0.06',\
                                   '0.06~0.46','0.46~1.24','>1.24']),y_new_count],axis=1)
    y_new_count.columns = ['分位数定义','num']
    y_new_count.to_excel(save_path+'lstm的y分类.xls')

    #去除相关性不高的因子
    x = pd.concat([df_hs[['g','vol','ts','zhenfu','macd','adx']],df_marco],axis=1)
    x = x.astype('float')
    dic = {}
    for i in x.columns:
        temp = x[i]
        dic[i] = temp.corr(y)
        print(i)
    df = pd.DataFrame.from_dict(dic,orient='index')
    df.to_excel(save_path+'LSTM因子相关性.xls')
    df_temp = df[abs(df)>0.01]
    df_temp.dropna(inplace=True)

    target = ['g','vol','ts','zhenfu','macd','adx','shibor','rf_10','dollar','aa','xinyong','cpi','m1_m2']
    df_temp2 = df_temp.loc[target]
    df_temp2.to_excel(save_path+'lstm筛选后因子相关性.xls')
    x = x[target]
    return x,y_new

def again_handle_data(x,y,df_backup):
    def adf_test(x,y):
        dic = {}
        dic['y'] = sts.adfuller(y)[1]
        for i in x.columns:
            dic[i] = sts.adfuller(x[i])[1]
        df = pd.DataFrame.from_dict(dic,orient='index')
        target = list(df[df>0.01].dropna().index)
        if len(target)>0:
            print(target)
        else:
            print('所有因子通过单位根检验')
        return df,target

    def adf_data(x, y, df_backup):
        df1, target1 = adf_test(x, y)
        for i in target1:
            x[i] = x[i].diff(1)
        x.dropna(inplace=True)
        y = y[x.index]

        # 重新diff cpi 和 m1-m2，然后赋值给x
        month_ = df_backup[['cpi','m1_m2']]
        month_.dropna(inplace=True)
        month_ = month_.shift(1)
        month_[['cpi','m1_m2']] = month_[['cpi','m1_m2']].diff(1)
        month_.dropna(inplace=True)
        for i in month_.columns:
            print(sts.adfuller(month_[i])[1])

        a = list(x.columns)
        a.remove('cpi')
        a.remove('m1_m2')
        b = x[a]
        c = pd.concat([b,month_],axis=1)
        x = c.fillna(method='bfill')
        x = x[x.index.year>=2009]
        x.dropna(inplace=True)
        s_index = x.index & y.index
        x = x.loc[s_index]
        y = y[s_index]
        # df2, target2 = adf_test(x, y)
        # df1.to_excel(save_path + 'lstm单位根检验.xls')
        # df2.to_excel(save_path + 'lstm差分后单位根检验.xls')
        return x, y
    x,y = adf_data(x,y,df_backup)
    x_new = preprocessing.scale(x,axis=0)
    x_new = pd.DataFrame(x_new,index=x.index,columns=x.columns)
    mat = np.dot(x_new.T, x_new)
    l,eig,r = np.linalg.svd(mat)
    eig = eig / eig.sum()
    eig = eig.cumsum()

    i = 0
    for e in eig:
        i += 1
        if e > 0.99:
            n_pca = i
            break
    print('最大特征数目：' + str(n_pca))

    # b = pd.DataFrame(y)
    y_new = to_categorical(y)
    y_new = pd.DataFrame(y_new,index=y.index)
    return x_new,y_new

def pre_data(x,y,n_steps=2):
    data = np.hstack([np.array(x),np.array(y)])

    n_feature = x.shape[1]
    train_pos = 0.9

    result = []
    for s in range(len(data)-n_steps):
        temp = data[s:s+n_steps]
        result.append(temp)
    result = np.array(result)

    row = round(train_pos * result.shape[0])
    x_train = result[:row, :, :-6]
    x_test = result[row:, :, :-6]
    y_train = result[:row, -1, -6:]
    y_test = result[row:, -1, -6:]

    x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], n_feature))
    x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], n_feature))
    return x_train, x_test, y_train, y_test

def bulid_model(x_train, neurons=[128,128,6], dropout=0.2):
    model = Sequential()
    model.add(LSTM(neurons[0], input_shape=(x_train.shape[1], x_train.shape[2]), return_sequences=False))
    model.add(Dropout(dropout))
    model.add(Dense(neurons[2], activation='softmax'))
    adam = keras.optimizers.Adam(decay=0.2)
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    model.summary()
    return model

    trainScore, testScore = model_score(model,x_train, y_train, x_test, y_test)

def run_model(x_train,y_train,save_path, neurons=[128,128,9], dropout=0.2):
    model = bulid_model(x_train, neurons=[128,128,6], dropout=0.2)
    history = model.fit(x_train,y_train,epochs=32,batch_size=64,verbose=2)

    fig = plt.figure()
    ax = fig.add_subplot(111)
    pic1 = ax.plot(history.history['loss'], label='loss',color='b')
    ax2 = ax.twinx()
    pic2 = ax2.plot(history.history['accuracy'],label='acc',color='r')
    pic = pic1+pic2
    labs = [l.get_label() for l in pic]
    ax.legend(pic,labs,loc=2)
    plt.savefig(save_path+'LSTM_loss.jpg',dpi=300)
    plt.show()
    plt.close()


def model_score(model, x_train, y_train, x_test, y_test):
    trainScore = model.evaluate(x_train, y_train, verbose=0)
    print('Train Score: %.5f MSE (%.2f RMSE)' % (trainScore[0], np.sqrt(trainScore[0])))

    testScore = model.evaluate(x_test, y_test, verbose=0)
    print('Test Score: %.5f MSE (%.2f RMSE)' % (testScore[0], np.sqrt(testScore[0])))
    return trainScore, testScore

def predict_model(model,x_test,y_test,save_path,x,train_pos=0.9):

    y_pred_prob = model.predict(x_test)
    #y_pred = tran_porb(y_pred_prob)
    y_class = model.predict_classes(x_test)
    y_pred = to_categorical(y_class)
    target = ['<-1.04', '-1.04~-0.37', '-0.37~-0.06', '0.06~0.46', '0.46~1.24', '>1.24']
    print(classification_report(y_test, y_pred, target_names=target))



    # target = ['<-1.04', '-1.04~-0.37', '-0.37~-0.06', '0.06~0.46', '0.46~1.24', '>1.24']
    # print(classification_report(y_test, y_pred, target_names=target))
    row = round(train_pos * x.shape[0])
    s_index = x.index[row:]
    y_pred_prob = pd.DataFrame(y_pred_prob,columns = target,index=s_index)
    y_pred = pd.DataFrame(y_pred,columns=target,index=s_index)
    y_test = pd.DataFrame(y_test,columns=target,index=s_index)
    y_class = pd.DataFrame(y_class,index=s_index)
    y_pred_prob.to_excel(save_path+'LSTM预测概率.xls')
    y_pred.to_excel(save_path+'LSTM预测类别dummy.xls')
    y_test.to_excel(save_path+'LSTM真实值类别.xls')
    y_class.to_excel(save_path+'LSTM预测类别class.xls')
    return y_pred_prob,y_pred

def tran_porb(y_pred_prob):
    y_pred = np.zeros((y_pred_prob.shape[0],y_pred_prob.shape[1]))
    x_max = y_pred_prob.argmax(axis=1)
    for i in range(y_pred_prob.shape[0]):
        y_pred[i,x_max[i]] = 1
    return y_pred


def pic_lstm(df_hs,y_class):
    y = df_hs['chg']
    y_new = y.copy()
    y_new[1.24 <= y_new] = 5
    y_new[(0.46 <= y_new) & (y_new < 1.24)] = 4
    y_new[(0.06 <= y_new) & (y_new < 0.46)] = 3
    y_new[(-0.37 <= y_new) & (y_new < 0.06)] = 2
    y_new[(-1.04 <= y_new) & (y_new < -0.37)] = 1
    y_new[y_new < -1.04] = 0
    y_new = y_new[y_class.index]

    df = pd.concat([y_new,y_class],axis=1)
    df.columns = ['真实值','预测值']
    df.to_excel(save_path+'预测汇总/lstm画图.xls')

    fig = plt.figure(figsize=(12,8))
    plt.plot(df['真实值'])
    plt.plot(df['预测值'])
    plt.show()
    plt.close()




#%%
if __name__ == '__main__':
    open_path = 'data1115/'
    view_path = 'view_path/'
    save_path = 'save_path/'

    #中文字体
    matplotlib.rcParams['axes.unicode_minus'] = False
    plt.rcParams['font.sans-serif'] = ['SimHei']

    df_hs,df_marco,df_backup = read_data(open_path)
    x,y = handle_data(df_hs,df_marco,)
    x,y = again_handle_data(x,y,df_backup)

    x_train, x_test, y_train, y_test = pre_data(x, y)


    model = bulid_model(x_train)
    model.save(save_path+'LSTM.H5')
    run_model(x_train, y_train, save_path)

    #trainScore, testScore = model_score(model, x_train, y_train, x_test, y_test)
    y_pred_prob, y_pred = predict_model(model,x_test,y_test,save_path,x,train_pos=0.9)