时序数据异常检测相关算法及实现

记录了采用三种不同方法来对不同时间序列进行异常检测的实现代码：

# 导入库

import numpy as np

import pandas as pd

from datetime import datetime

# 导入数据，将时间戳格式进行转换，时间序列的间隔为1分钟

data_raw = pd.read_csv('train.csv')

data_raw["timestamp"]=pd.to_datetime(data_raw.timestamp,unit = "s")

data_raw.rename(columns={"KPI ID":"KPI_ID"},inplace=True)

data_raw.index=data_raw['timestamp']

data_raw.drop(labels = ['timestamp'],axis=1, inplace = True)

data_raw.head()

# 分析数据中有多少条KPI曲线数据

kpi_number = data_raw['KPI_ID'].nunique() #返回对应列的不同值,发现总共有26条不同的KPI_ID

kpi_number

# 绘制26条KPI曲线和异常点

import matplotlib.pyplot as plt

groups = data_raw.groupby(data_raw.KPI_ID)

for name, group in groups:

    plt.figure(figsize=(15,4))

    plt.plot(group['value'])

    plt.title(name, fontsize=20)

    plt.ylabel('Values', fontsize=16)

    a = group.loc[group['label'] == 1, ['value']] #anomaly

    plt.scatter(a.index,a['value'], color='red', label = 'Anomaly')

# 取出第一个kpi相关的指标

kpi_name = list(groups)[0][0]

kpi_data = list(groups)[0][1]

print(kpi_name)

print(kpi_data)

# 方法一：采用z-score方法来处理异常值

# 计算时间序列的均值和平均值

mean = np.mean(kpi_data['value'])

std = np.std(kpi_data['value'])

print(mean,std)

# 按照默认参数计算上下界，这里默认是3

upper_bound = mean + 3 * std

lower_bound = mean - 3 * std

print(upper_bound, lower_bound)

# 根据上下界确定异常点

anomaly_zscore = kpi_data[(kpi_data['value'] > upper_bound)|(kpi_data['value'] < lower_bound)]

plt.rcParams['figure.figsize'] = (24.0, 4.0)

plt.plot(kpi_data.index, kpi_data['value'])

plt.axhline(y = upper_bound, color='green', linestyle='--')

plt.axhline(y = lower_bound, color='green', linestyle='--')

plt.axhline(y = mean, color='red', linestyle='--')

plt.plot(anomaly_zscore.index, anomaly_zscore['value'], 'ro')

# 方法二：采用EWMA，滑动平均的方法，其中a=0.6，窗口w=3

kpi_data['time'] = kpi_data.index

kpi_data.reset_index(drop=True, inplace=True)

kpi_data['value_ewma'] = kpi_data['value']

base = 1 / (1 + 0.6 + 0.36)

for i in range(3, len(kpi_data.index)):

    kpi_data.loc[i, 'value_ewma'] = kpi_data.loc[i, 'value'] + base * 0.6 * kpi_data.loc[i-1, 'value'] + base * 0.36 * kpi_data.loc[i-2, 'value']

plt.rcParams['figure.figsize'] = (24.0, 4.0)

plt.plot(kpi_data.index, kpi_data['value'])

plt.plot(kpi_data.index, kpi_data['value_ewma'], linestyle='--',color='red')

# 定义残差阀值，这里定义的阀值为1.7

kpi_data['is_anomaly_ewma'] = kpi_data['value_ewma'] - kpi_data['value'] > 1.7

anomaly_ewma = kpi_data[kpi_data["is_anomaly_ewma"] == True]

# 绘制图形

plt.rcParams['figure.figsize'] = (24.0, 4.0)

plt.plot(kpi_data.index, kpi_data['value'])

plt.plot(kpi_data.index, kpi_data['value_ewma'], linestyle='--',color='black')

plt.plot(anomaly_ewma.index, anomaly_ewma['value'],'ro')

# 方法三：采用boxplot做异常检测

kpi_data['value_boxplot_upper'] = kpi_data['value'] * 1.1

kpi_data['value_boxplot_lower'] = kpi_data['value'] * 0.9

for i in range(168, len(kpi_data.index)):

    boxplot_samples = [];

    boxplot_samples += kpi_data.loc[i-26:i-22,'value'].values.tolist()

    boxplot_samples += kpi_data.loc[i-50:i-46,'value'].values.tolist()

    boxplot_samples += kpi_data.loc[i-74:i-70,'value'].values.tolist()

    boxplot_samples += kpi_data.loc[i-98:i-94,'value'].values.tolist()

    boxplot_samples += kpi_data.loc[i-122:i-118,'value'].values.tolist()

    boxplot_samples += kpi_data.loc[i-146:i-142,'value'].values.tolist()

    boxplot_samples += kpi_data.loc[i-170:i-166,'value'].values.tolist()

    q3 = np.percentile(boxplot_samples, 75)

    q1 = np.percentile(boxplot_samples, 25)

    iqr = q3 - q1

    kpi_data.loc[i, 'value_boxplot_upper'] = q3 + 12*iqr

    kpi_data.loc[i, 'value_boxplot_lower'] = q1 - 12*iqr

# 绘制图形

plt.rcParams['figure.figsize'] = (24.0, 4.0)

plt.plot(kpi_data.index, kpi_data['value'])

plt.plot(kpi_data.index, kpi_data['value_boxplot_upper'], linestyle='--',color='red')

plt.plot(kpi_data.index, kpi_data['value_boxplot_lower'], linestyle='--',color='green')

# 计算出异常点的位置

kpi_data['is_anomaly_boxplot'] = (kpi_data['value'] > kpi_data['value_boxplot_upper']) | (kpi_data['value'] < kpi_data['value_boxplot_lower'])

anomaly_boxplot = kpi_data[kpi_data['is_anomaly_boxplot'] == True]

# 绘制图形

plt.rcParams['figure.figsize'] = (24.0, 4.0)

plt.plot(kpi_data.index, kpi_data['value'])

plt.plot(anomaly_boxplot.index, anomaly_boxplot['value'], 'ro')