大师兄的Python机器学习笔记:数据预处理
2020-04-19 本文已影响0人
superkmi
大师兄的Python机器学习笔记:Numpy库、Scipy库和Matplotlib库 (三)
大师兄的Python机器学习笔记:数据重抽样
一、获得数据
- 机器学习需要大量的真实数据，可以通过互联网获得。
1. 关于Kaggle
- Kaggle(https://www.kaggle.com/)成立于2010年，是一个进行数据发掘和预测竞赛的在线平台。
- 通过Kaggle，我们可以获得一些真实数据。
2. 下载数据
- 例如，我们可以在 此处下载美国各县的新冠肺炎数据。
3. 读取数据
- 由于数据基本都是.csv或者.json格式的，可以用python直接读取。
import os
from pprint import pprint
def read_csv(file):
# 读取数据并转换为list
with open(file,'r') as f:
return list(f.readlines())
if __name__ == '__main__':
file_path = os.path.join('D:\\','dataset','us-counties.csv')
data = read_csv(file_path)
pprint(data) # 为了让数据看起来更直观，使用pprint
['date,county,state,fips,cases,deaths\n',
'2020-01-21,Snohomish,Washington,53061,1,0\n',
'2020-01-22,Snohomish,Washington,53061,1,0\n',
'2020-01-23,Snohomish,Washington,53061,1,0\n',
'2020-01-24,Cook,Illinois,17031,1,0\n',
'2020-01-24,Snohomish,Washington,53061,1,0\n',
'2020-01-25,Orange,California,06059,1,0\n',
'2020-01-25,Cook,Illinois,17031,1,0\n',
'2020-01-25,Snohomish,Washington,53061,1,0\n',
... ... (省略)
'2020-04-13,Teton,Wyoming,56039,56,0\n',
'2020-04-13,Uinta,Wyoming,56041,4,0\n',
'2020-04-13,Washakie,Wyoming,56043,4,0']
二、判断数据缺失
- 为了保证结果的准确性，需要对缺失数据进行处理。
1. 筛选完整数据
- 通过判断跳过不完整的数据。
>>>import os
>>>from csv import reader
>>>from pprint import pprint
>>>def read_csv(file):
>>> # 读取数据，跳过缺失的行或数据不完整的行
>>> dataset = []
>>> with open(file,'r') as f:
>>> lines = list(reader(f))
>>> data_len = len(list(lines)[0]) # 获取标题列的长度
>>> for line in lines:
>>> if line and len(line) == data_len: # 如果行为空或者数据不完整则跳过
>>> dataset.append(line)
>>> return dataset
>>>if __name__ == '__main__':
>>> file_path = os.path.join('D:\\','dataset','us-counties.csv')
>>> data = read_csv(file_path)
>>> pprint(data)
2. 判断元素是否缺失**
- 检查每个元素是否有缺失的
>>>import os
>>>import pandas as pd
>>>from pprint import pprint
>>>def read_csv(file):
>>> return pd.read_csv(file)
>>>def find_null(data):
>>> return data.isnull()
>>>if __name__ == '__main__':
>>> file_path = os.path.join('D:\\','dataset','us-counties.csv')
>>> data = read_csv(file_path)
>>> pprint(find_null(data))
date county state fips cases deaths
0 False False False False False False
1 False False False False False False
2 False False False False False False
3 False False False False False False
4 False False False False False False
... ... ... ... ... ... ...
56536 False False False False False False
56537 False False False False False False
56538 False False False False False False
56539 False False False False False False
56540 False False False False False False
[56541 rows x 6 columns]
3. 判断缺失列
- 检查每列是否包含缺失的元素。
>>>import os
>>>import pandas as pd
>>>from pprint import pprint
>>>def read_csv(file):
>>> return pd.read_csv(file)
>>>def find_null_column(data):
>>> return data.isnull().any()
>>>if __name__ == '__main__':
>>> file_path = os.path.join('D:\\','dataset','us-counties.csv')
>>> data = read_csv(file_path)
>>> pprint(find_null_column(data))
date False
county False
state False
fips True
cases False
deaths False
dtype: bool
4. 统计缺失元素
- 统计每列缺失元素的数目。
>>>import os
>>>import pandas as pd
>>>from pprint import pprint
>>>def read_csv(file):
>>> return pd.read_csv(file)
>>>def find_null_column(data):
>>> return data.isnull().any()
>>>def count_null(data,null_column):
>>> missing = data.columns[null_column].tolist()
>>> return data[missing].isnull().sum()
>>>if __name__ == '__main__':
>>> file_path = os.path.join('D:\\','dataset','us-counties.csv')
>>> data = read_csv(file_path)
>>> null_column = find_null_column(data)
>>> print(count_null(data,null_column))
fips 746
dtype: int64
5. 替换缺失值
- 将缺失值替换为一个默认值。
>>>import os
>>>import pandas as pd
>>>from pprint import pprint
>>>def read_csv(file):
>>> # 获得文件中的数据
>>> return pd.read_csv(file)
>>>def find_null_column(data):
>>> # 返回所有包含空的列
>>> return data.isnull().any()
>>>def get_null_column_name(null_column):
>>> # 返回包含空列的列名
>>> return data.columns[null_column].tolist()
>>>def replace_null(data,columns,value):
>>> # 替换空值
>>> for column in columns:
>>> data.loc[data[column].isnull(),column] = value
>>> return data
>>>if __name__ == '__main__':
>>> file_path = os.path.join('D:\\','dataset','us-counties.csv')
>>> data = read_csv(file_path)
>>> null_column_list = get_null_column_name(find_null_column(data)) # 获得空列名
>>> new_data = replace_null(data,null_column_list,0) # 将空数据替换为0
>>> pprint(find_null_column(new_data))
date False
county False
state False
fips False
cases False
deaths False
dtype: bool
6. 缺失值比对
- 判断两列的缺失值是否同时为空，并获得对比数据。
>>>import os
>>>import pandas as pd
>>>from pprint import pprint
>>>def read_csv(file):
>>> # 获得文件中的数据
>>> return pd.read_csv(file)
>>>def compare_columns(col1,col2):
>>> # 对比两列的缺失值
>>> res = data[[col1,col2]][data[col2].isnull()==True]
>>> # 获得对比数据
>>> return res.describe()
>>>if __name__ == '__main__':
>>> file_path = os.path.join('D:\\','dataset','us-counties.csv')
>>> data = read_csv(file_path)
>>> pprint(compare_columns('state','fips'))
fips
count 0.0
mean NaN
std NaN
min NaN
25% NaN
50% NaN
75% NaN
max NaN
三. 数据类型转换
- 为了保障结果的统一性，需要尽量将数据类型转换为浮点数(float)。
>>>import os
>>>import pandas as pd
>>>from pprint import pprint
>>>def read_csv(file):
>>> # 获得文件中的数据
>>> return pd.read_csv(file)
>>>def to_float(data):
>>> for column in data:
>>> if column =='date':continue # 跳过日期
>>> if str(data[column][1]).isdigit(): # 如果是数字
>>> data[column] = data[column].astype('float') # 将列转为浮点数
>>> return data
>>>if __name__ == '__main__':
>>> file_path = os.path.join('D:\\','dataset','us-counties.csv')
>>> data = read_csv(file_path)
>>> print(to_float(data))
date county state fips cases deaths
0 2020-01-21 Snohomish Washington 53061.0 1.0 0.0
1 2020-01-22 Snohomish Washington 53061.0 1.0 0.0
2 2020-01-23 Snohomish Washington 53061.0 1.0 0.0
3 2020-01-24 Cook Illinois 17031.0 1.0 0.0
4 2020-01-24 Snohomish Washington 53061.0 1.0 0.0
... ... ... ... ... ... ...
56536 2020-04-13 Sublette Wyoming 56035.0 1.0 0.0
56537 2020-04-13 Sweetwater Wyoming 56037.0 9.0 0.0
56538 2020-04-13 Teton Wyoming 56039.0 56.0 0.0
56539 2020-04-13 Uinta Wyoming 56041.0 4.0 0.0
56540 2020-04-13 Washakie Wyoming 56043.0 4.0 0.0
[56541 rows x 6 columns]
四. 数据特征缩放
- 为了保证数据的特征具有相近的尺度，有时需要对数据进行特征缩放。
1. 归一化(Rescaling)
- 将所有特征缩放到0～1之间，使梯度下降法能更快的收敛。
- 公式
>>>import os
>>>import pandas as pd
>>>import numpy as np
>>>from pprint import pprint
>>>def read_csv(file):
>>> # 获得文件中的数据
>>> return pd.read_csv(file)
>>>def to_float(data):
>>> # 将数据改为浮点数
>>> for column in data:
>>> if column =='date':continue # 跳过日期
>>> if str(data[column][1]).isdigit(): # 如果是数字
>>> data[column] = data[column].astype('float') # 将列转为浮点数
>>> return data
>>>def min_max_normalization(data):
>>> # 归一化特征缩放
>>> for column in data:
>>> if column == 'date': continue # 跳过日期
>>> if isinstance(data[column][1],float): # 如果是浮点数
>>> x = data[column]
>>> x = (x - np.min(x))/(np.max(x)-np.min(x))
>>> data[column] = x
>>> return data
>>>if __name__ == '__main__':
>>> file_path = os.path.join('D:\\','dataset','us-counties.csv')
>>> data = read_csv(file_path)
>>> pprint(min_max_normalization(to_float(data)))
date county state fips cases deaths
0 2020-01-21 Snohomish Washington 0.945823 0.000008 0.0
1 2020-01-22 Snohomish Washington 0.945823 0.000008 0.0
2 2020-01-23 Snohomish Washington 0.945823 0.000008 0.0
3 2020-01-24 Cook Illinois 0.291232 0.000008 0.0
4 2020-01-24 Snohomish Washington 0.945823 0.000008 0.0
... ... ... ... ... ... ...
61966 2020-04-15 Sublette Wyoming 0.999855 0.000008 0.0
61967 2020-04-15 Sweetwater Wyoming 0.999891 0.000085 0.0
61968 2020-04-15 Teton Wyoming 0.999927 0.000499 0.0
61969 2020-04-15 Uinta Wyoming 0.999964 0.000034 0.0
61970 2020-04-15 Washakie Wyoming 1.000000 0.000034 0.0
[61971 rows x 6 columns]
2. 均值归一化(Mean Normalization)
- 归一化的另一种方法，数据离平均值的距离。
- 公式
>>>import os
>>>import pandas as pd
>>>import numpy as np
>>>from pprint import pprint
>>>def read_csv(file):
>>> # 获得文件中的数据
>>> return pd.read_csv(file)
>>>def to_float(data):
>>> # 将数据改为浮点数
>>> for column in data:
>>> if column =='date':continue # 跳过日期
>>> if str(data[column][1]).isdigit(): # 如果是数字
>>> data[column] = data[column].astype('float') # 将列转为浮点数
>>> return data
>>>def mean_normalization(data):
>>> # 均值归一化特征缩放
>>> for column in data:
>>> if column == 'date': continue # 跳过日期
>>> if isinstance(data[column][1],float): # 如果是浮点数
>>> x = data[column]
>>> x = (x - np.mean(x))/(np.max(x)-np.min(x))
>>> data[column] = x
>>> return data
>>>if __name__ == '__main__':
>>> file_path = os.path.join('D:\\','dataset','us-counties.csv')
>>> data = read_csv(file_path)
>>> pprint(mean_normalization(to_float(data)))
date county state fips cases deaths
0 2020-01-21 Snohomish Washington 0.426211 -0.001020 -0.00049
1 2020-01-22 Snohomish Washington 0.426211 -0.001020 -0.00049
2 2020-01-23 Snohomish Washington 0.426211 -0.001020 -0.00049
3 2020-01-24 Cook Illinois -0.228380 -0.001020 -0.00049
4 2020-01-24 Snohomish Washington 0.426211 -0.001020 -0.00049
... ... ... ... ... ... ...
61966 2020-04-15 Sublette Wyoming 0.480243 -0.001020 -0.00049
61967 2020-04-15 Sweetwater Wyoming 0.480279 -0.000944 -0.00049
61968 2020-04-15 Teton Wyoming 0.480315 -0.000530 -0.00049
61969 2020-04-15 Uinta Wyoming 0.480352 -0.000995 -0.00049
61970 2020-04-15 Washakie Wyoming 0.480388 -0.000995 -0.00049
[61971 rows x 6 columns]
3. 标准化(Standardlization)
- 特征标准化使得数据中每个特征的值具有零均值和单位方差。
- 公式
>>>import os
>>>import pandas as pd
>>>import numpy as np
>>>from pprint import pprint
>>>def read_csv(file):
>>> # 获得文件中的数据
>>> return pd.read_csv(file)
>>>def to_float(data):
>>> # 将数据改为浮点数
>>> for column in data:
>>> if column =='date':continue # 跳过日期
>>> if str(data[column][1]).isdigit(): # 如果是数字
>>> data[column] = data[column].astype('float') # 将列转为浮点数
>>> return data
>>>def standardlization(data):
>>> # 标准化
>>> for column in data:
>>> if column == 'date': continue # 跳过日期
>>> if isinstance(data[column][1],float): # 如果是浮点数
>>> x = data[column]
>>> x = (x - np.mean(x))/(np.var(x))
>>> data[column] = x
>>> return data
>>>if __name__ == '__main__':
>>> file_path = os.path.join('D:\\','dataset','us-counties.csv')
>>> data = read_csv(file_path)
>>> pprint(standardlization(to_float(data)))
date county state fips cases deaths
0 2020-01-21 Snohomish Washington 0.000097 -0.000052 -0.000585
1 2020-01-22 Snohomish Washington 0.000097 -0.000052 -0.000585
2 2020-01-23 Snohomish Washington 0.000097 -0.000052 -0.000585
3 2020-01-24 Cook Illinois -0.000052 -0.000052 -0.000585
4 2020-01-24 Snohomish Washington 0.000097 -0.000052 -0.000585
... ... ... ... ... ... ...
61966 2020-04-15 Sublette Wyoming 0.000110 -0.000052 -0.000585
61967 2020-04-15 Sweetwater Wyoming 0.000110 -0.000048 -0.000585
61968 2020-04-15 Teton Wyoming 0.000110 -0.000027 -0.000585
61969 2020-04-15 Uinta Wyoming 0.000110 -0.000051 -0.000585
61970 2020-04-15 Washakie Wyoming 0.000110 -0.000051 -0.000585
[61971 rows x 6 columns]
4. 缩放至单位长度(Scaling to Unit Length)
- 该方法也在机器学习中常用。缩放特征向量的分量，将每个分量除以向量的欧几里得距离，使整个向量的长度为1。
- 公式：
>>>import os
>>>import pandas as pd
>>>import numpy as np
>>>from pprint import pprint
>>>def read_csv(file):
>>> # 获得文件中的数据
>>> return pd.read_csv(file)
>>>def to_float(data):
>>> # 将数据改为浮点数
>>> for column in data:
>>> if column =='date':continue # 跳过日期
>>> if str(data[column][1]).isdigit(): # 如果是数字
>>> data[column] = data[column].astype('float') # 将列转为浮点数
>>> return data
>>>def scaling_to_Unit_Length(data):
>>> # 缩放至单位长度
>>> for column in data:
>>> if column == 'date': continue # 跳过日期
>>> if isinstance(data[column][1],float): # 如果是浮点数
>>> x = data[column]
>>> x = x/np.linalg.norm(x)
>>> data[column] = x
>>> return data
>>>if __name__ == '__main__':
>>> file_path = os.path.join('D:\\','dataset','us-counties.csv')
>>> data = read_csv(file_path)
>>> pprint(standardlization(to_float(data)))
date county state fips cases deaths
0 2020-01-21 Snohomish Washington 0.000097 -0.000052 -0.000585
1 2020-01-22 Snohomish Washington 0.000097 -0.000052 -0.000585
2 2020-01-23 Snohomish Washington 0.000097 -0.000052 -0.000585
3 2020-01-24 Cook Illinois -0.000052 -0.000052 -0.000585
4 2020-01-24 Snohomish Washington 0.000097 -0.000052 -0.000585
... ... ... ... ... ... ...
61966 2020-04-15 Sublette Wyoming 0.000110 -0.000052 -0.000585
61967 2020-04-15 Sweetwater Wyoming 0.000110 -0.000048 -0.000585
61968 2020-04-15 Teton Wyoming 0.000110 -0.000027 -0.000585
61969 2020-04-15 Uinta Wyoming 0.000110 -0.000051 -0.000585
61970 2020-04-15 Washakie Wyoming 0.000110 -0.000051 -0.000585
[61971 rows x 6 columns]
参考资料
- numpy包的应用 作者:你们都厉害
- 理解numpy的rollaxis与swapaxes函数 作者：liaoyuecai
- Python机器学习及分析工具：Scipy篇 作者：殉道者之花火
- 奇客谷 作者: 吴吃辣
- 统计stats模块 作者: 火锅侠
