机器学习技法笔记：Homework #5 特征变换&So
2017-12-03 本文已影响408人
cherryleechen
特征变换
问题描述
图1 2-3
图2 4-5
程序实现
# coding: utf-8
import numpy as np
from cvxopt import matrix, solvers
from sklearn import svm
def gen_data():
X = [[1, 0], [0, 1], [0, -1], [-1, 0], [0, 2], [0, -2], [-2, 0]]
X = np.array(X)
y = [-1, -1, -1, 1, 1, 1, 1]
y = np.array(y)
assert X.shape[0] == y.shape[0] and X.shape[1] == 2, "wrong data shape!"
return X, y
def explict_transform(X):
assert X.shape[1] == 2, "wrong shape of X!"
num = X.shape[0]
X1 = X[:, 0]
X2 = X[:, 1]
new_X1 = X2 ** 2 - 2 * X1 + 3
new_X2 = X1 ** 2 - 2 * X2 - 3
new_X = np.concatenate((new_X1.reshape((num, 1)), new_X2.reshape(num, 1)), axis=1)
return new_X
def svm_hard_linear(X, y):
num, dim = X.shape
P = matrix(np.concatenate((np.zeros((1, 1 + dim)),
np.concatenate((np.zeros((dim, 1)), np.eye(dim)), axis=1)), axis=0), tc='d')
q = matrix(np.zeros((1 + dim, 1)), tc='d')
G = matrix(-y * np.concatenate((np.ones((num, 1), dtype=np.float), X), axis=1), tc='d')
h = matrix(-np.ones((num, 1)), tc='d')
sol = solvers.qp(P, q, G, h)
return sol['x']
def implicit_transform(X):
assert X.shape[1] == 2, "wrong shape of X!"
num=X.shape[0]
X1 = X[:, 0]
X2 = X[:, 1]
new_X1=np.ones((num,1))
new_X2=2**(0.5)*X1
new_X3=2**(0.5)*X2
new_X4=X1**2
new_X5=X2**2
new_X6=2**(0.5)*X1*X2
new_X = np.concatenate((new_X1.reshape((num, 1)), new_X2.reshape(num, 1),new_X3.reshape(num, 1),
new_X4.reshape(num, 1),new_X5.reshape(num, 1),new_X6.reshape(num, 1)), axis=1)
return new_X
if __name__ == "__main__":
np.set_printoptions(precision=6,suppress=True)
X, y = gen_data()
# explicit
# 2
exp_X= explict_transform(X)
u = np.array(svm_hard_linear(exp_X, y.reshape(y.shape[0],1)))
b = u[0, :]
w = u[1:, :]
print("b:\n", b)
print("w:\n", w)
# implicit
clf=svm.SVC(C=1000000,kernel='poly',degree=2,gamma=1,coef0=1)
clf.fit(X,y)
# 3
alpha_y=clf.dual_coef_
alpha_y=alpha_y.reshape((alpha_y.shape[1],))
sv_ID=clf.support_
sv_y=[]
for i in range(sv_ID.shape[0]):
sv_y.append(y[sv_ID[i]])
alpha=[alpha_y[i]/sv_y[i] for i in range(sv_ID.shape[0])]
print("alpha*y:\n",alpha_y)
print("alpha:\n",alpha)
sv_X=clf.support_vectors_
print("support vectors:\n",sv_X)
# 4
b=clf.intercept_
print("b:\n",b)
w=np.dot(alpha_y,implicit_transform(sv_X)).reshape((6,1))
print("w:\n",w)
运行结果
图3 运行结果
Soft-Margin SVM
问题描述
图4 15
图5 16-17
图6 18-20
程序实现
# coding: utf-8
import numpy as np
from sklearn import svm
import matplotlib.pyplot as plt
def read_data(dataFile):
with open(dataFile,'r') as f:
lines=f.readlines()
data_list=[]
for line in lines:
line=line.strip().split()
data_list.append([float(l) for l in line])
dataArray=np.array(data_list)
num_data=dataArray.shape[0]
num_dim=dataArray.shape[1]-1
dataX=dataArray[:,1:].reshape((num_data,num_dim))
dataY=dataArray[:,0].reshape((num_data,))
return dataX,dataY
data_X,data_Y=read_data("features.train")
test_X,test_Y=read_data("features.test")
def convert_label(dataY,chosen_class):
num=dataY.shape[0]
new_Y=-np.ones_like(dataY)
for i in range(num):
if dataY[i]==chosen_class:
new_Y[i]=1
return new_Y
def zero_one_cost(pred,Y):
assert pred.shape==Y.shape,"wrong shape of pred and Y!"
return np.sum(np.not_equal(pred,Y))/Y.shape[0]
def question15():
c_list=[-6,-4,-2,0,2]
w_list=[]
new_Y=convert_label(data_Y,0)
for i in c_list:
clf=svm.LinearSVC(loss="hinge",C=10**i)
clf.fit(data_X,new_Y)
w_list.append(np.sqrt(np.sum(clf.coef_**2)))
plt.figure(figsize=(10,6))
plt.plot(c_list,w_list,'b')
plt.plot(c_list,w_list,'ro')
for (c,w) in zip(c_list,w_list):
plt.text(c+0.1,w,str(round(w,4)))
plt.xlabel("log10(C)")
plt.ylabel("||w||")
plt.xlim(-8,4)
plt.title("||w|| versus log10(C)")
plt.savefig("15.png")
def question16and17():
# 16
c_list = [-6, -4, -2, 0, 2]
Ein_list=[]
alpha_sum_list=[]
new_Y=convert_label(data_Y,8)
for i in c_list:
clf=svm.SVC(C=10**i,kernel='poly',degree=2,gamma=1,coef0=1)
clf.fit(data_X,new_Y)
pred=clf.predict(data_X)
Ein_list.append(zero_one_cost(pred,new_Y))
alpha_sum_list.append(np.sum(np.abs(clf.dual_coef_)))
# print(np.sum(clf.dual_coef_))
# print(clf.n_support_)
plt.figure(figsize=(10,6))
plt.plot(c_list,Ein_list,'b')
plt.plot(c_list,Ein_list,'ro')
for (c,e) in zip(c_list,Ein_list):
plt.text(c+0.1,e,str(round(e,4)))
plt.xlabel("log10(C)")
plt.ylabel("Ein")
plt.xlim(-8, 4)
plt.title("Ein versus log10(C)")
plt.savefig("16.png")
# 17
plt.figure(figsize=(10,6))
plt.plot(c_list,alpha_sum_list,'b')
plt.plot(c_list,alpha_sum_list,'ro')
for (c,a) in zip(c_list,alpha_sum_list):
plt.text(c+0.1,a,str(round(a,6)))
plt.xlabel("log10(C)")
plt.ylabel("sum of alpha")
plt.xlim(-8, 4)
plt.title("sum of alpha versus log10(C)")
plt.savefig("17.png")
def question18():
c_list=[-3,-2,-1,0,1]
dis_list=[]
new_Y=convert_label(data_Y,0)
for i in c_list:
clf=svm.SVC(C=10**i,kernel='rbf',gamma=100)
clf.fit(data_X,new_Y)
sv_ID=clf.support_
dis_list.append(new_Y[sv_ID[0]]*clf.decision_function(data_X)[sv_ID[0]])
plt.figure(figsize=(10,6))
plt.plot(c_list,dis_list,'b')
plt.plot(c_list,dis_list,'ro')
for (c,w) in zip(c_list,dis_list):
plt.text(c+0.1,w,str(round(w,4)))
plt.xlabel("log10(C)")
plt.ylabel("free sv's function distance to hyperplane")
plt.xlim(-5, 3)
plt.ylim(ymax=1.01)
plt.title("free sv's function distance to hyperplane versus log10(C)")
plt.savefig("18.png")
def question19():
new_Y=convert_label(data_Y,0)
new_test_Y=convert_label(test_Y,0)
gamma_list=[0,1,2,3,4]
Eout_list=[]
for i in gamma_list:
clf=svm.SVC(C=0.1,kernel='rbf',gamma=10**i)
clf.fit(data_X,new_Y)
pred=clf.predict(test_X)
Eout_list.append(zero_one_cost(pred,new_test_Y))
plt.figure(figsize=(10,6))
plt.plot(gamma_list,Eout_list,'b')
plt.plot(gamma_list,Eout_list,'ro')
for (c,w) in zip(gamma_list,Eout_list):
plt.text(c+0.1,w,str(round(w,4)))
plt.xlabel("log10(gamma)")
plt.ylabel("Eout")
plt.xlim(-1, 5)
plt.ylim(ymax=0.19)
plt.title("Eout versus log10(C)")
plt.savefig("19.png")
def question20():
new_Y=convert_label(data_Y,0)
gamma_list=[0,1,2,3,4]
chosen_gamma=[]
for t in range(100):
np.random.seed(t)
chosenID=np.random.randint(0,data_X.shape[0],1000)
train_X=[]
train_Y=[]
val_X=[]
val_Y=[]
for i in range(data_X.shape[0]):
if(i not in chosenID):
train_X.append(data_X[i,:])
train_Y.append(new_Y[i])
else:
val_X.append(data_X[i,:])
val_Y.append(new_Y[i])
train_X=np.array(train_X)
train_Y=np.array(train_Y)
val_X=np.array(val_X)
val_Y=np.array(val_Y)
Eval_list=[]
for g in gamma_list:
clf=svm.SVC(C=0.1,kernel='rbf',gamma=10**g)
clf.fit(train_X,train_Y)
pred=clf.predict(val_X)
Eval_list.append(zero_one_cost(pred,val_Y))
chosen_gamma.append(gamma_list[Eval_list.index(min(Eval_list))])
times=[]
for i in gamma_list:
times.append(chosen_gamma.count(i))
plt.figure(figsize=(10,6))
plt.bar(left=(gamma_list),height=(times),width=1,align="center",yerr=0.000001)
for (c,w) in zip(gamma_list,times):
plt.text(c,w*1.03,str(round(w,4)))
plt.xlabel("log10(gamma)")
plt.ylabel("the number of chosen times")
plt.xlim(-1, 5)
plt.ylim(0,80)
plt.title("the number of chosen times for gamma")
plt.savefig("20.png")
if __name__=="__main__":
question15()
question16and17()
question18()
question19()
question20()
运行结果
图7 15结果
图8 16结果
图9 17结果
图10 18结果
图11 19结果
图12 20结果
