优化器

import torch
import torch.utils.data as Data
import torch.nn.functional as F
import matplotlib.pyplot as plt
import numpy as np

torch.manual_seed(1)

LR = 0.01
BATCH_SIZE = 32
EPOCH = 12

x = torch.unsqueeze(torch.linspace(-1,1,1000),dim = 1)
y = x.pow(2)+ 0.1 *torch.normal(torch.zeros(x.size()))

#plt.scatter(x.numpy(),y.numpy())
#plt.show()

#先转换成torch能识别的dataset
torch_dataset = Data.TensorDataset(x,y)
#把dataset放入DataLoader
loader = Data.DataLoader(
    dataset=torch_dataset,      # torch TensorDataset format
    batch_size=BATCH_SIZE,      # mini batch size
    shuffle=True,               # 要不要打乱数据 (打乱比较好)
    num_workers=2,              # 多线程来读数据
)
class Net(torch.nn.Module):
    def __init__(self):
        super(Net,self).__init__()
        self.hidden = torch.nn.Linear(1,20)
        self.predict = torch.nn.Linear(20,1)
    def forward(self, x):
        x = F.relu(self.hidden(x))
        x = self.predict(x)
        return x
if __name__ == '__main__':
    net_SGD = Net()
    net_Momentum = Net()
    net_RMSprop = Net()
    net_Adam = Net()
    nets = [net_SGD,net_Momentum,net_RMSprop,net_Adam]

    #创建不同的优化器
    opt_SGD = torch.optim.SGD(net_SGD.parameters(),lr = LR)
    opt_Momentum = torch.optim.SGD(net_Momentum.parameters(), lr=LR,momentum = 0.8)
    opt_RNSprop = torch.optim.RMSprop(net_RMSprop.parameters(), lr=LR,alpha=0.9)
    opt_Adam = torch.optim.Adam(net_Adam.parameters(), lr=LR,betas=(0.9,0.99))
    optimizers = [opt_SGD,opt_Momentum,opt_RNSprop,opt_Adam]

    #损失函数
    loss_fun = torch.nn.MSELoss()
    losses_his = [[],[],[],[]]#记录各个网络的loss
    for epoch in range(EPOCH):
        print("epoch:",epoch)
        for step,(b_x,b_y) in enumerate(torch_dataset):
            for net,opt,l_his in zip(nets,optimizers,losses_his):
                output = net(b_x)
                loss = loss_fun(output,b_y)
                opt.zero_grad()
                loss.backward()
                opt.step()
                l_his.append((loss.data.numpy()))

    labels = ['SGD', 'Momentum', 'RMSprop', 'Adam']
    max_steps = 400
    for i, l_his in enumerate(losses_his):
        plt.plot(l_his[0:400], label=labels[i])
    plt.legend(loc='best')
    plt.xlabel('Steps')
    plt.ylabel('Loss')
    plt.ylim((0, 0.2))
    plt.show()
    pass