Pytorch学习记录-torchtext使用语言模型

Pytorch学习记录-torchtext使用语言模型
昨天写的那个太粗糙了。又找了一个教程来看。
主要包括三个方面

• 使用torchtext进行文本预处理
• 使用Keras和PyTorch构建数据集进行文本预处理（暂时放弃）
• 使用gensim加载预训练的词向量，并使用PyTorch实现语言模型

2.0 Glove词向量处理

搞了好久才发现，是glove拿来没有做转换

from gensim.scripts.glove2word2vec import glove2word2vec

glove_input_file = r'D:\DesktopBackup\right\MLHomework\AllenNLP\data\glove.6B.100d.txt'
word2vec_output_file = r'D:\DesktopBackup\right\MLHomework\AllenNLP\data\glove.6B.100d.word2vec.txt'
(count, dimensions) = glove2word2vec(glove_input_file, word2vec_output_file)
print(count, '\n', dimensions)

2.1 引入必要库

import torch
from torch import nn
import torch.nn.functional as F
import torch.optim as optim
import gensim

# 2-gram
CONTEXT_SIZE = 2

test_sentence = '''When forty winters shall besiege thy brow,
And dig deep trenches in thy beauty's field,
Thy youth's proud livery so gazed on now,
Will be a totter'd weed of small worth held:
Then being asked, where all thy beauty lies,
Where all the treasure of thy lusty days;
To say, within thine own deep sunken eyes,
Were an all-eating shame, and thriftless praise.
How much more praise deserv'd thy beauty's use,
If thou couldst answer 'This fair child of mine
Shall sum my count, and make my old excuse,'
Proving his beauty by succession thine!
This were to be new made when thou art old,
And see thy blood warm when thou feel'st it cold.'''.split()

2.2 加载预训练模型

# 赋予每个单词编码，用数字表示单词，这样才能传入word embedding得到词向量
vocab = set(test_sentence)
word_to_idx = {word: i + 1 for i, word in enumerate(vocab)}
# 定义一个unknown的词汇，如果没有出现在词表中，都作为unk
word_to_idx['<unk>'] = 0
idx_to_word = {i + 1: word for i, word in enumerate(vocab)}
idx_to_word[0] = '<unk>'

trigram = [((test_sentence[i], test_sentence[i + 1]), test_sentence[i + 2]) for i in range(len(test_sentence) - 2)]

wvmodel = gensim.models.KeyedVectors.load_word2vec_format(
    r'D:\DesktopBackup\right\MLHomework\AllenNLP\data\glove.6B.100d.word2vec.txt', binary=False)

vocab_size = len(word_to_idx)
embed_size = 100
weight = torch.zeros(vocab_size, embed_size)

for i in range(len(wvmodel.index2word)):
    try:
        index = word_to_idx[wvmodel.index2word[i]]
    except:
        continue
    weight[index, :] = torch.from_numpy(wvmodel.get_vector(idx_to_word[word_to_idx[wvmodel.index2word[i]]]))

2.3 定义和训练模型

# 定义模型
class NgramModel(nn.Module):
    def __init__(self, vocab_size, context_size, n_dim):
        super(NgramModel, self).__init__()
        self.n_word = vocab_size

        self.embedding = nn.Embedding.from_pretrained(weight)
        self.embedding.weight.requires_grad = True

        self.linear1 = nn.Linear(context_size * n_dim, 128)
        self.linear2 = nn.Linear(128, self.n_word)

    def forward(self, x):
        emb = self.embedding(x)
        emb = emb.view(1, -1)
        out = self.linear1(emb)
        out = F.relu(out)
        out = self.linear2(out)
        log_prob = F.log_softmax(out)
        return log_prob


ngrammodel = NgramModel(len(word_to_idx), CONTEXT_SIZE, 100)
criterion = nn.NLLLoss()
optimizer = optim.SGD(ngrammodel.parameters(), lr=1e-3)

# 训练模型
for epoch in range(100):
    print('epoch {}'.format(epoch + 1))
    print('*' * 10)
    running_loss = 0.0
    for data in trigram:
        word, label = data
        word = torch.LongTensor([word_to_idx[i] for i in word])
        label = torch.LongTensor([word_to_idx[label]])
        out = ngrammodel(word)
        loss = criterion(out, label)
        running_loss += loss.item()
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    print('loss:{:.6f}'.format(running_loss / len(word_to_idx)))

2.4 测试模型

# 测试模型的效果
word, label = trigram[3]
word = torch.LongTensor([word_to_idx[i] for i in word])
out = ngrammodel(word)
_, predict_label = torch.max(out, 1)
predict_word = idx_to_word[predict_label.item()]
print('real word is {}, predict word is {}'.format(label, predict_word))