根据李沐老师的课做的记录。

构建文本数据集

文本数据集可以将其看作一串单词序列或者字符序列。

构建时一般有以下几个步骤。

1. 文本清洗（比如去除乱码和标点符号、当然在很多时候并不会去掉标点符号）。
2. 将文本存入内存。
3. 将文本拆分成词或者字符。
4. 建立词汇表和对应索引。

文本清洗和读取

这里使用课上提到的《time machine》


百度网盘链接


提取码：pypt

1. 读取文本
   
   只保留字母并统一成小写。

text_path = './timemachine.txt'


def read_time_machine():
    with open(text_path, 'r') as f:
        lines = f.readlines()
    return [re.sub(r'[^a-zA-Z]+', ' ', line).strip().lower() for line in lines]

lines = read_time_machine()
print('len=', len(lines), '\n', lines[0], '\n', lines[9])

输出如下

len= 3221 
the time machine by h g wells 
was expounding a recondite matter to us his grey eyes shone and

tokenize

将文本划分为词或者字符。

def tokenize(lines, token='word'):
    if token == 'word':
        return [line.split() for line in lines]
    elif token == 'char':
        return [list(line) for line in lines]
    else:
        print('wrong')

tokens = tokenize(lines)
print(tokens[0])

输出为

在这里插入代码片['the', 'time', 'machine', 'by', 'h', 'g', 'wells']

定义计数函数

统计每个词的出现频数。

def count_corpus(tokens):
    if len(tokens) == 0:
        tokens = []
    elif isinstance(tokens[0], list):
        tokens = [token for line in tokens for token in line]
    return collections.Counter(tokens)

定义vocab类

用于返回id_to_token和token_to_id。

class Vocab:
    def __init__(self, tokens=None, min_freq=0, reversed_token=None):
        if not tokens:
            tokens = []
        if not reversed_token:
            reversed_token = []
        counter = count_corpus(tokens)
        self.token_freq = sorted(counter.items(), key=lambda x: x[1], reverse=True)
        self.unk, uniq_tokens = 0, ['UNK'] + reversed_token
        uniq_tokens += [token for token, freq in self.token_freq if freq > min_freq and token not in uniq_tokens]
        self.idx_to_token, self.token_to_idx = [], {}
        for token in uniq_tokens:
            self.idx_to_token.append(token)
            self.token_to_idx[token] = len(self.idx_to_token) - 1

    def __len__(self):
        return len(self.idx_to_token)

    def __getitem__(self, tokens):
        if not isinstance(tokens, (list, tuple)):  # 判断是否为一个词语
            return self.token_to_idx.get(tokens, self.unk)
        return [self.__getitem__(token) for token in tokens]  # 如果为句子则以list形式返回

    def to_tokens(self, indices):  # 将id转为句子（主要用于多个id）
        if not isinstance(indices, (list, tuple)):  # 判断是否为一个id
            return self.idx_to_token[indices]
        return [self.idx_to_token[index] for index in indices]  # 如果为句子则以list形式返回

tokens = tokenize(lines)
vocab = Vocab(tokens, min_freq=2)
print(vocab['time'], vocab.to_tokens([1, 2, 3, 4, 5, 6]), len(vocab))

输出为

19 ['the', 'i', 'and', 'of', 'a', 'to'] 1420

生成样本

随机抽样生成样本

import random
import torch
# 随机抽样生成批量样本
def sequence_radom_iter(corpus, batch_size, num_step):
    #  随机起始
    corpus = corpus[random.randint(0, num_step - 1):]
    #  样例总数 -1 是因为要预测，对于最后一个x要保证有y
    sequence_num = (len(corpus)-1)//num_step
    init_index = list(range(0, sequence_num*num_step, num_step))
    random.shuffle(init_index)

    def data(pos):
        return corpus[pos: pos+num_step]
    num_batches = sequence_num//batch_size
    for i in range(0, batch_size*num_batches, batch_size):
        init_pre_batch_idex = init_index[i:i+batch_size]
        X = [data(i) for i in init_pre_batch_idex]
        Y = [data(i+1) for i in init_pre_batch_idex]
        yield torch.tensor(X), torch.tensor(Y)

data = sequence_radom_iter(vocab.__getitem__(corpus), 3, 5)
x, y = next(data)
print(x, '\n', y)

输出：

tensor([[ 119,    0,    0,    4,    0],
        [  11, 1170,    3,    0,  368],
        [   0,  348,  563,   51,  898]]) 
 tensor([[   0,    0,    4,    0,    1],
        [1170,    3,    0,  368,    0],
        [ 348,  563,   51,  898,   95]])

按顺序生成样本

def sequence_data_seq(corpus, batch_size, num_step):
    #  随机起始
    corpus = corpus[random.randint(0, num_step - 1):]
    #  样例总数 -1 是因为要预测，对于最后一个x要保证有y
    sequence_num = (len(corpus) - 1) // batch_size * batch_size
    Xs = torch.tensor(corpus[:sequence_num])
    Ys = torch.tensor(corpus[1:sequence_num+1])
    Xs, Ys = Xs.reshape(batch_size, -1), Ys.reshape(batch_size, -1)
    for i in range(0, Xs.shape[1]//num_step * num_step, num_step):
        X, Y = Xs[:, i: i + num_step], Ys[:, i: i + num_step]
        yield X, Y

data = sequence_data_seq(vocab.__getitem__(corpus), 3, 5)
x, y = next(data)
print(x, '\n', y)
testData = list(range(100))
data = sequence_data_seq(testData, 3, 5)
x, y = next(data)
print(x, '\n', y)

输出：

tensor([[ 19,  50,  40,   0,   0],
        [  4,   1,  56,   0, 799],
        [ 11, 180,  63,   6,  13]]) 
 tensor([[ 50,  40,   0,   0, 400],
        [  1,  56,   0, 799,   4],
        [180,  63,   6,  13,   9]])

tensor([[ 3,  4,  5,  6,  7],
        [35, 36, 37, 38, 39],
        [67, 68, 69, 70, 71]]) 
 tensor([[ 4,  5,  6,  7,  8],
        [36, 37, 38, 39, 40],
        [68, 69, 70, 71, 72]])

封装

class SeqDataLoader:
    def __init__(self, tokens, batch_size, num_steps, use_random_iter):
        if use_random_iter:
            self.data_iter_fn = sequence_radom_iter
        else:
            self.data_iter_fn = sequence_seq_iter
        self.batch_size = batch_size
        self.num_steps = num_steps
        self.vocab = Vocab(tokens, min_freq=2)
        self.corpus = self.vocab.__getitem__([token for line in tokens for token in line])

    def __iter__(self):
        return self.data_iter_fn(self.corpus, self.batch_size, self.num_steps)

def load_text(tokens, batch_size, num_steps, use_random_iter=False):
    iter = SeqDataLoader(tokens, batch_size, num_steps, use_random_iter)
    return iter, iter.vocab

lines = read_time_machine()
tokens = tokenize(lines, token='char')

data_iter, vocab = load_text(tokens, 4, 20)
print(next(data_iter.__iter__()))

(tensor([[ 2,  1, 13,  4, 15,  9,  5,  6,  2,  1, 21, 19,  1,  9,  1, 18,  1, 17,
          2, 12],
        [17,  4, 19,  1, 17,  4,  8,  1, 10,  5, 15,  9, 12, 19,  1, 15,  4, 10,
         22,  2],
        [ 1,  3,  9,  5,  8,  1, 21, 12,  2,  4, 15,  9,  2, 11,  7, 21,  8, 15,
          2,  6],
        [ 2,  1,  4,  1, 18, 10,  2,  4,  3,  1,  2, 16, 16,  7, 10,  3,  1,  7,
         16,  1]]), tensor([[ 1, 13,  4, 15,  9,  5,  6,  2,  1, 21, 19,  1,  9,  1, 18,  1, 17,  2,
         12, 12],
        [ 4, 19,  1, 17,  4,  8,  1, 10,  5, 15,  9, 12, 19,  1, 15,  4, 10, 22,
          2, 11],
        [ 3,  9,  5,  8,  1, 21, 12,  2,  4, 15,  9,  2, 11,  7, 21,  8, 15,  2,
          6,  2],
        [ 1,  4,  1, 18, 10,  2,  4,  3,  1,  2, 16, 16,  7, 10,  3,  1,  7, 16,
          1, 13]]))