博客

# 爬取李白诗词保存到  libai.txt
import re
import requests


def crawl(start_url):
    base_url = 'http://so.gushiwen.org'

    req_headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
    }

    for i in range(1, 126):

        restart_url = start_url + str(i) + '.aspx'
        print(restart_url)

        res = requests.get(restart_url, headers=req_headers)
        if res.status_code == requests.codes.ok:
            html = res.text

            # 获取所有诗的链接
            parttern_href = re.compile(r'<div class="cont">.*?<p><a .*? href="(.*?)" .*?>.*?</p>', flags=re.DOTALL)
            hrefs = re.findall(parttern_href, html)

            # 获取每一首诗的内容,并保存到本地
            with open('libai.txt', mode='a', encoding='utf-8') as f:
                for href in hrefs:
                    href = base_url + href
                    res = requests.get(href, headers=req_headers)
                    if res.status_code == requests.codes.ok:
                        html = res.text
                        # 标题
                        parttern_title = re.compile(r'<div class="cont">.*?<h1 .*?>(.*?)</h1>', re.DOTALL)
                        title = re.search(parttern_title, html).group(1)
                        # 内容
                        parttern_content = re.compile(r'<div class="cont">.*?<div class="contson" id=".*?">(.*?)</div>',
                                                      re.DOTALL)
                        content = re.search(parttern_content, html).group(1)
                        content = re.sub(r'<br />', '\n', content)
                        content = re.sub(r'<p>', '', content)
                        content = re.sub(r'</p>', '', content)

                        print('正在获取 {title}'.format(title=title))
                        f.write('{title}{content}\n'.format(title=title, content=content))


if __name__ == '__main__':
    start_url = 'https://so.gushiwen.org/authors/authorvsw_b90660e3e492A'
    crawl(start_url)

# 数据处理
with open('libai.txt', 'r', encoding='utf8') as file:
    content = file.read().replace('\n', '')
# print(content.replace('\n', ''))
chars = list(set(content))
print(chars)
n_chars = len(chars)
print(n_chars)
# 字符与对应数字标记
char_indices = dict((c, i) for i, c in enumerate(chars))
print(char_indices)
# 数字标记与对应的字符
indices_char = dict((i, c) for i, c in enumerate(chars))
print(indices_char)

maxlen = 20
step = 3
sentences = []
next_chars = []
for i in range(0, len(content) - maxlen, step):
    sentences.append(content[i: i+maxlen])
    next_chars.append(content[i+maxlen])

print(sentences)
print(next_chars)

# 将所有句子中的字符转换为独热编码的形式
import numpy as np
X_train = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)
y_train = np.zeros((len(sentences), len(chars)), dtype=np.bool)

for i, sentence in enumerate(sentences):
    for j, char in enumerate(sentence):
        X_train[i, j, char_indices[char]] = 1
    y_train[i, char_indices[next_chars[i]]] = 1

print(X_train)
print(y_train)

# 诗词生成
from keras.models import Sequential
from keras.layers import LSTM, Dense
from keras.optimizers import RMSprop

model = Sequential()
model.add(LSTM(units=128,
               input_shape=(maxlen, n_chars)))
model.add(Dense(units=n_chars, activation='softmax'))

model.compile(loss='categorical_crossentropy',
              optimizer=RMSprop(lr=0.01),
              metrics=None)

model.fit(X_train, y_train, batch_size=128, epochs=20)


def sample(preds, temperature=1.0):
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(n=1, pvals=preds, size=1)
    return np.argmax(probas)

import random
def generate_text(length, diversity):
    start_index = random.randint(0, len(content) - maxlen - 1)
    sentence = content[start_index: start_index+maxlen]
    generated = ''
    for i in range(length):
        x_pred = np.zeros((1, maxlen, len(chars)))
        for i, char in enumerate(sentence):
            x_pred[0, i, char_indices[char]] = 1
        preds = model.predict(x_pred, verbose=0)[0]
        next_index = sample(preds, diversity)
        next_char = indices_char[next_index]
        generated += next_char
        sentence = sentence[1:] + next_char
    return generated


if __name__ == '__main__':
    print(generate_text(24, 0.2))