博客

训练

# -*- coding: utf-8 -*-
# @Project: company
# @Author: little fly
# @File name: 文本分类
# @Create time: 2020/10/31 9:48

# 导入必要的包
import jieba
import numpy as np
import pandas as pd
from gensim.models.word2vec import Word2Vec
import joblib
from sklearn.svm import SVC

# 读取两个类别的语料
pos = pd.read_csv('weather_pos.txt', encoding='UTF-8', header=None)
neg = pd.read_csv('weather_neg.txt', encoding='UTF-8', header=None)

# 进行分词处理
pos['words'] = pos[0].apply(lambda x: jieba.lcut(x))
neg['words'] = neg[0].apply(lambda x: jieba.lcut(x))

# 将正负语料进行合并成训练语料然后并打上标签，正语料打上标签1，负语料打上标签0
x = np.concatenate((pos['words'], neg['words']))
y = np.concatenate((np.ones(len(pos)), np.zeros(len(neg))))

# 训练词向量128维
word2vec = Word2Vec(x, size=128, window=3, min_count=5, sg=1, hs=1, iter=10, workers=25)
word2vec.save('word2vec.model')

# 封装词转换词向量的方法，list型
def total_vector(words):
    vec = np.zeros(128).reshape((1, 128))
    for word in words:
        try:
            vec += word2vec.wv[word].reshape((1, 128))
        except KeyError:
            continue
    return vec

# 对x所有词转换成词向量，即合成训练集
train_vec = np.concatenate([total_vector(words) for words in x])

# 开始训练分类模型
model = SVC(kernel='rbf', verbose=True)
model.fit(train_vec, y)

# 保存模型为pkl文件
joblib.dump(model, 'weather_svm.pkl')

测试

# -*- coding: utf-8 -*-
# @Project: company
# @Author: little fly
# @File name: test
# @Create time: 2020/10/31 9:51
import jieba
import numpy as np
from gensim.models.word2vec import Word2Vec
import joblib

# 加载词向量模型
word2vec = Word2Vec.load('word2vec.model')
# 加载svm分类模型
model = joblib.load('weather_svm.pkl')

# 封装词转换词向量的方法，list型
def total_vector(words):
    vec = np.zeros(128).reshape((1, 128))
    for word in words:
        try:
            vec += word2vec.wv[word].reshape((1, 128))
        except KeyError:
            continue
    return vec

# 封装预测方法
def svm_predict(query):
    words = jieba.lcut(str(query))
    words_vec = total_vector(words)
    result = model.predict(words_vec)
    if int(result) == 1:
        print('类别：天气')
    elif int(result) == 0:
        print('类别：其他')

# 调用预测
while 1:
    str_ = input('请输入：')
    svm_predict(str_)

结果