jieba分词练习

2018-05-27 本文已影响0人 Yangjialin

实验内容：

使用Python分词模块：Jieba，对四川大学公共管理学院2015-2018年308条新闻内容进行分词分析。

实验环境：

操作系统：CentOS Linux release 7.2.1511 (Core)
实验语言：Python 3.6.5
所用框架：jieba wordcloud matplotlib

实验过程：

（1）部分内容示例：每条新闻记录共由三部分内容组成：（标题）（时间）（正文）。新闻内容来自之前的新闻爬虫实验。

内容示例.png

（2）选取所有新闻标题，使用jieba精准模式进行分词，并进行词频统计

分词与词频统计功能代码：
参考demo：https://blog.csdn.net/fengjianc/article/details/78929121

# showCutAndCount.py
# -*- coding: utf-8 -*-


import jieba
import codecs
from scipy.misc import imread
import os
from os import path
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from PIL import Image, ImageDraw, ImageFont
from wordcloud import WordCloud, ImageColorGenerator
import drawBar

newsTitle = open('newstitle.txt','r').read()
d = path.dirname(__file__)


def segment(news):
    '''
    用jieba分词对输入文档进行分词，并保存至本地（根据情况可跳过）
    '''
    jieba.load_userdict("userdict.txt")
    seg_list = " ".join(jieba.cut(news, cut_all=False)) #使用jieba分词，采用精准模式

    document_after_segment = open('newsTitleAfterCut.txt', 'w+')
    document_after_segment.write(seg_list)
    document_after_segment.close()

    return seg_list


def wordCount(segment_list):
    '''
        该函数实现词频的统计，并将统计结果存储至本地。
        在制作词云的过程中用不到，主要是在画词频统计图时用到。
    '''
    word_lst = []
    word_dict = {}
    with open('wordCount.txt','w') as wf2:
        word_lst.append(segment_list.split(' '))
        for item in word_lst:
            for item2 in item:
                if item2 not in word_dict:
                    word_dict[item2] = 1
                else:
                    word_dict[item2] += 1

        word_dict_sorted = dict(sorted(word_dict.items(), \
        key = lambda item:item[1], reverse=True))#按照词频从大到小排序
        for key in word_dict_sorted:
            wf2.write(key+' '+str(word_dict_sorted[key])+'\n')
    wf2.close()

def drawWordCloud(seg_list):
    '''
        制作词云
        设置词云参数
    '''
    #color_mask = imread("background.jpg") # 读取背景图片,注意路径
    #image_colors = np.array(Image.open(path.join(d, "back.png")))
    wc = WordCloud(
        #设置字体，不指定就会出现乱码，注意字体路径
        font_path="HYQiHei-25J.ttf",
        #font_path=path.join(d,'simsun.ttc'),
        #设置背景色
        background_color='white',
        #词云形状
        #mask=image_colors,
        #允许最大词汇
        max_words=1000,
        #最大号字体
        max_font_size=60,
        width=550,
        height=340
    )
    wc.generate(seg_list) # 产生词云
    #image_colors = ImageColorGenerator(color_mask)
    #wc.recolor(color_func=image_colors)
    wc.to_file("newsTitle.jpg") #保存图片
    plt.imshow(wc, interpolation='bilinear')
    plt.axis("off")
    plt.figure()
    #plt.imshow(image_colors, cmap=plt.cm.gray, interpolation='bilinear')
    plt.axis("off")
    plt.show()

def removeStopWords(seg_list):
    '''
    自行下载stopwords1893.txt停用词表，该函数实现去停用词
    '''
    wordlist_stopwords_removed = []

    stop_words = open('/home/ir08yjl/project/jieba/stopwords/stopwords1893.txt')
    stop_words_text = stop_words.read()

    stop_words.close()

    stop_words_text_list = stop_words_text.split('\n')
    after_seg_text_list = seg_list.split(' ')

    for word in after_seg_text_list:
        if word not in stop_words_text_list:
            wordlist_stopwords_removed.append(word)

    without_stopwords = open('newsTiltleCutWithoutStopWords.txt', 'w')
    without_stopwords.write(' '.join(wordlist_stopwords_removed))
    return ' '.join(wordlist_stopwords_removed)


if __name__ == "__main__":
    segment_list = segment(newsTitle)
    segment_list_remove_stopwords = removeStopWords(segment_list)
    drawWordCloud(segment_list_remove_stopwords)
    wordCount(segment_list_remove_stopwords)
    drawBar.drawStatBarh()#绘制词频统计图

分词结果：（此处使用空格作为词间隔）