当前位置 博文首页 > 广大菜鸟的博客：python 第三方库jieba（结巴）的学习(作用分词
import jieba
def test_jieba(s):
# 精确模式 返回列表类型
print("精确模式", jieba.lcut(s), jieba.cut(s))
# 全模式，返回存在冗余的列表
print("全模式", jieba.lcut(s, cut_all=True), jieba.cut(s, cut_all=True))
# 搜索引擎模式，返回存在冗余的列表
print("搜索引擎模式的列表", jieba.lcut_for_search(s), jieba.cut_for_search(s))
print()
s = '博主广大菜鸟是广大菜鸟中最菜的那一个'
test_jieba(s)
# 向分词词典增加新词w,只是在本次运行时增加词典
jieba.add_word('广大菜鸟')
test_jieba(s)
https://python123.io/resources/pye/hamlet.txt
# CalHamletV1.py
import urllib.request # 联网搜索
# 文本去噪及归一化，使用字典表达词频
def geTxt():
txt = urllib.request.urlopen("https://python123.io/resources/pye/hamlet.txt").read().decode()
txt = txt.lower()
for ch in '!"#$%&()*+,-./:;<=>?@[\\]^_‘{|}~':
txt = txt.replace(ch, " ")
return txt
if __name__ == '__main__':
hamletTxt = geTxt()
words = hamletTxt.split()
counts = {}
for word in words:
counts[word] = counts.get(word, 0) + 1 # 这里默认get不到就返回0
items_list = list(counts.items())
items_list.sort(key=lambda x: x[1], reverse=True) # x: x[1], 表示根据value排序，reverse参数为True意味着按照降序排序
for i in range(len(items_list)):
(word, count) = items_list[i]
print('{0:<10}{1:>5}'.format(word, count)) # {0:<10} 表示第0个参数，最小宽度为10，左对齐（bai<）
https://python123.io/resources/pye/threekingdoms.txt
# CalThreeKingdomsV1.py
import urllib.request # 联网搜索
import jieba
# 文本去噪及归一化，使用字典表达词频
def geTxt():
txt = urllib.request.urlopen("https://python123.io/resources/pye/threekingdoms.txt").read().decode()
words = jieba.lcut(txt)
return words
if __name__ == '__main__':
words = geTxt()
counts = {}
for word in words:
if len(word) == 1:
continue
else:
counts[word] = counts.get(word, 0) + 1 # 这里默认get不到就返回0
items_list = list(counts.items())
items_list.sort(key=lambda x: x[1], reverse=True) # x: x[1], 表示根据value排序，reverse参数为True意味着按照降序排序
for i in range(len(items_list)):
(word, count) = items_list[i]
print('{0:<10}{1:>5}'.format(word, count)) # {0:<10} 表示第0个参数，最小宽度为10，左对齐（bai<）
# CalThreeKingdomsV2.py
import jieba
txt = open("C:\\Users\\Lenovo\\Desktop\\threekingdoms.txt", "r", encoding="utf-8").read()
excludes = {"将军", "却说", "荆州", "二人", "不可", "不能", "如此"}
words = jieba.lcut(txt)
counts = {}
for word in words:
if len(word) == 1:
continue
elif word == "诸葛亮" or word == "孔明曰" or word == '诸葛孔明':
rword = "孔明"
elif word == "关公" or word == "云长" or word == '关云长':
rword = "关羽"
elif word == "玄德" or word == "玄德曰" or word == '刘玄德':
rword = "刘备"
elif word == "孟德" or word == "丞相" or word == '曹阿瞒':
rword = "曹操"
else:
rword = word
counts[rword] = counts.get(rword, 0) + 1
for word in excludes:
del counts[word]
items = list(counts.items())
items.sort(key=lambda x: x[1], reverse=True)
for i in range(10):
word, count = items[i]
print("{0:<10}{1:>5}".format(word, count))
