import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
from lxml import etree
import time
import pandas as pd

ua = UserAgent()
"""  #headers模板
headers = {'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Mobile Safari/537.36',
 'cookie': '将复制的cookie信息贴在此处'}
"""

cookie='XXXXX '
headers = {'User-Agent': ua.chrome,'cookie':cookie}
start=time.clock()

comment_word=[]
for ii in range(820):
    ss=ii*20
    url='https://movie.douban.com/subject/30306570/comments?start=%s'%ss+'&limit=20&sort=new_score&status=P'
    req= requests.get(url,headers=headers)
    soup=BeautifulSoup(req.text,'lxml')#生成一个BeautifulSoup 对象，用以后面的查找工作，lxml 字典
    xml_word=soup.find_all('span',class_='short')  #评论内容位置
    xml_zan=soup.find_all('span',class_='votes')  #点赞数位置
    xml_time=soup.find_all('span',class_='comment-time')  #点赞数位置
  
    time.sleep(1)
    for i in range(len(xml_word)):
        msg__word=xml_word[i].text
        msg_zan=int(xml_zan[i].text)
        msg_time=xml_time[i].text.split('\n')[1].split(' ')[-1]
        comment_word.append([msg_time, msg_zan,msg__word])
                
comentv=pd.DataFrame(comment_word,columns=['评论时间','点赞数量','评论内容'])
comentv.to_csv('C:/Users/lenovo/Desktop/muvieos.csv',encoding='utf_8_sig', index=False)         

end=time.clock()
print('time is:',end-start)

我的程序中抓取了评论时间、评论内容、觉得有用的数量，同时记录一下程序运行时耗费的时间。经过漫长的等待，结果发现一次只能爬500条数据，然后IP就被限制了，需要输入验证码，网上有破解的办法，具体也不想去研究了，就先拿500条数据来玩吧。

电影是在2020/1/25上映，看一看评论在电影上映前后的热度，显然上映前讨论得比较热闹，点击有用的个数还是比较高的，随着时间的推移，受关注度就越来越低。

import matplotlib.pyplot as plt

plt.figure(figsize=[40,12])
plt.plot(DATAS.评论时间,DATAS.点赞数量,'*')

# 分词训练
from snownlp import seg
seg.train('data.txt')
seg.save('seg.marshal')
# 词性标注训练
# from snownlp import tag
# tag.train('199801.txt')
# tag.save('tag.marshal')
# 情感分析训练
# from snownlp import sentiment
# sentiment.train('neg.txt', 'pos.txt')
# sentiment.save('sentiment.marshal')

这样训练好的文件就存储为seg.marshal了，之后修改snownlp/seg/init.py里的data_path指向刚训练好的文件即可

from snownlp import sentiment
sentiment.train('C:/Users/lenovo/Desktop/orial/neg.txt','C:/Users/lenovo/Desktop/orial/pos.txt')
sentiment.save('sentiment.marshal')

from snownlp import SnowNLP
s='我讨厌这部电影，甚至作呕，强行灌鸡汤，强行要把所谓的黑暗面撕出来，然后结尾强行煽情和解。没有人没和解，只是徐峥自己没和解吧！'
ss=SnowNLP(s)
ss.sentiments

pip  uninstall snownlp
pip install snownlp

from snownlp import SnowNLP
taidu=[]
taidutime=[]
for i in range(300,len(DATAS)):
    ss=SnowNLP(DATAS.评论内容[i])
    taidu.append(ss.sentiments)
    taidutime.append(DATAS.评论时间[i])
    
plt.figure(figsize=[40,12])
plt.plot(taidutime,taidu,'*')

import re 
import jieba    #分词包
import pandas as pd  
import numpy    #numpy计算包

#提取评论内容
comments = '  '
for k in range(len(DATAS)):
    comments = comments + (str(DATAS.评论内容[k])).strip()
    
#进行正则化处理，去除标点符号等
pattern = re.compile(r'[\u4e00-\u9fa5]+')
filterdata = re.findall(pattern,comments)
cleaned_comments = ''.join(filterdata)

#分词
segment = jieba.lcut(cleaned_comments)
words_df=pd.DataFrame({'segment':segment})

#去除停用词
stopwords=pd.read_csv("C:/Users/lenovo/Desktop/chineseStopWords.txt",index_col=False,quoting=3,sep="\t",names=['stopword'], encoding='gb18030')#quoting=3全不引用
words_df=words_df[~words_df.segment.isin(stopwords.stopword)]

#统计词出现的次数
words_stat=words_df.groupby(by=['segment'])['segment'].agg({"计数":numpy.size})
words_stat=words_stat.reset_index().sort_values(by=["计数"],ascending=False)
words_stat