├── .idea
    ├── Micro-blogReview.iml
    ├── inspectionProfiles
    │   └── Project_Default.xml
    ├── misc.xml
    ├── modules.xml
    ├── other.xml
    └── vcs.xml
├── DataAnalysis.py
├── README.md
├── WeiboApi
    ├── readme.md
    ├── sinaweibopy3.py
    └── weiboAPI.py
├── modelTraining.py
├── outPut
    ├── final.html
    ├── sentiment.marshal.3
    └── things.txt
├── requirements.txt
├── start.py
└── visualization.py


/.idea/Micro-blogReview.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module type="PYTHON_MODULE" version="4">
 3 |   <component name="NewModuleRootManager">
 4 |     <content url="file://$MODULE_DIR$" />
 5 |     <orderEntry type="inheritedJdk" />
 6 |     <orderEntry type="sourceFolder" forTests="false" />
 7 |   </component>
 8 |   <component name="PyDocumentationSettings">
 9 |     <option name="renderExternalDocumentation" value="true" />
10 |   </component>
11 |   <component name="TestRunnerService">
12 |     <option name="PROJECT_TEST_RUNNER" value="Unittests" />
13 |   </component>
14 | </module>


--------------------------------------------------------------------------------
/.idea/inspectionProfiles/Project_Default.xml:
--------------------------------------------------------------------------------
 1 | <component name="InspectionProjectProfileManager">
 2 |   <profile version="1.0">
 3 |     <option name="myName" value="Project Default" />
 4 |     <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
 5 |       <option name="ignoredPackages">
 6 |         <value>
 7 |           <list size="1">
 8 |             <item index="0" class="java.lang.String" itemvalue="json" />
 9 |           </list>
10 |         </value>
11 |       </option>
12 |     </inspection_tool>
13 |   </profile>
14 | </component>


--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="JavaScriptSettings">
4 |     <option name="languageLevel" value="ES6" />
5 |   </component>
6 |   <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7" project-jdk-type="Python SDK" />
7 | </project>


--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectModuleManager">
4 |     <modules>
5 |       <module fileurl="file://$PROJECT_DIR$/.idea/Micro-blogReview.iml" filepath="$PROJECT_DIR$/.idea/Micro-blogReview.iml" />
6 |     </modules>
7 |   </component>
8 | </project>


--------------------------------------------------------------------------------
/.idea/other.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="PySciProjectComponent">
4 |     <option name="PY_SCI_VIEW" value="true" />
5 |     <option name="PY_SCI_VIEW_SUGGESTED" value="true" />
6 |   </component>
7 | </project>


--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="VcsDirectoryMappings">
4 |     <mapping directory="$PROJECT_DIR$" vcs="Git" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/DataAnalysis.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python3.7
 2 | # -*- coding: utf-8 -*-
 3 | # @Time    : 2019/11/30 下午 10:25
 4 | # @Email   : pasalai@qq.com
 5 | # @Github  : github.com/laishouchao
 6 | # @File    : DataAnalysis.py
 7 | # @Software: PyCharm
 8 | 
 9 | from snownlp import SnowNLP
10 | import visualization
11 | 
12 | 
13 | class dataAnalysis:
14 |     def __init__(self):
15 |         # 初始化数组
16 |         self.temp0 = []
17 |         self.temp1 = []
18 |         self.temp2 = []
19 |         self.temp3 = []
20 |         self.temp4 = []
21 |         self.temp5 = []
22 |         self.temp6 = []
23 |         self.temp7 = []
24 |         self.temp8 = []
25 |         self.temp9 = []
26 | 
27 |     def DataAnalysis(self, keywords):
28 |         # 情感分析，通过更换微博预料训练生成的.model.3模型可提高分析的精确度
29 |         Affective_value = SnowNLP(keywords)
30 |         # 计算消息的正负面情绪值，并写入Getvalue.xlsx中
31 |         Affective_value_good = Affective_value.sentiments
32 |         Affective_value_bad = 1 - Affective_value_good
33 |         List = [keywords, Affective_value_good, Affective_value_bad]
34 |         print(List[0], List[1], List[2])  # 输出事件及对应的情感值
35 | 
36 |         # 对情感归类统计
37 |         if Affective_value.sentiments < 0.1:
38 |             self.temp0.append(keywords)
39 |         elif Affective_value_good < 0.2:
40 |             self.temp1.append(keywords)
41 |         elif Affective_value_good < 0.3:
42 |             self.temp2.append(keywords)
43 |         elif Affective_value_good < 0.4:
44 |             self.temp3.append(keywords)
45 |         elif Affective_value_good < 0.5:
46 |             self.temp4.append(keywords)
47 |         elif Affective_value_good < 0.6:
48 |             self.temp5.append(keywords)
49 |         elif Affective_value_good < 0.7:
50 |             self.temp6.append(keywords)
51 |         elif Affective_value_good < 0.8:
52 |             self.temp7.append(keywords)
53 |         elif Affective_value_good < 0.9:
54 |             self.temp8.append(keywords)
55 |         elif Affective_value_good >= 0.9:
56 |             self.temp9.append(keywords)
57 |         else:
58 |             return "err"
59 |         return List
60 | 
61 |     def Statistics(self):
62 |         # elif endmark == "1":
63 |         #     # final = []
64 |         #     # final.append(temp0)
65 |         #     # final.append(temp1)
66 |         #     # final.append(temp2)
67 |         #     # final.append(temp3)
68 |         #     # final.append(temp4)
69 |         #     # final.append(temp5)
70 |         #     # final.append(temp6)
71 |         #     # final.append(temp7)
72 |         #     # final.append(temp8)
73 |         #     # final.append(temp9)
74 |         #     # # print(final)
75 |         #     # list_len = [len(final[0]), len(final[1]), len(final[2]), len(final[3]), len(final[4]), len(final[5]),
76 |         #     #             len(final[6]),
77 |         #     #             len(final[7]), len(final[8]), len(final[9])]
78 |         list_len = [len(self.temp0), len(self.temp1), len(self.temp2), len(self.temp3), len(self.temp4),
79 |                     len(self.temp5),
80 |                     len(self.temp6), len(self.temp7), len(self.temp8), len(self.temp9)]
81 |         # print(list_len)
82 |         return list_len
83 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # WeiboReview  微博评论获取并情感分析    
 2 | ![](https://img.shields.io/badge/python-3.7-green) ![](https://img.shields.io/badge/NLP-SnowNLP-green) ![](https://img.shields.io/badge/weiboAPI-sinaweibopy3-green) ![](https://img.shields.io/badge/Visualization-PyEcharts-green)         
 3 | 爬取指定的微博的评论，并进行情感分析    
 4 | ## 运行说明    
 5 | * 首先在[微博开发者平台](https://open.weibo.com)注册登录，并申请，[申请流程点这里](https://www.douban.com/note/449162780/)。
 6 | * 获取到开发者接口的App Key及App Secret(无需开发者认证及高级接口申请)
 7 | * 在应用接口管理页面的<code>应用信息>高级信息>OAuth2.0 授权设置</code>中的<code>授权回调页</code>填写<code>http://api.weibo.com/oauth2/default.html</code>
 8 | * 获取<code>accessoken</code>:    
 9 | 将得到的App Key及App Secret值填入本程序<code>./weiboApi/weiboApi.py</code>中对应的值，运行后将自动打开浏览器的<code>微博OAuth2.0</code>页面，在url中，包含一个<code>code=*****</code>，复制code值，回到本程序，将code的值填入程序运行后的输入栏，运行后输出<code>accesstoken</code>
10 | * 根据开发者文档的评论接口页面的接口说明，使用获得的accesstoken及相关微博文章的ID构造API链接，访问，并复制cookie,将<code>./start.py</code>中链接及cookie替换。
11 | * 执行<code>pip3 install -r requirements.txt</code>安装引用到的库，或直接安装[Anaconda](https://www.anaconda.com/)
12 | * 运行<code>./start.py</code>，数据会自动格式化并保存至<code>./outPut/评论.csv</code>
13 | ## 更新日志    
14 | ### 2019年12月5日    
15 | * 创建项目
16 | * 分析微博手机端H5页面，得到地址爬取评论
17 | * 太难了，地址得手动获取，每页50条，且反扒机制TQL！    
18 | ### 2019年12月6日
19 | * 申请了微博API
20 | * 添加了通过AppKey及AppSecret获取access_token的程序（./weiboAPI）:[olwolf/sinaweibopy3](https://github.com/olwolf/sinaweibopy3)
21 | * 根据[开发者文档](https://open.weibo.com/wiki/2/comments/show)构建获取评论的链接    
22 | <code>https://api.weibo.com/2/comments/show.json?access_token=[your_access_token]&id=[微博的ID]&count=[1~200]200&page=1</code>
23 | ### 2019年12月10日
24 | * 更新README.md,增加运行过程说明
25 | * 爬取香港相关微博的评论，数据保存为<code>./outPut/getInfo.json</code>及评论内容<code>./outPut/评论.csv</code>
26 | 


--------------------------------------------------------------------------------
/WeiboApi/readme.md:
--------------------------------------------------------------------------------
1 | # 此处是获取weiboAPI的access_token的代码    
2 | 带代码引用自[github@olwolf的sinaweibopy3项目](https://github.com/olwolf/sinaweibopy3)


--------------------------------------------------------------------------------
/WeiboApi/sinaweibopy3.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | 
  3 | import urllib.request
  4 | import urllib.parse
  5 | import json
  6 | import logging
  7 | import time
  8 | 
  9 | 
 10 | # general json object that can bind any fields but also act as a dict.
 11 | # a json class inhert dict class which can use d['key'] or d.key to get
 12 | class JsonDict(dict):
 13 |     def __getattr__(self, attr):
 14 |         return self[attr]
 15 | 
 16 |     def __setattr__(self, attr, value):
 17 |         self[attr] = value
 18 | 
 19 | 
 20 | # convert json object to python object.
 21 | def _obj_hook(paris):
 22 |     odict = JsonDict()
 23 |     for key, value in paris.items():
 24 |         odict[str(key)] = value
 25 |     return odict
 26 | 
 27 | 
 28 | # Encode parameters
 29 | def _encode_params(**kw):
 30 |     args = []
 31 |     for key, value in kw.items():
 32 |         para = value.encode('utf-8') if isinstance(value, str) else str(value)
 33 |         args.append('%s=%s' % (key, urllib.parse.quote(para)))
 34 |     return '&'.join(args)
 35 | 
 36 | 
 37 | # Build a multipart/form-data body with generated random boundary.
 38 | def _encode_multipart(**kw):
 39 |     # 'encode mulyipart data'
 40 |     boundary = '----------%s' % hex(int(time.time() * 1000))
 41 |     data = []
 42 |     for key, value in kw.items():
 43 |         data.append('--%s' % boundary)
 44 |         if hasattr(value, 'read'):
 45 |             filename = getattr(value, 'name', '')
 46 |             name = filename.rfind('.')
 47 |             ext = filename[name:].lower() if name != (-1) else ""
 48 |             content = value.read()
 49 |             content = content.decode('ISO-8859-1')
 50 |             data.append('Content-Disposition: form-data; name="%s"; filename="hidden"' % key)
 51 |             data.append('Content-Length: %d' % len(content))
 52 |             data.append('Content-Type: %s\r\n' % _guess_content_type(ext))
 53 |             data.append(content)
 54 |         else:
 55 |             data.append('Content-Disposition: form-data; name="%s"\r\n' % key)
 56 |             data.append(value if isinstance(value, str) else value.decode('utf-8'))
 57 |     data.append('--%s--\r\n' % boundary)
 58 |     return '\r\n'.join(data), boundary
 59 | 
 60 | 
 61 | _CONTENT_TYPES = {'.png': 'image/png', '.gif': 'image/gif', '.jpg': 'image/jpeg',
 62 |                   '.jpeg': 'image/jpeg', '.jpe': 'image/jpeg'}
 63 | 
 64 | 
 65 | def _guess_content_type(ext):
 66 |     return _CONTENT_TYPES.get(ext, 'application/octet-stream')
 67 | 
 68 | 
 69 | _HTTP_GET = 0
 70 | _HTTP_POST = 1
 71 | _HTTP_UPLOAD = 2
 72 | 
 73 | 
 74 | def _http_get(url, authorization=None, **kw):
 75 |     logging.info('GET %s' % url)
 76 |     return _http_request(url, _HTTP_GET, authorization, **kw)
 77 | 
 78 | 
 79 | def _http_post(url, authorization=None, **kw):
 80 |     logging.info('POST %s' % url)
 81 |     return _http_request(url, _HTTP_POST, authorization, **kw)
 82 | 
 83 | 
 84 | def _http_upload(url, authorization=None, **kw):
 85 |     logging.info('UPLOAD %s' % url)
 86 |     return _http_request(url, _HTTP_UPLOAD, authorization, **kw)
 87 | 
 88 | 
 89 | # send an http request and expect to return a json object if no error.
 90 | def _http_request(url, method, authorization, **kw):
 91 |     params = None
 92 |     boundary = None
 93 |     if method == _HTTP_UPLOAD:
 94 |         params, boundary = _encode_multipart(**kw)
 95 |     else:
 96 |         params = _encode_params(**kw)
 97 |     http_url = '%s?%s' % (url, params) if method == _HTTP_GET else url
 98 |     http_para = None if method == _HTTP_GET else params.encode(encoding='utf-8')
 99 |     # print(http_para)
100 |     req = urllib.request.Request(http_url, data=http_para)
101 |     if authorization:
102 |         req.add_header('Authorization', 'OAuth2 %s' % authorization)
103 |     if boundary:
104 |         req.add_header('Content-Type', 'multipart/form-data; boundary=%s' % boundary)
105 |     resq = urllib.request.urlopen(req)
106 |     body = resq.read().decode("utf-8")
107 |     result = json.loads(body, object_hook=_obj_hook)
108 |     if 'error_code' in result:
109 |         print('error')
110 |     return result
111 | 
112 | 
113 | class HttpObject(object):
114 |     'post get or updload object'
115 | 
116 |     def __init__(self, client, method):
117 |         self.client = client
118 |         self.method = method
119 | 
120 |     def __getattr__(self, attr):
121 |         def wrap(**kw):
122 |             'request param'
123 |             if self.client.is_expires():
124 |                 raise AttributeError
125 |             return _http_request('%s%s.json' % (self.client.api_url, attr.replace('__', '/')), self.method,
126 |                                  self.client.access_token, **kw)
127 | 
128 |         return wrap
129 | 
130 | 
131 | # APIClient class
132 | class APIClient(object):
133 |     def __init__(self, app_key, app_secret, redirect_uri=None, response_type='code', domain='api.weibo.com',
134 |                  version='2'):
135 |         self.client_id = app_key
136 |         self.client_secret = app_secret
137 |         self.redirect_uri = redirect_uri
138 |         self.response_type = response_type
139 |         self.auth_url = 'https://%s/oauth2/' % domain
140 |         self.api_url = 'https://%s/%s/' % (domain, version)
141 |         self.access_token = None
142 |         self.expires = 0.0  # 到期
143 |         self.get = HttpObject(self, _HTTP_GET)
144 |         self.post = HttpObject(self, _HTTP_POST)
145 |         self.upload = HttpObject(self, _HTTP_UPLOAD)
146 | 
147 |     # get authorize url得到授权url
148 |     def get_authorize_url(self):
149 |         return "https://api.weibo.com/oauth2/authorize?response_type=code&client_id=%s&redirect_uri=%s" % (
150 |         self.client_id, self.redirect_uri)
151 | 
152 |     # post a request and then get a access_token
153 |     def request_access_token(self, code):
154 |         result = _http_post('%s%s' % (self.auth_url, 'access_token'),
155 |                             client_id=self.client_id,
156 |                             client_secret=self.client_secret,
157 |                             redirect_uri=self.redirect_uri,
158 |                             code=code, grant_type='authorization_code')
159 |         result.expires_in += int(time.time())
160 |         return result
161 | 
162 |     # set access_token and expires_in
163 |     def set_access_token(self, access_token, expires_in):
164 |         self.access_token = str(access_token)
165 |         self.expires = float(expires_in)
166 | 
167 |     # Determine if the access token expires
168 |     def is_expires(self):
169 |         return not self.access_token or time.time() > self.expires
170 | 
171 |     # Custom function:Used to get the latest public Weibo
172 |     def public_timeline(self):
173 |         '''
174 |         get new public weibo,the parameters followed can be used in _http_get in this method
175 |         access_token : (string) the token you got after OAuth
176 |         count : (int) the record items in one single page,default 50 items
177 |         page : (int) the page number,default one page
178 |         base_app : (int) whether get data in current app or not,0 is not(all data),1 is yes(current app),default 0
179 |         '''
180 |         result = _http_get('%s' % (self.api_url) + 'statuses/public_timeline.json',
181 |                            access_token=self.access_token,
182 |                            count=50,
183 |                            page=1,
184 |                            base_app=0,
185 |                            )
186 |         return result


--------------------------------------------------------------------------------
/WeiboApi/weiboAPI.py:
--------------------------------------------------------------------------------
 1 | import webbrowser
 2 | from WeiboApi import sinaweibopy3
 3 | 
 4 | 
 5 | def main():
 6 |     try:
 7 |         APP_KEY = '2047157240'
 8 |         APP_SECRET = '57a21639366669c3f92c9c1672081f10'
 9 |         REDIRECT_URL = 'http://api.weibo.com/oauth2/default.html'
10 |         client = sinaweibopy3.APIClient(app_key=APP_KEY, app_secret=APP_SECRET, redirect_uri=REDIRECT_URL)
11 |         url = client.get_authorize_url()
12 |         webbrowser.open_new(url)
13 |         result = client.request_access_token(
14 |             input("please input code : "))
15 |         print(result)
16 |         client.set_access_token(result.access_token, result.expires_in)
17 |         print(client.public_timeline())
18 |         print(client.get.statuses__public_timeline())
19 |         print(client.get.account__get_uid())
20 | 
21 |     except ValueError:
22 |         print('pyOauth2Error')
23 | 
24 | 
25 | if __name__ == '__main__':
26 |     main()
27 | 


--------------------------------------------------------------------------------
/modelTraining.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python3.7
 2 | # -*- coding: utf-8 -*-
 3 | # @Time    : 2019/12/5 下午 05:09
 4 | # @Email   : pasalai@qq.com
 5 | # @Github  : github.com/laishouchao
 6 | # @File    : modelTraining.py
 7 | # @Software: PyCharm
 8 | 
 9 | import re
10 | # from snownlp import sentiment
11 | # import numpy as np
12 | import pymysql
13 | from snownlp import SnowNLP
14 | # import matplotlib.pyplot as plt
15 | from snownlp import sentiment
16 | # from snownlp.sentiment import Sentiment
17 | 
18 | conn = pymysql.connect(host='localhost', user='root', password='root', charset="utf8", use_unicode=False)  # 连接服务器
19 | with conn:
20 |     cur = conn.cursor()
21 |     cur.execute("SELECT * FROM test.weibo WHERE weiboId < '%d'" % 6000000)
22 |     rows = cur.fetchall()
23 | comment = []
24 | for row in rows:
25 |     row = list(row)
26 |     comment.append(row[18])
27 | 
28 | 
29 | def train_model(texts):
30 |     for li in texts:
31 |         comm = li.decode('utf-8')
32 |         text = re.sub(r'(?:回复)?(?://)?@[\w\u2E80-\u9FFF]+:?|\[\w+\]', ',', comm)
33 |         socre = SnowNLP(text)
34 |         if socre.sentiments > 0.8:
35 |             with open('./outPut/pos.txt', mode='a', encoding='utf-8') as g:
36 |                 g.writelines(comm + "\n")
37 |         elif socre.sentiments < 0.3:
38 |             with open('./outPut/neg.txt', mode='a', encoding='utf-8') as f:
39 |                 f.writelines(comm + "\n")
40 |         else:
41 |             pass
42 | 
43 | 
44 | train_model(comment)
45 | sentiment.train('./outPut/neg.txt', './outPut/pos.txt')
46 | sentiment.save('./outPut/sentiment.marshal')


--------------------------------------------------------------------------------
/outPut/final.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html>
  3 | <head>
  4 |     <meta charset="UTF-8">
  5 |     <title>Awesome-pyecharts</title>
  6 |             <script type="text/javascript" src="https://assets.pyecharts.org/assets/echarts.min.js"></script>
  7 | 
  8 | </head>
  9 | <body>
 10 |     <div id="15b41dae75814146b22ef22526f96807" class="chart-container" style="width:900px; height:500px;"></div>
 11 |     <script>
 12 |         var chart_15b41dae75814146b22ef22526f96807 = echarts.init(
 13 |             document.getElementById('15b41dae75814146b22ef22526f96807'), 'white', {renderer: 'canvas'});
 14 |         var option_15b41dae75814146b22ef22526f96807 = {
 15 |     "animation": true,
 16 |     "animationThreshold": 2000,
 17 |     "animationDuration": 1000,
 18 |     "animationEasing": "cubicOut",
 19 |     "animationDelay": 0,
 20 |     "animationDurationUpdate": 300,
 21 |     "animationEasingUpdate": "cubicOut",
 22 |     "animationDelayUpdate": 0,
 23 |     "color": "pink",
 24 |     "series": [
 25 |         {
 26 |             "type": "bar",
 27 |             "name": "\u4fe1\u606f\u6570\u91cf",
 28 |             "data": [
 29 |                 40,
 30 |                 9,
 31 |                 21,
 32 |                 44,
 33 |                 45,
 34 |                 351,
 35 |                 56,
 36 |                 45,
 37 |                 19,
 38 |                 3
 39 |             ],
 40 |             "barCategoryGap": "20%",
 41 |             "label": {
 42 |                 "show": true,
 43 |                 "position": "top",
 44 |                 "margin": 8
 45 |             }
 46 |         }
 47 |     ],
 48 |     "legend": [
 49 |         {
 50 |             "data": [
 51 |                 "\u4fe1\u606f\u6570\u91cf"
 52 |             ],
 53 |             "selected": {
 54 |                 "\u4fe1\u606f\u6570\u91cf": true
 55 |             },
 56 |             "show": true,
 57 |             "padding": 5,
 58 |             "itemGap": 10,
 59 |             "itemWidth": 25,
 60 |             "itemHeight": 14
 61 |         }
 62 |     ],
 63 |     "tooltip": {
 64 |         "show": true,
 65 |         "trigger": "item",
 66 |         "triggerOn": "mousemove|click",
 67 |         "axisPointer": {
 68 |             "type": "line"
 69 |         },
 70 |         "textStyle": {
 71 |             "fontSize": 14
 72 |         },
 73 |         "borderWidth": 0
 74 |     },
 75 |     "xAxis": [
 76 |         {
 77 |             "show": true,
 78 |             "scale": false,
 79 |             "nameLocation": "end",
 80 |             "nameGap": 15,
 81 |             "gridIndex": 0,
 82 |             "inverse": false,
 83 |             "offset": 0,
 84 |             "splitNumber": 5,
 85 |             "minInterval": 0,
 86 |             "splitLine": {
 87 |                 "show": false,
 88 |                 "lineStyle": {
 89 |                     "width": 1,
 90 |                     "opacity": 1,
 91 |                     "curveness": 0,
 92 |                     "type": "solid"
 93 |                 }
 94 |             },
 95 |             "data": [
 96 |                 "<0.1",
 97 |                 "<0.2",
 98 |                 "<0.3",
 99 |                 "<0.4",
100 |                 "<0.5",
101 |                 "<0.6",
102 |                 "<0.7",
103 |                 "<0.8",
104 |                 "<0.9",
105 |                 "<1"
106 |             ]
107 |         }
108 |     ],
109 |     "yAxis": [
110 |         {
111 |             "show": true,
112 |             "scale": false,
113 |             "nameLocation": "end",
114 |             "nameGap": 15,
115 |             "gridIndex": 0,
116 |             "inverse": false,
117 |             "offset": 0,
118 |             "splitNumber": 5,
119 |             "minInterval": 0,
120 |             "splitLine": {
121 |                 "show": false,
122 |                 "lineStyle": {
123 |                     "width": 1,
124 |                     "opacity": 1,
125 |                     "curveness": 0,
126 |                     "type": "solid"
127 |                 }
128 |             }
129 |         }
130 |     ],
131 |     "title": [
132 |         {
133 |             "text": "\u70ed\u641c\u60c5\u7eea\u5206\u6790"
134 |         }
135 |     ]
136 | };
137 |         chart_15b41dae75814146b22ef22526f96807.setOption(option_15b41dae75814146b22ef22526f96807);
138 |     </script>
139 | </body>
140 | </html>
141 | 


--------------------------------------------------------------------------------
/outPut/sentiment.marshal.3:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/pasalai/WeiboReview/ffcabfae5be51536cd279cead4c677fa73eeeb00/outPut/sentiment.marshal.3


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | snownlp
2 | json
3 | jieba
4 | pyecharts
5 | requests
6 | numpy


--------------------------------------------------------------------------------
/start.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python3.7
 2 | # -*- coding: utf-8 -*-
 3 | # @Time    : 2019/12/5 下午 05:35
 4 | # @Email   : pasalai@qq.com
 5 | # @Github  : github.com/laishouchao
 6 | # @File    : test.py
 7 | # @Software: PyCharm
 8 | 
 9 | import requests
10 | import time
11 | import random
12 | import json
13 | import re
14 | import pandas as pd
15 | import DataAnalysis
16 | import visualization
17 | 
18 | 
19 | class getInfo(object):
20 |     def __init__(self, page):
21 |         self.url = f"https://api.weibo.com/2/comments/show.json?access_token=2.00AICJkHKHfXOC0ca80471b00LcIop&id=4440448116340041&count=200&page={page}"
22 |         self.headers = {
23 |             "cookie": "Cookie: SINAGLOBAL=8274801057093.762.1564200851915; wvr=6; _s_tentry=www.baidu.com; Apache=6352301425561.806.1575945766559; ULV=1575945766630:13:7:1:6352301425561.806.1575945766559:1575617438977; ALF=1607481877; SSOLoginState=1575945878; SCF=AhkCB2rNvAv-9nbPzKRrD-HllSeFy8O2oFXjbSOgN5kqCRZSH96yV8fPl7TiizuN9zbfTU4_Fnc5HM7iz1cpBJU.; SUB=_2A25w63bGDeThGeFO4lYW-SrIyjqIHXVTge8OrDV8PUNbmtANLWnckW9NQVRWknLtmOSI5VUJquO4t5Xyv9Z4ocet; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWypBWbg6zJc73PgN7FFQAe5JpX5KzhUgL.FoM71KBN1KBXeKq2dJLoIp5LxKqL1-BLBKH0q0-0So.0; SUHB=0QSpCPOkz_IEJf; UOR=,,graph.qq.com; webim_unReadCount=%7B%22time%22%3A1575953836049%2C%22dm_pub_total%22%3A0%2C%22chat_group_client%22%3A0%2C%22allcountNum%22%3A0%2C%22msgbox%22%3A0%7D; JSESSIONID=F4590F83329600BDFB09BCD7A41F3B97",
24 |             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3941.4 Safari/537.36"}
25 | 
26 |     def Getinfo(self):
27 |         print(self.url)
28 |         response = requests.get(url=self.url, headers=self.headers).text
29 |         print(response)
30 |         # 将数据保存到json文件中，防止被反扒后没有数据
31 |         with open("./outPut/getinfo.json", "a", encoding="utf-8")as fp:
32 |             fp.write(response)
33 |         return response
34 | 
35 | 
36 | class run:
37 |     def __init__(self):
38 |         self.initList = []
39 | 
40 |     def run(self):
41 |         for pageNum in range(1, 10):
42 |             initgetinfo = getInfo(page=pageNum)  # 初始化网络访问
43 |             response = initgetinfo.Getinfo()
44 |             if "text" in response:
45 |                 infoJson = json.loads(response)
46 |                 print(infoJson["comments"][0]["text"])
47 |                 for num in range(len(infoJson["comments"])):
48 |                     infocont = infoJson["comments"][num]["text"]
49 |                     # 把一些链接去掉，保留纯文本内容。
50 |                     label_filter = re.compile(r'<span class="url-icon">.*?</span>', re.S)
51 |                     comment = re.sub(label_filter, '', infocont)
52 |                     print(comment)
53 |                     self.initList.append(comment)
54 |                     with open("./outPut/things.txt", "a", encoding="utf-8") as fp:
55 |                         fp.write(comment + "\n")
56 |                     data = pd.DataFrame(data=self.initList, columns=["评论内容"])
57 |                     data.to_csv("./outPut/评论.csv", mode="a", encoding="utf_8_sig")
58 |             else:
59 |                 print("continue")
60 |                 continue
61 |             time.sleep(random.randint(1, 5))
62 |             # return self.initList
63 | 
64 | 
65 | if __name__ == '__main__':
66 |     # 数据分析
67 |     initRun = run()
68 |     infoli = initRun.run()
69 |     print(infoli)
70 | 
71 |     initDataAnalysis = DataAnalysis.dataAnalysis()  # 初始化数据分析接口
72 |     with open("./outPut/things.txt", "r", encoding="utf-8") as fp:
73 |         for line in fp:
74 |             # print(line)
75 |             DAresponse = initDataAnalysis.DataAnalysis(keywords=line)  # 调用数据分析接口
76 |     # 数据统计
77 |     list_len = initDataAnalysis.Statistics()
78 |     # 数据可视化
79 |     visualization.visualization(list_len)  # 调用可视化绘图
80 | 


--------------------------------------------------------------------------------
/visualization.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python3.7
 2 | # -*- coding: utf-8 -*-
 3 | # @Time    : 2019/11/30 下午 11:39
 4 | # @Email   : pasalai@qq.com
 5 | # @Github  : github.com/laishouchao
 6 | # @File    : visualization.py
 7 | # @Software: PyCharm
 8 | import time
 9 | from pyecharts.charts import Bar
10 | from pyecharts.faker import Faker
11 | from pyecharts import options as opts
12 | from pyecharts.charts import Scatter
13 | 
14 | 
15 | def pycharts_visit():
16 |     c = (
17 |         Scatter()
18 |             .add_yaxis("负面情绪")
19 |             .add_xaxis("正面情绪")
20 |             .render("散点图.html")
21 |             .set_global_opts(title_opts=opts.TitleOpts(title="Scatter-基本示例"))
22 |     )
23 |     return c
24 | 
25 | 
26 | def visualization(list_len):
27 |     bar = Bar()
28 |     bar.set_global_opts(title_opts=opts.TitleOpts(title="热搜情绪分析"))
29 |     bar.set_colors(colors="pink")
30 |     bar.add_xaxis(["<0.1", "<0.2", "<0.3", "<0.4", "<0.5", "<0.6", "<0.7", "<0.8", "<0.9", "<1"])
31 |     bar.add_yaxis("信息数量", list_len)
32 |     bar.render("./outPut/final.html")
33 |     time.sleep(3)  # 给人一种运行起来贼麻烦的仪式感
34 |     print("\n\n[√]可视化页面生成完成！")
35 | 


--------------------------------------------------------------------------------