39 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | env/
12 | build/
13 | develop-eggs/
14 | dist/
15 | downloads/
16 | eggs/
17 | .eggs/
18 | lib/
19 | lib64/
20 | parts/
21 | sdist/
22 | var/
23 | wheels/
24 | *.egg-info/
25 | .installed.cfg
26 | *.egg
27 |
28 | # PyInstaller
29 | # Usually these files are written by a python script from a template
30 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 | *.manifest
32 | *.spec
33 |
34 | # Installer logs
35 | pip-log.txt
36 | pip-delete-this-directory.txt
37 |
38 | # Unit test / coverage reports
39 | htmlcov/
40 | .tox/
41 | .coverage
42 | .coverage.*
43 | .cache
44 | nosetests.xml
45 | coverage.xml
46 | *.cover
47 | .hypothesis/
48 |
49 | # Translations
50 | *.mo
51 | *.pot
52 |
53 | # Django stuff:
54 | *.log
55 | local_settings.py
56 |
57 | # Flask stuff:
58 | instance/
59 | .webassets-cache
60 |
61 | # Scrapy stuff:
62 | .scrapy
63 |
64 | # Sphinx documentation
65 | docs/_build/
66 |
67 | # PyBuilder
68 | target/
69 |
70 | # Jupyter Notebook
71 | .ipynb_checkpoints
72 |
73 | # pyenv
74 | .python-version
75 |
76 | # celery beat schedule file
77 | celerybeat-schedule
78 |
79 | # SageMath parsed files
80 | *.sage.py
81 |
82 | # dotenv
83 | .env
84 |
85 | # virtualenv
86 | .venv
87 | venv/
88 | ENV/
89 |
90 | # Spyder project settings
91 | .spyderproject
92 | .spyproject
93 |
94 | # Rope project settings
95 | .ropeproject
96 |
97 | # mkdocs documentation
98 | /site
99 |
100 | # mypy
101 | .mypy_cache/
102 |
--------------------------------------------------------------------------------
/data_analys.py:
--------------------------------------------------------------------------------
1 | import json
2 | import os
3 | import xlwt
4 | import pymysql
5 | #存入Excel
6 | # def dataToExcel():
7 | # d=[i for i in os.listdir('mood_detail') if not i.endswith('.xls')]
8 | # for ii in d:
9 | # wb=xlwt.Workbook()
10 | # sheet=wb.add_sheet('sheet1',cell_overwrite_ok=True)
11 | # sheet.write(0,0,'content')
12 | # sheet.write(0,1,'createTime')
13 | # sheet.write(0,2,'commentlist')
14 | # sheet.write(0,3,'source_name')
15 | # sheet.write(0,4,'cmtnum')
16 | # fl=[i for i in os.listdir('mood_detail/'+ii) if i.endswith('.json')]
17 | # print('mood_detail/'+ii)
18 | # k=1
19 | # for i in fl:
20 | # with open('mood_detail/'+ii+"/"+i,'r',encoding='latin-1') as w:
21 | # s=w.read()[17:-2]
22 | # js=json.loads(s)
23 | # print(i)
24 | # for s in js['msglist']:
25 | # m=-1
26 | # sheet.write(k,m+1,str(s['content']))
27 | # sheet.write(k,m+2,str(s['createTime']))
28 | # if not s['commentlist']:
29 | # s['commentlist']=list()
30 | # sheet.write(k,m+3,str([(x['content'],x['createTime2'],x['name'],x['uin']) for x in list(s['commentlist'])]))
31 | # sheet.write(k,m+4,str(s['source_name']))
32 | # sheet.write(k,m+5,str(s['cmtnum']))
33 | # k+=1
34 | # if not os.path.exists('mood_detail/Excel/'):
35 | # os.mkdir('mood_detail/Excel/')
36 | # try:
37 | # wb.save('mood_detail/Excel/'+ii+'.xls')
38 | # except Exception:
39 | # print("error")
40 | #
41 |
42 |
43 | def dataToExcel():
44 | ll=0
45 | d=[i for i in os.listdir('mood_detail') if not i.endswith('.xls')]
46 | for ii in d:
47 | # wb=xlwt.Workbook()
48 | # sheet=wb.add_sheet('sheet1',cell_overwrite_ok=True)
49 | # sheet.write(0,0,'content')
50 | # sheet.write(0,1,'createTime')
51 | # sheet.write(0,2,'commentlist')
52 | # sheet.write(0,3,'source_name')
53 | # sheet.write(0,4,'cmtnum')
54 | fl=[i for i in os.listdir('mood_detail/'+ii) if i.endswith('.json')]
55 | print('mood_detail/'+ii)
56 | k=1
57 | for i in fl:
58 | with open('mood_detail/'+ii+"/"+i,'r',encoding='latin-1') as w:
59 | s=w.read()[17:-2]
60 | js=json.loads(s)
61 | print(i)
62 | for s in js['msglist']:
63 | ll+=1
64 | m=-1
65 | # sheet.write(k,m+1,str(s['content']))
66 | # sheet.write(k,m+2,str(s['createTime']))
67 | if not s['commentlist']:
68 | s['commentlist']=list()
69 | # sheet.write(k,m+3,str([(x['content'],x['createTime2'],x['name'],x['uin']) for x in list(s['commentlist'])]))
70 | # sheet.write(k,m+4,str(s['source_name']))
71 | # sheet.write(k,m+5,str(s['cmtnum']))
72 | k+=1
73 | if not os.path.exists('mood_detail/Excel/'):
74 | os.mkdir('mood_detail/Excel/')
75 | print(ll)
76 | # try:
77 | # wb.save('mood_detail/Excel/'+ii+'.xls')
78 | # except Exception:
79 | # print("error")
80 |
81 | dataToExcel()
82 |
83 |
84 |
85 |
86 | '''
87 | 存入数据库
88 | def dataToExcel():
89 | con=pymysql.connect(
90 | host='127.0.0.1',
91 | user='root',
92 | password="×××",
93 | database='qq_z',
94 | port=3306,
95 | )
96 | cur=con.cursor()
97 | sql="insert into info (qq_number,created_time,content,commentlist,source_name,cmtnum,name) values ({},{},{},{},{},{},{});"
98 |
99 | d=[i for i in os.listdir('mood_detail') if not i.endswith('.xls')]
100 | for ii in d:
101 | fl=[i for i in os.listdir('mood_detail/'+ii) if i.endswith('.json')]
102 | print('mood_detail/'+ii)
103 | k=1
104 | for i in fl:
105 | with open('mood_detail/'+ii+"/"+i,'r',encoding='latin-1') as w:
106 | s=w.read()[17:-2]
107 | js=json.loads(s)
108 | print(i)
109 | for s in js['msglist']:
110 | m=-1
111 | if not s['commentlist']:
112 | s['commentlist']=list()
113 | cur.execute(sql.format(int(i[:i.find('_')]),s['created_time'],str(s['content']),str([(x['content'],x['createTime2'],x['name'],x['uin']) for x in list(s['commentlist'])]),str(s['source_name']),int(s['cmtnum']),str(s['name'])))
114 | k+=1
115 | con.commit()
116 | con.close()
117 | '''
--------------------------------------------------------------------------------
/qq_spider.py:
--------------------------------------------------------------------------------
1 | #coding:utf-8
2 | from selenium import webdriver
3 | import requests
4 | import time
5 | import os
6 | from urllib import parse
7 | import configparser
8 |
9 | class Spider(object):
10 | def __init__(self):
11 | self.web=webdriver.Chrome()
12 | self.web.get('https://user.qzone.qq.com')
13 | config = configparser.ConfigParser(allow_no_value=False)
14 | config.read('userinfo.ini')
15 | self.__username =config.get('qq_info','qq_number')
16 | self.__password=config.get('qq_info','qq_password')
17 | self.headers={
18 | 'host': 'h5.qzone.qq.com',
19 | 'accept-encoding':'gzip, deflate, br',
20 | 'accept-language':'zh-CN,zh;q=0.8',
21 | 'accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
22 | 'user-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36',
23 | 'connection': 'keep-alive'
24 | }
25 | self.req=requests.Session()
26 | self.cookies={}
27 |
28 |
29 |
30 | def login(self):
31 | self.web.switch_to_frame('login_frame')
32 | log=self.web.find_element_by_id("switcher_plogin")
33 | log.click()
34 | time.sleep(1)
35 | username=self.web.find_element_by_id('u')
36 | username.send_keys(self.__username)
37 | ps=self.web.find_element_by_id('p')
38 | ps.send_keys(self.__password)
39 | btn=self.web.find_element_by_id('login_button')
40 | time.sleep(1)
41 | btn.click()
42 | time.sleep(2)
43 | self.web.get('https://user.qzone.qq.com/{}'.format(self.__username))
44 | cookie=''
45 | for elem in self.web.get_cookies():
46 | cookie+=elem["name"]+"="+ elem["value"]+";"
47 | self.cookies=cookie
48 | self.get_g_tk()
49 | self.headers['Cookie']=self.cookies
50 | self.web.quit()
51 |
52 |
53 | def get_frends_url(self):
54 | url='https://h5.qzone.qq.com/proxy/domain/base.qzone.qq.com/cgi-bin/right/get_entryuinlist.cgi?'
55 | params = {"uin": self.__username,
56 | "fupdate": 1,
57 | "action": 1,
58 | "g_tk": self.g_tk}
59 | url = url + parse.urlencode(params)
60 | return url
61 |
62 | def get_frends_num(self):
63 | t=True
64 | offset=0
65 | url=self.get_frends_url()
66 | while(t):
67 | url_=url+'&offset='+str(offset)
68 | page=self.req.get(url=url_,headers=self.headers)
69 | if "\"uinlist\":[]" in page.text:
70 | t=False
71 | else:
72 |
73 | if not os.path.exists("./frends/"):
74 | os.mkdir("frends/")
75 | with open('./frends/'+str(offset)+'.json','w',encoding='utf-8') as w:
76 | w.write(page.text)
77 | offset += 50
78 |
79 | def get_mood_url(self):
80 | url='https://h5.qzone.qq.com/proxy/domain/taotao.qq.com/cgi-bin/emotion_cgi_msglist_v6?'
81 | params = {
82 | "sort":0,
83 | "start":0,
84 | "num":20,
85 | "cgi_host": "http://taotao.qq.com/cgi-bin/emotion_cgi_msglist_v6",
86 | "replynum":100,
87 | "callback":"_preloadCallback",
88 | "code_version":1,
89 | "inCharset": "utf-8",
90 | "outCharset": "utf-8",
91 | "notice": 0,
92 | "format":"jsonp",
93 | "need_private_comment":1,
94 | "g_tk": self.g_tk
95 | }
96 | url = url + parse.urlencode(params)
97 | return url
98 |
99 |
100 | def get_mood_detail(self):
101 | from getFrends import frends_list
102 | url = self.get_mood_url()
103 | for u in frends_list[245:]:
104 | t = True
105 | QQ_number=u['data']
106 | url_ = url + '&uin=' + str(QQ_number)
107 | pos = 0
108 | while (t):
109 | url__ = url_ + '&pos=' + str(pos)
110 | mood_detail = self.req.get(url=url__, headers=self.headers)
111 | print(QQ_number,u['label'],pos)
112 | if "\"msglist\":null" in mood_detail.text or "\"message\":\"对不起,主人设置了保密,您没有权限查看\"" in mood_detail.text:
113 | t = False
114 | else:
115 | if not os.path.exists("./mood_detail/"):
116 | os.mkdir("mood_detail/")
117 | if not os.path.exists("./mood_detail/"+u['label']):
118 | os.mkdir("mood_detail/"+u['label'])
119 | with open('./mood_detail/'+u['label']+"/" +str(QQ_number)+"_"+ str(pos) + '.json', 'w',encoding='utf-8') as w:
120 | w.write(mood_detail.text)
121 | pos += 20
122 | time.sleep(2)
123 |
124 |
125 | def get_g_tk(self):
126 | p_skey = self.cookies[self.cookies.find('p_skey=')+7: self.cookies.find(';', self.cookies.find('p_skey='))]
127 | h=5381
128 | for i in p_skey:
129 | h+=(h<<5)+ord(i)
130 | print('g_tk',h&2147483647)
131 | self.g_tk=h&2147483647
132 |
133 |
134 |
135 |
136 | if __name__=='__main__':
137 | sp=Spider()
138 | sp.login()
139 | sp.get_frends_num()
140 | sp.get_mood_detail()
141 | from data_analys import dataToExcel
142 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "{}"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright {yyyy} {name of copyright owner}
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
66 |
67 |
68 |
69 |
70 | true
71 | DEFINITION_ORDER
72 |
73 |
74 |