├── .gitignore
├── .idea
├── WFACat.iml
├── dictionaries
│ └── Mo.xml
├── encodings.xml
├── libraries
│ └── R_User_Library.xml
├── misc.xml
├── modules.xml
├── vcs.xml
└── workspace.xml
├── LICENSE
├── Pipfile
├── Pipfile.lock
├── README.md
├── README_img
├── 图1.PNG
├── 图2.PNG
├── 图3.PNG
├── 图4.PNG
├── 图5.PNG
├── 图6.PNG
├── 图7.PNG
├── 图8-1.PNG
├── 图8.PNG
└── 图9.PNG
├── WFACatPro
├── main.py
└── modules
│ ├── __init__.py
│ ├── analysis_to_csv.py
│ ├── analysis_to_mysql.py
│ ├── fixed.py
│ ├── get_data.py
│ ├── help.py
│ ├── mysql_query.py
│ ├── settings.py
│ ├── tools.py
│ └── version.py
└── docs
└── 软件架构文档.PNG
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | *.egg-info/
24 | .installed.cfg
25 | *.egg
26 | MANIFEST
27 |
28 | # PyInstaller
29 | # Usually these files are written by a python script from a template
30 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 | *.manifest
32 | *.spec
33 |
34 | # Installer logs
35 | pip-log.txt
36 | pip-delete-this-directory.txt
37 |
38 | # Unit test / coverage reports
39 | htmlcov/
40 | .tox/
41 | .nox/
42 | .coverage
43 | .coverage.*
44 | .cache
45 | nosetests.xml
46 | coverage.xml
47 | *.cover
48 | .hypothesis/
49 | .pytest_cache/
50 |
51 | # Translations
52 | *.mo
53 | *.pot
54 |
55 | # Django stuff:
56 | *.log
57 | local_settings.py
58 | db.sqlite3
59 |
60 | # Flask stuff:
61 | instance/
62 | .webassets-cache
63 |
64 | # Scrapy stuff:
65 | .scrapy
66 |
67 | # Sphinx documentation
68 | docs/_build/
69 |
70 | # PyBuilder
71 | target/
72 |
73 | # Jupyter Notebook
74 | .ipynb_checkpoints
75 |
76 | # IPython
77 | profile_default/
78 | ipython_config.py
79 |
80 | # pyenv
81 | .python-version
82 |
83 | # celery beat schedule file
84 | celerybeat-schedule
85 |
86 | # SageMath parsed files
87 | *.sage.py
88 |
89 | # Environments
90 | .env
91 | .venv
92 | env/
93 | venv/
94 | ENV/
95 | env.bak/
96 | venv.bak/
97 |
98 | # Spyder project settings
99 | .spyderproject
100 | .spyproject
101 |
102 | # Rope project settings
103 | .ropeproject
104 |
105 | # mkdocs documentation
106 | /site
107 |
108 | # mypy
109 | .mypy_cache/
110 | .dmypy.json
111 | dmypy.json
112 | /WFACatPro/WFACat_data/
--------------------------------------------------------------------------------
/.idea/WFACat.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
--------------------------------------------------------------------------------
/.idea/dictionaries/Mo.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/.idea/encodings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/libraries/R_User_Library.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
48 |
49 |
50 |
51 |
52 |
53 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
255 |
261 |
262 |
263 |
269 |
270 |
271 |
272 |
273 |
274 |
275 |
276 |
277 | 1554732209692
278 |
279 |
280 | 1554732209692
281 |
282 |
283 |
284 |
285 |
286 |
287 |
288 |
289 |
290 |
291 |
292 |
293 |
294 |
295 |
296 |
297 |
298 |
299 |
300 |
301 |
302 |
303 |
304 |
305 |
306 |
307 |
308 |
309 |
310 |
311 |
312 |
313 |
314 |
315 |
316 |
317 |
318 |
319 |
320 |
321 |
322 |
323 |
324 |
325 |
326 |
327 |
328 |
329 |
330 |
331 |
332 |
333 |
334 |
335 |
336 |
337 |
338 |
339 |
340 |
341 |
342 |
343 |
344 |
345 |
346 |
347 |
348 |
349 |
350 |
351 |
352 |
353 |
354 |
355 |
356 |
357 |
358 |
359 |
360 |
361 |
362 |
363 |
364 |
365 |
366 |
367 |
368 |
369 |
370 |
371 |
372 |
373 |
374 |
375 |
376 |
377 |
378 |
379 |
380 |
381 |
382 |
383 |
384 |
385 |
386 |
387 |
388 |
389 |
390 |
391 |
392 |
393 |
394 |
395 |
396 |
397 |
398 |
399 |
400 |
401 |
402 |
403 |
404 |
405 |
406 |
407 |
408 |
409 |
410 |
411 |
412 |
413 |
414 |
415 |
416 |
417 |
418 |
419 |
420 |
421 |
422 |
423 |
424 |
425 |
426 |
427 |
428 |
429 |
430 |
431 |
432 |
433 |
434 |
435 |
436 |
437 |
438 |
439 |
440 |
441 |
442 |
443 |
444 |
445 |
446 |
447 |
448 |
449 |
450 |
451 |
452 |
453 |
454 |
455 |
456 |
457 |
458 |
459 |
460 |
461 |
462 |
463 |
464 |
465 |
466 |
467 |
468 |
469 |
470 |
471 |
472 |
477 |
478 |
479 |
480 |
481 |
482 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | GNU GENERAL PUBLIC LICENSE
2 | Version 2, June 1991
3 |
4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
6 | Everyone is permitted to copy and distribute verbatim copies
7 | of this license document, but changing it is not allowed.
8 |
9 | Preamble
10 |
11 | The licenses for most software are designed to take away your
12 | freedom to share and change it. By contrast, the GNU General Public
13 | License is intended to guarantee your freedom to share and change free
14 | software--to make sure the software is free for all its users. This
15 | General Public License applies to most of the Free Software
16 | Foundation's software and to any other program whose authors commit to
17 | using it. (Some other Free Software Foundation software is covered by
18 | the GNU Lesser General Public License instead.) You can apply it to
19 | your programs, too.
20 |
21 | When we speak of free software, we are referring to freedom, not
22 | price. Our General Public Licenses are designed to make sure that you
23 | have the freedom to distribute copies of free software (and charge for
24 | this service if you wish), that you receive source code or can get it
25 | if you want it, that you can change the software or use pieces of it
26 | in new free programs; and that you know you can do these things.
27 |
28 | To protect your rights, we need to make restrictions that forbid
29 | anyone to deny you these rights or to ask you to surrender the rights.
30 | These restrictions translate to certain responsibilities for you if you
31 | distribute copies of the software, or if you modify it.
32 |
33 | For example, if you distribute copies of such a program, whether
34 | gratis or for a fee, you must give the recipients all the rights that
35 | you have. You must make sure that they, too, receive or can get the
36 | source code. And you must show them these terms so they know their
37 | rights.
38 |
39 | We protect your rights with two steps: (1) copyright the software, and
40 | (2) offer you this license which gives you legal permission to copy,
41 | distribute and/or modify the software.
42 |
43 | Also, for each author's protection and ours, we want to make certain
44 | that everyone understands that there is no warranty for this free
45 | software. If the software is modified by someone else and passed on, we
46 | want its recipients to know that what they have is not the original, so
47 | that any problems introduced by others will not reflect on the original
48 | authors' reputations.
49 |
50 | Finally, any free program is threatened constantly by software
51 | patents. We wish to avoid the danger that redistributors of a free
52 | program will individually obtain patent licenses, in effect making the
53 | program proprietary. To prevent this, we have made it clear that any
54 | patent must be licensed for everyone's free use or not licensed at all.
55 |
56 | The precise terms and conditions for copying, distribution and
57 | modification follow.
58 |
59 | GNU GENERAL PUBLIC LICENSE
60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
61 |
62 | 0. This License applies to any program or other work which contains
63 | a notice placed by the copyright holder saying it may be distributed
64 | under the terms of this General Public License. The "Program", below,
65 | refers to any such program or work, and a "work based on the Program"
66 | means either the Program or any derivative work under copyright law:
67 | that is to say, a work containing the Program or a portion of it,
68 | either verbatim or with modifications and/or translated into another
69 | language. (Hereinafter, translation is included without limitation in
70 | the term "modification".) Each licensee is addressed as "you".
71 |
72 | Activities other than copying, distribution and modification are not
73 | covered by this License; they are outside its scope. The act of
74 | running the Program is not restricted, and the output from the Program
75 | is covered only if its contents constitute a work based on the
76 | Program (independent of having been made by running the Program).
77 | Whether that is true depends on what the Program does.
78 |
79 | 1. You may copy and distribute verbatim copies of the Program's
80 | source code as you receive it, in any medium, provided that you
81 | conspicuously and appropriately publish on each copy an appropriate
82 | copyright notice and disclaimer of warranty; keep intact all the
83 | notices that refer to this License and to the absence of any warranty;
84 | and give any other recipients of the Program a copy of this License
85 | along with the Program.
86 |
87 | You may charge a fee for the physical act of transferring a copy, and
88 | you may at your option offer warranty protection in exchange for a fee.
89 |
90 | 2. You may modify your copy or copies of the Program or any portion
91 | of it, thus forming a work based on the Program, and copy and
92 | distribute such modifications or work under the terms of Section 1
93 | above, provided that you also meet all of these conditions:
94 |
95 | a) You must cause the modified files to carry prominent notices
96 | stating that you changed the files and the date of any change.
97 |
98 | b) You must cause any work that you distribute or publish, that in
99 | whole or in part contains or is derived from the Program or any
100 | part thereof, to be licensed as a whole at no charge to all third
101 | parties under the terms of this License.
102 |
103 | c) If the modified program normally reads commands interactively
104 | when run, you must cause it, when started running for such
105 | interactive use in the most ordinary way, to print or display an
106 | announcement including an appropriate copyright notice and a
107 | notice that there is no warranty (or else, saying that you provide
108 | a warranty) and that users may redistribute the program under
109 | these conditions, and telling the user how to view a copy of this
110 | License. (Exception: if the Program itself is interactive but
111 | does not normally print such an announcement, your work based on
112 | the Program is not required to print an announcement.)
113 |
114 | These requirements apply to the modified work as a whole. If
115 | identifiable sections of that work are not derived from the Program,
116 | and can be reasonably considered independent and separate works in
117 | themselves, then this License, and its terms, do not apply to those
118 | sections when you distribute them as separate works. But when you
119 | distribute the same sections as part of a whole which is a work based
120 | on the Program, the distribution of the whole must be on the terms of
121 | this License, whose permissions for other licensees extend to the
122 | entire whole, and thus to each and every part regardless of who wrote it.
123 |
124 | Thus, it is not the intent of this section to claim rights or contest
125 | your rights to work written entirely by you; rather, the intent is to
126 | exercise the right to control the distribution of derivative or
127 | collective works based on the Program.
128 |
129 | In addition, mere aggregation of another work not based on the Program
130 | with the Program (or with a work based on the Program) on a volume of
131 | a storage or distribution medium does not bring the other work under
132 | the scope of this License.
133 |
134 | 3. You may copy and distribute the Program (or a work based on it,
135 | under Section 2) in object code or executable form under the terms of
136 | Sections 1 and 2 above provided that you also do one of the following:
137 |
138 | a) Accompany it with the complete corresponding machine-readable
139 | source code, which must be distributed under the terms of Sections
140 | 1 and 2 above on a medium customarily used for software interchange; or,
141 |
142 | b) Accompany it with a written offer, valid for at least three
143 | years, to give any third party, for a charge no more than your
144 | cost of physically performing source distribution, a complete
145 | machine-readable copy of the corresponding source code, to be
146 | distributed under the terms of Sections 1 and 2 above on a medium
147 | customarily used for software interchange; or,
148 |
149 | c) Accompany it with the information you received as to the offer
150 | to distribute corresponding source code. (This alternative is
151 | allowed only for noncommercial distribution and only if you
152 | received the program in object code or executable form with such
153 | an offer, in accord with Subsection b above.)
154 |
155 | The source code for a work means the preferred form of the work for
156 | making modifications to it. For an executable work, complete source
157 | code means all the source code for all modules it contains, plus any
158 | associated interface definition files, plus the scripts used to
159 | control compilation and installation of the executable. However, as a
160 | special exception, the source code distributed need not include
161 | anything that is normally distributed (in either source or binary
162 | form) with the major components (compiler, kernel, and so on) of the
163 | operating system on which the executable runs, unless that component
164 | itself accompanies the executable.
165 |
166 | If distribution of executable or object code is made by offering
167 | access to copy from a designated place, then offering equivalent
168 | access to copy the source code from the same place counts as
169 | distribution of the source code, even though third parties are not
170 | compelled to copy the source along with the object code.
171 |
172 | 4. You may not copy, modify, sublicense, or distribute the Program
173 | except as expressly provided under this License. Any attempt
174 | otherwise to copy, modify, sublicense or distribute the Program is
175 | void, and will automatically terminate your rights under this License.
176 | However, parties who have received copies, or rights, from you under
177 | this License will not have their licenses terminated so long as such
178 | parties remain in full compliance.
179 |
180 | 5. You are not required to accept this License, since you have not
181 | signed it. However, nothing else grants you permission to modify or
182 | distribute the Program or its derivative works. These actions are
183 | prohibited by law if you do not accept this License. Therefore, by
184 | modifying or distributing the Program (or any work based on the
185 | Program), you indicate your acceptance of this License to do so, and
186 | all its terms and conditions for copying, distributing or modifying
187 | the Program or works based on it.
188 |
189 | 6. Each time you redistribute the Program (or any work based on the
190 | Program), the recipient automatically receives a license from the
191 | original licensor to copy, distribute or modify the Program subject to
192 | these terms and conditions. You may not impose any further
193 | restrictions on the recipients' exercise of the rights granted herein.
194 | You are not responsible for enforcing compliance by third parties to
195 | this License.
196 |
197 | 7. If, as a consequence of a court judgment or allegation of patent
198 | infringement or for any other reason (not limited to patent issues),
199 | conditions are imposed on you (whether by court order, agreement or
200 | otherwise) that contradict the conditions of this License, they do not
201 | excuse you from the conditions of this License. If you cannot
202 | distribute so as to satisfy simultaneously your obligations under this
203 | License and any other pertinent obligations, then as a consequence you
204 | may not distribute the Program at all. For example, if a patent
205 | license would not permit royalty-free redistribution of the Program by
206 | all those who receive copies directly or indirectly through you, then
207 | the only way you could satisfy both it and this License would be to
208 | refrain entirely from distribution of the Program.
209 |
210 | If any portion of this section is held invalid or unenforceable under
211 | any particular circumstance, the balance of the section is intended to
212 | apply and the section as a whole is intended to apply in other
213 | circumstances.
214 |
215 | It is not the purpose of this section to induce you to infringe any
216 | patents or other property right claims or to contest validity of any
217 | such claims; this section has the sole purpose of protecting the
218 | integrity of the free software distribution system, which is
219 | implemented by public license practices. Many people have made
220 | generous contributions to the wide range of software distributed
221 | through that system in reliance on consistent application of that
222 | system; it is up to the author/donor to decide if he or she is willing
223 | to distribute software through any other system and a licensee cannot
224 | impose that choice.
225 |
226 | This section is intended to make thoroughly clear what is believed to
227 | be a consequence of the rest of this License.
228 |
229 | 8. If the distribution and/or use of the Program is restricted in
230 | certain countries either by patents or by copyrighted interfaces, the
231 | original copyright holder who places the Program under this License
232 | may add an explicit geographical distribution limitation excluding
233 | those countries, so that distribution is permitted only in or among
234 | countries not thus excluded. In such case, this License incorporates
235 | the limitation as if written in the body of this License.
236 |
237 | 9. The Free Software Foundation may publish revised and/or new versions
238 | of the General Public License from time to time. Such new versions will
239 | be similar in spirit to the present version, but may differ in detail to
240 | address new problems or concerns.
241 |
242 | Each version is given a distinguishing version number. If the Program
243 | specifies a version number of this License which applies to it and "any
244 | later version", you have the option of following the terms and conditions
245 | either of that version or of any later version published by the Free
246 | Software Foundation. If the Program does not specify a version number of
247 | this License, you may choose any version ever published by the Free Software
248 | Foundation.
249 |
250 | 10. If you wish to incorporate parts of the Program into other free
251 | programs whose distribution conditions are different, write to the author
252 | to ask for permission. For software which is copyrighted by the Free
253 | Software Foundation, write to the Free Software Foundation; we sometimes
254 | make exceptions for this. Our decision will be guided by the two goals
255 | of preserving the free status of all derivatives of our free software and
256 | of promoting the sharing and reuse of software generally.
257 |
258 | NO WARRANTY
259 |
260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
268 | REPAIR OR CORRECTION.
269 |
270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
278 | POSSIBILITY OF SUCH DAMAGES.
279 |
280 | END OF TERMS AND CONDITIONS
281 |
282 | How to Apply These Terms to Your New Programs
283 |
284 | If you develop a new program, and you want it to be of the greatest
285 | possible use to the public, the best way to achieve this is to make it
286 | free software which everyone can redistribute and change under these terms.
287 |
288 | To do so, attach the following notices to the program. It is safest
289 | to attach them to the start of each source file to most effectively
290 | convey the exclusion of warranty; and each file should have at least
291 | the "copyright" line and a pointer to where the full notice is found.
292 |
293 |
294 | Copyright (C)
295 |
296 | This program is free software; you can redistribute it and/or modify
297 | it under the terms of the GNU General Public License as published by
298 | the Free Software Foundation; either version 2 of the License, or
299 | (at your option) any later version.
300 |
301 | This program is distributed in the hope that it will be useful,
302 | but WITHOUT ANY WARRANTY; without even the implied warranty of
303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
304 | GNU General Public License for more details.
305 |
306 | You should have received a copy of the GNU General Public License along
307 | with this program; if not, write to the Free Software Foundation, Inc.,
308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
309 |
310 | Also add information on how to contact you by electronic and paper mail.
311 |
312 | If the program is interactive, make it output a short notice like this
313 | when it starts in an interactive mode:
314 |
315 | Gnomovision version 69, Copyright (C) year name of author
316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
317 | This is free software, and you are welcome to redistribute it
318 | under certain conditions; type `show c' for details.
319 |
320 | The hypothetical commands `show w' and `show c' should show the appropriate
321 | parts of the General Public License. Of course, the commands you use may
322 | be called something other than `show w' and `show c'; they could even be
323 | mouse-clicks or menu items--whatever suits your program.
324 |
325 | You should also get your employer (if you work as a programmer) or your
326 | school, if any, to sign a "copyright disclaimer" for the program, if
327 | necessary. Here is a sample; alter the names:
328 |
329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program
330 | `Gnomovision' (which makes passes at compilers) written by James Hacker.
331 |
332 | , 1 April 1989
333 | Ty Coon, President of Vice
334 |
335 | This General Public License does not permit incorporating your program into
336 | proprietary programs. If your program is a subroutine library, you may
337 | consider it more useful to permit linking proprietary applications with the
338 | library. If this is what you want to do, use the GNU Lesser General
339 | Public License instead of this License.
340 |
--------------------------------------------------------------------------------
/Pipfile:
--------------------------------------------------------------------------------
1 | [[source]]
2 | name = "pypi"
3 | url = "https://pypi.org/simple"
4 | verify_ssl = true
5 |
6 | [dev-packages]
7 |
8 | [packages]
9 | pymysql = "*"
10 | requests = "*"
11 |
12 | [requires]
13 | python_version = "3.7"
14 |
--------------------------------------------------------------------------------
/Pipfile.lock:
--------------------------------------------------------------------------------
1 | {
2 | "_meta": {
3 | "hash": {
4 | "sha256": "fb8200a1d06ab3a60dbfb8d798d2a175b87a9c2b356b19c248c26bf65e0f80c2"
5 | },
6 | "pipfile-spec": 6,
7 | "requires": {
8 | "python_version": "3.7"
9 | },
10 | "sources": [
11 | {
12 | "name": "pypi",
13 | "url": "https://pypi.org/simple",
14 | "verify_ssl": true
15 | }
16 | ]
17 | },
18 | "default": {
19 | "certifi": {
20 | "hashes": [
21 | "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5",
22 | "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae"
23 | ],
24 | "version": "==2019.3.9"
25 | },
26 | "chardet": {
27 | "hashes": [
28 | "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
29 | "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
30 | ],
31 | "version": "==3.0.4"
32 | },
33 | "idna": {
34 | "hashes": [
35 | "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
36 | "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
37 | ],
38 | "version": "==2.8"
39 | },
40 | "pymysql": {
41 | "hashes": [
42 | "sha256:3943fbbbc1e902f41daf7f9165519f140c4451c179380677e6a848587042561a",
43 | "sha256:d8c059dcd81dedb85a9f034d5e22dcb4442c0b201908bede99e306d65ea7c8e7"
44 | ],
45 | "index": "pypi",
46 | "version": "==0.9.3"
47 | },
48 | "requests": {
49 | "hashes": [
50 | "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
51 | "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
52 | ],
53 | "index": "pypi",
54 | "version": "==2.21.0"
55 | },
56 | "urllib3": {
57 | "hashes": [
58 | "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
59 | "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
60 | ],
61 | "version": "==1.24.1"
62 | }
63 | },
64 | "develop": {}
65 | }
66 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # WFACat
2 | ## Introduction - 介绍
3 | Weibo friends' net Deep analysis 微博用户好友人际关系网络深度分析（人脉深度为二）
4 |
5 | ### Summary - 概要
6 | - 你只需一个账号的授权链接、输入所要研究的对象。
7 |
8 | - 即可得到 a）此研究对象的二度人脉 node.csv、edge.csv 文件供数据可视化软件 Gephi 使用。b）数据分析后写入数据库，可以得到更多详细信息，详见 “特性”。c）使用 Tableau 进行详细的数据统计可视化分析。
9 |
10 | - 软件产生的数据在 WFACat_data 文件夹下。
11 |
12 | ### Feature - 特性
13 | 1. 查询：
14 | - 通过微博用户名查某用户基本信息
15 |
16 | - 通过 uid 查某用户基本信息
17 |
18 | - 通过微博用户名查某用户的互关好友列表及其好友信息
19 |
20 | - 通过微博用户名查某一度好友能通过圈内二度好友认识的一度好友
21 |
22 | - 所有一度好友信息
23 |
24 | 2. 统计：
25 | - 总体概况：总人数、一度好友数、圈内二度好友数、二度好友数
26 |
27 | - 一度好友中与其他一度好友互关最多的人（排序）、与圈内二度好友互关最多的人
28 |
29 | - 一度好友 / 圈内二度好友 / 二度好友中认证情况统计
30 |
31 | - 一度好友地理位置统计、性别统计、关注数、粉丝数、状态数、点赞数、微博创建时间、互关好友总数、客户端
32 |
33 | - 圈内二度好友地理位置统计、性别统计、关注数、粉丝数、状态数、点赞数、微博创建时间、客户端
34 |
35 | - 二度好友地理位置统计、性别统计、关注数、粉丝数、状态数、点赞数、微博创建时间、客户端
36 |
37 | - 所有用户微博创建时间统计
38 |
39 | 3. 推测：
40 | - 根据统计的结果做出一些有趣的推测（如统计手机客户端型号等）。
41 |
42 | ## Requirements - 必要条件
43 | - OS：Windows 10
44 | - IDE：PyCharm
45 | - Python 3.7（64 bit）
46 |
47 | ## Usage - 用法
48 | 1. 克隆此仓库使用 IDE 运行，或直接使用 Release 文件夹中打包好的程序。
49 |
50 | 2. 使用流程：
51 | - 使用 `help` 命令查看帮助；
52 | - 使用 `conf` 命令进行配置；
53 | - 使用 `get` 命令获得基本数据；
54 | - 使用 `tocsv` 命令生成需要的文件，供 Gephi 软件数据可视化人际关系网络；
55 | - 使用 `tomysql` 命令将获得的基本数据存入 MySQL；
56 | - 使用 `mysqld` 命令查看深度分析的各种结果信息；
57 | - 使用 `tool` 命令实时查询单个用户信息。
58 | - 使用 Gephi 软件通过之前生成的 csv 文件来数据可视化分析人际关系网络。
59 | - 使用 Tableau 软件来数据可视化分析详细信息。
60 |
61 | 3. 软件使用效果截图：
62 | - 主界面、help：
63 | 
64 |
65 | - 配置界面：
66 | 
67 |
68 | - 获得基本的数据：
69 | 
70 |
71 | - 数据存储到 csv 文件、MySQL 数据库：
72 | 
73 | 
74 |
75 | - 查看深度分析的各种结果信息：
76 | 
77 |
78 | - 实时查询单个用户信息：
79 | 
80 |
81 | 4. 使用 Gephi 进行数据可视化分析：
82 | [使用方法参考我的博文：《用爬虫和 Gephi 研究微博好友二度人脉分析》](https://marlous.github.io/2019/04/06/%E7%94%A8%E7%88%AC%E8%99%AB%E5%92%8C-Gephi-%E7%A0%94%E7%A9%B6%E5%BE%AE%E5%8D%9A%E5%A5%BD%E5%8F%8B%E4%BA%8C%E5%BA%A6%E4%BA%BA%E8%84%89%E5%88%86%E6%9E%90/)
83 | 
84 | 
85 |
86 | 5. 使用 Tableau 进行数据统计可视化分析：
87 | 使用 Tableau 直接连接创建的数据库即可，然后分析。
88 | [使用方法参考我的博文：《用 Tableau 数据可视化分析微博人际关系数据（基于 WFACat 项目）》](https://marlous.github.io/2019/06/08/%E7%94%A8-Tableau-%E6%95%B0%E6%8D%AE%E5%8F%AF%E8%A7%86%E5%8C%96%E5%88%86%E6%9E%90%E5%BE%AE%E5%8D%9A%E4%BA%BA%E9%99%85%E5%85%B3%E7%B3%BB%E6%95%B0%E6%8D%AE%EF%BC%88%E5%9F%BA%E4%BA%8E-WFACat-%E9%A1%B9%E7%9B%AE%EF%BC%89/)
89 | 
90 |
91 | ## Support - 支持
92 | ### Contact - 联系
93 | - By Marlous
94 | - E-mail：Goonecat@foxmail.com
95 |
96 | ## License - 版权信息
97 | WFACat is released under the GPL license. See [LICENSE](https://github.com/Marlous/WFACat/blob/master/LICENSE) for additional details.
--------------------------------------------------------------------------------
/README_img/图1.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Marlous/WFACat/cc717dc0fa9b596507575258ac55a62c03afcb29/README_img/图1.PNG
--------------------------------------------------------------------------------
/README_img/图2.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Marlous/WFACat/cc717dc0fa9b596507575258ac55a62c03afcb29/README_img/图2.PNG
--------------------------------------------------------------------------------
/README_img/图3.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Marlous/WFACat/cc717dc0fa9b596507575258ac55a62c03afcb29/README_img/图3.PNG
--------------------------------------------------------------------------------
/README_img/图4.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Marlous/WFACat/cc717dc0fa9b596507575258ac55a62c03afcb29/README_img/图4.PNG
--------------------------------------------------------------------------------
/README_img/图5.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Marlous/WFACat/cc717dc0fa9b596507575258ac55a62c03afcb29/README_img/图5.PNG
--------------------------------------------------------------------------------
/README_img/图6.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Marlous/WFACat/cc717dc0fa9b596507575258ac55a62c03afcb29/README_img/图6.PNG
--------------------------------------------------------------------------------
/README_img/图7.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Marlous/WFACat/cc717dc0fa9b596507575258ac55a62c03afcb29/README_img/图7.PNG
--------------------------------------------------------------------------------
/README_img/图8-1.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Marlous/WFACat/cc717dc0fa9b596507575258ac55a62c03afcb29/README_img/图8-1.PNG
--------------------------------------------------------------------------------
/README_img/图8.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Marlous/WFACat/cc717dc0fa9b596507575258ac55a62c03afcb29/README_img/图8.PNG
--------------------------------------------------------------------------------
/README_img/图9.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Marlous/WFACat/cc717dc0fa9b596507575258ac55a62c03afcb29/README_img/图9.PNG
--------------------------------------------------------------------------------
/WFACatPro/main.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | """
4 | WFACatPro.main
5 | ~~~~~~~~~~~~~~~~~~~
6 | This is a main module.
7 | """
8 |
9 |
10 | import json
11 | import os
12 |
13 | from modules import version
14 |
15 |
16 | if __name__ == '__main__':
17 | version.print_version_info()
18 | print('You could use \'help\' command to get help!')
19 | print('If appear Python ERROR, please config correct configuration!')
20 | print('Warning! If you want store data to MySQL, you must set MySQL default charset is utf8mb4 first!!!')
21 | print(
22 | 'Change MySQL chartset: find my.ini, change [mysql] and [mysqld], then change to utf8mb4, restart service.')
23 | print('')
24 |
25 | # 创建存放数据的文件夹
26 | if not os.path.exists('./WFACat_data'):
27 | os.makedirs('./WFACat_data')
28 |
29 | """
30 | 第一次运行没有配置文件，会自动初始化一个 config.json
31 | """
32 | if not os.path.isfile('./WFACat_data/config.json'):
33 | with open('./WFACat_data/config.json', 'w', encoding='utf-8') as init_config_info_json_file:
34 | privacy_url_local = ' '
35 | person_name_local = ' '
36 | each_follower_count_local = 200
37 | set_level_local = 2
38 | db_host_local = 'localhost'
39 | db_port_local = '3306'
40 | db_user_local = 'root'
41 | db_user_password_local = '12345'
42 | db_charset_local = 'utf8mb4'
43 |
44 | init_config_info = {
45 | 'PRIVACY_URL': privacy_url_local,
46 | 'PERSON_NAME': person_name_local,
47 | 'EACH_FOLLOWER_COUNT': each_follower_count_local,
48 | 'SET_LEVEL': set_level_local,
49 | 'DB_HOST': db_host_local,
50 | 'DB_PORT': db_port_local,
51 | 'DB_USER': db_user_local,
52 | 'DB_USER_PASSWORD': db_user_password_local,
53 | 'DB_CHARSET': db_charset_local}
54 |
55 | init_config_info_handle = json.dumps(init_config_info)
56 | init_config_info_json_file.write(init_config_info_handle)
57 |
58 | """
59 | 读取配置
60 | """
61 | with open('./WFACat_data/config.json', 'r', encoding='utf-8') as config_file:
62 | config_file_handle = json.loads(config_file.read())
63 | PRIVACY_URL = config_file_handle['PRIVACY_URL']
64 | PERSON_NAME = config_file_handle['PERSON_NAME']
65 |
66 | """
67 | 每次运行，判断是否进行了配置
68 | """
69 | if PRIVACY_URL == ' ':
70 | print('No configuration! Please setting! using command \'conf\' !')
71 | elif PERSON_NAME == ' ':
72 | print('No configuration! Please setting! using command \'conf\' !')
73 |
74 | """
75 | 接受命令并运行对应的模块
76 | """
77 | this_file_path = os.path.abspath(__file__)[:-7]
78 |
79 | while True:
80 | cmd = input('> ')
81 | if cmd == 'help':
82 | os.system("\"" + this_file_path + "modules/help.py\"")
83 | elif cmd == 'conf':
84 | os.system("\"" + this_file_path + "modules/settings.py\"")
85 | elif cmd == 'get':
86 | os.system("\"" + this_file_path + "modules/get_data.py\"")
87 | elif cmd == 'tocsv':
88 | os.system("\"" + this_file_path + "modules/analysis_to_csv.py\"")
89 | elif cmd == 'tomysql':
90 | os.system("\"" + this_file_path + "modules/analysis_to_mysql.py\"")
91 | elif cmd == 'mysqld':
92 | os.system("\"" + this_file_path + "modules/mysql_query.py\"")
93 | elif cmd == 'tool':
94 | os.system("\"" + this_file_path + "modules/tools.py\"")
95 | elif cmd == 'fix':
96 | os.system("\"" + this_file_path + "modules/fixed.py\"")
97 | elif cmd == 'quit':
98 | exit('Bye ~')
99 |
--------------------------------------------------------------------------------
/WFACatPro/modules/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Marlous/WFACat/cc717dc0fa9b596507575258ac55a62c03afcb29/WFACatPro/modules/__init__.py
--------------------------------------------------------------------------------
/WFACatPro/modules/analysis_to_csv.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | """
4 | WFACatPro.analysis_store_data
5 | ~~~~~~~~~~~~~~~~~~~
6 | This is a analysis store data module.
7 | """
8 |
9 |
10 | import csv
11 | import json
12 | import os
13 |
14 | import settings
15 |
16 |
17 | def save_all_person_info_json_file_as_csv():
18 | level_local = 1
19 |
20 | """
21 | 遍历 n 个人脉文件夹
22 | 想要继续遍历下一层，在本层给出下层遍历需要的本层文件列表
23 | """
24 | while level_local <= settings.SET_LEVEL:
25 | next_level = int(level_local) + 1 # 创建下一度人脉文件夹，从文件夹 2 开始创建
26 | next_level_file_path = './WFACat_data/temp/' + str(next_level)
27 | if not os.path.exists(next_level_file_path):
28 | os.makedirs(next_level_file_path)
29 |
30 | level_file_path = './WFACat_data/temp/' + str(level_local) # 某度人脉文件夹
31 | file_list = os.listdir(level_file_path) # 某度文件夹中每个用户文件夹列表
32 |
33 | """
34 | 遍历 n 度文件夹下的每个用户文件夹
35 | """
36 | for user_file_num in range(0, len(file_list)):
37 | user_file_name_path = './WFACat_data/temp/' + str(level_local) + '/' + file_list[user_file_num] # 某用户文件夹
38 | user_file_list = os.listdir(user_file_name_path) # 某用户文件夹中 json 列表
39 |
40 | document_file_name = file_list[user_file_num] # file_name 是存用户好友信息 json 的文件夹名，即其 uid
41 |
42 | """
43 | 遍历每个用户文件夹中的每个以数字命名的 json 文件
44 | """
45 | for json_file_num in range(0, len(user_file_list)):
46 | json_file_name = user_file_name_path + '/' + user_file_list[json_file_num]
47 |
48 | """
49 | 遍历每个 json 文件中的每个用户 uid，将信息写入 csv 文件
50 | """
51 | with open(json_file_name, 'r', encoding='utf-8') as f: # 打开一个 json 文件
52 | # 将 json 文件内容转为字典，注意字典的索引可以是字符串或整数
53 | json_file_to_dict = json.loads(f.read())
54 |
55 | user_num_count = 0
56 | # 开始提取遍历 json 中每个用户的 uid、name
57 | for user_item in json_file_to_dict['users']:
58 | user_uid = json_file_to_dict['users'][user_num_count]['id']
59 | user_name = json_file_to_dict['users'][user_num_count]['name']
60 | user_num_count = user_num_count + 1
61 |
62 | # 判断是否已记录过该用户
63 | if user_uid not in node_list:
64 | with open('./WFACat_data/data/node.csv', 'a', newline='') as f_node:
65 | node_csv_file = csv.writer(f_node)
66 | node_one_data = [] # 将两个变量数据组成列表
67 | node_one_data.append(user_uid)
68 | node_one_data.append(user_name)
69 | node_csv_file.writerow(node_one_data)
70 | node_list.append(user_uid)
71 | print('node: ' + str(user_uid) + ': ' + user_name)
72 |
73 | with open('./WFACat_data/data/edge.csv', 'a', newline='') as f_edge:
74 | edge_csv_file = csv.writer(f_edge)
75 | edge_one_data = []
76 | edge_one_data.append(document_file_name) # 存 json 的文件夹名
77 | edge_one_data.append(user_uid)
78 | edge_one_data.append(level_local)
79 | edge_csv_file.writerow(edge_one_data)
80 |
81 | print('edge: ' + file_list[json_file_num][0:10] +
82 | ' -> ' + str(user_uid) +
83 | '( ' + user_name + ' )')
84 |
85 | level_local = int(level_local) + 1 # 完成第 n 度人脉 json 文件信息写入
86 |
87 | print('')
88 | print('Congratulations! All data analysis completed!')
89 | print('node.csv and edge.csv you had now ~ You can find it in ./WFACat_data/data')
90 | print('Thanks!')
91 |
92 |
93 | if __name__ == '__main__':
94 | print('= Data analysis =')
95 | values = input('Do you want to start or restart data analysis ?[Y/N]')
96 | while values not in ('Y', 'N'):
97 | values = input('Please enter Y or N:')
98 |
99 | if values == 'Y':
100 | if not os.path.exists('./WFACat_data/temp'):
101 | print('ERROR! Please use \' get \' command to get data first!')
102 |
103 | # 记录已经写入 csv 的节点，避免重复写入
104 | global node_list
105 | node_list = []
106 |
107 | if os.path.exists('./WFACat_data/data'):
108 | os.remove('./WFACat_data/data/node.csv')
109 | os.remove('./WFACat_data/data/edge.csv')
110 | else:
111 | # 创建用于存放 csv 文件的 data 目录（存放节点、边文件供 Gephi 数据可视化软件分析）
112 | os.makedirs('./WFACat_data/data')
113 |
114 | """
115 | 创建节点、边的文件给可视化软件分析（初始化文件，接下来会这两个文件中写数据）
116 | """
117 | with open('./WFACat_data/data/node.csv', 'a', newline='') as f:
118 | csv_file = csv.writer(f)
119 | csv_file.writerow(['id', 'label'])
120 |
121 | with open('./WFACat_data/data/edge.csv', 'a', newline='') as f:
122 | csv_file = csv.writer(f)
123 | csv_file.writerow(['source', 'target', 'weight'])
124 |
125 | """
126 | 先将研究对象的节点写入 node.csv
127 | """
128 | with open('./WFACat_data/temp/person.json', 'r', encoding='utf-8') as person_json_file:
129 | json_file_trans = json.loads(person_json_file.read())
130 | person_uid_local = json_file_trans['cards'][1]['card_group'][0]['user']['id']
131 | person_uid = person_uid_local
132 |
133 | with open('./WFACat_data/data/node.csv', 'a', newline='') as f_node:
134 | node_csv_file = csv.writer(f_node)
135 | node_one_data = [] # 将两个变量数据组成列表
136 | node_one_data.append(person_uid)
137 | node_one_data.append(settings.PERSON_NAME)
138 | node_csv_file.writerow(node_one_data)
139 | node_list.append(person_uid)
140 | print('node: ' + str(person_uid) + ': ' + settings.PERSON_NAME)
141 |
142 | """
143 | 遍历保存所有 json 文件中需要的信息到 csv 文件
144 | """
145 | save_all_person_info_json_file_as_csv()
146 |
147 | elif values == 'N':
148 | print('Thanks!')
149 |
--------------------------------------------------------------------------------
/WFACatPro/modules/analysis_to_mysql.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | """
4 | WFACatPro.analysis_to_mysql
5 | ~~~~~~~~~~~~~~~~~~~
6 | This is a analysis to mysql module.
7 | """
8 |
9 |
10 | import json
11 | import os
12 | import re
13 | import warnings
14 |
15 | import pymysql
16 |
17 | import settings
18 |
19 |
20 | def judge_and_create_db(db_name_params):
21 | """
22 | :param db_name_params:
23 | :return: 无返回值。判断是否存在此名字的数据库，存在则询问是否删除，不存在则创建。
24 | """
25 | db_try = pymysql.connect(
26 | host=settings.DB_HOST,
27 | port=int(settings.DB_PORT),
28 | user=settings.DB_USER,
29 | passwd=settings.DB_USER_PASSWORD,
30 | charset=settings.DB_CHARSET)
31 | cur_try = db_try.cursor()
32 |
33 | cur_try.execute('SHOW DATABASES;')
34 | # 这里列表得到的是一个个元组，每个元组由字符串和字符串后一个逗号构成
35 | db_list = list(cur_try.fetchall())
36 |
37 | if (db_name_params,) in db_list:
38 | values = input('Database name exist! Delete this database? [Y/N]')
39 | while values not in ('Y', 'N'):
40 | values = input('Please enter Y or N !')
41 |
42 | if values == 'Y':
43 | cur_try.execute('drop database ' + db_name_params + ';')
44 | print('%s database deleted success!' % db_name_params)
45 | cur_try.execute('create database ' + db_name_params + ';')
46 | print('%s database created success!' % db_name_params)
47 | elif values == 'N':
48 | exit('Noting occured!')
49 |
50 | else:
51 | cur_try.execute('create database ' + db_name_params + ' CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;')
52 | print('%s database created success!' % db_name_params)
53 |
54 | cur_try.close()
55 | db_try.close()
56 |
57 |
58 | def create_db_table(db_name_params):
59 | """
60 | 创建相关的表
61 | """
62 | db = pymysql.connect(
63 | host=settings.DB_HOST,
64 | port=int(settings.DB_PORT),
65 | user=settings.DB_USER,
66 | passwd=settings.DB_USER_PASSWORD,
67 | charset=settings.DB_CHARSET,
68 | db=db_name_params)
69 |
70 | cur = db.cursor()
71 |
72 | cur.execute('USE ' + db_name_params + ';')
73 |
74 | sql_create_peopleinfo_table = "CREATE TABLE " + "`" + db_name_params + "`.`peopleinfo` (" \
75 | "`uid` CHAR(10) NOT NULL," \
76 | "`name` VARCHAR(45) NULL," \
77 | "`rel_me` VARCHAR(5) NULL," \
78 | "`connect_to_my_friends` TEXT NULL," \
79 | "`connect_to_my_friends_count` INT NULL," \
80 | "`connect_to_two_level_friends` TEXT NULL," \
81 | "`connect_to_two_level_friends_count` INT NULL," \
82 | "`province` VARCHAR(5) NULL," \
83 | "`city` VARCHAR(5) NULL," \
84 | "`location` VARCHAR(20) NULL," \
85 | "`description` VARCHAR(1000) NULL," \
86 | "`url` VARCHAR(100) NULL," \
87 | "`profile_image_url` VARCHAR(200) NULL," \
88 | "`profile_url` VARCHAR(45) NULL," \
89 | "`domain` VARCHAR(45) NULL," \
90 | "`gender` VARCHAR(1) NULL," \
91 | "`followers_count` INT NULL," \
92 | "`friends_count` INT NULL," \
93 | "`statuses_count` INT NULL," \
94 | "`video_status_count` INT NULL," \
95 | "`favourites_count` INT NULL," \
96 | "`created_at` VARCHAR(45) NULL," \
97 | "`verified` VARCHAR(10) NULL," \
98 | "`total_number` INT NULL," \
99 | "`status_source` VARCHAR(50) NULL,PRIMARY KEY (`uid`)" \
100 | ");"
101 |
102 | sql_create_mutualinfo_table = "CREATE TABLE " + "`" + db_name_params + "`.`mutualinfo` (" \
103 | "`uid` CHAR(10) NOT NULL," \
104 | "`mutual_follow` TEXT NULL,PRIMARY KEY (`uid`)" \
105 | ");"
106 |
107 | cur.execute(sql_create_peopleinfo_table)
108 | cur.execute(sql_create_mutualinfo_table)
109 | db.commit()
110 |
111 | cur.close()
112 | db.close()
113 |
114 |
115 | def produce_my_friendinfo_dict():
116 | """
117 | :return: 无参数。返回一个字典，字典键是研究对象的 uid，值分别是其互关好友列表
118 | """
119 | """
120 | 遍历研究对象用户文件夹中的每个以数字命名的 json 文件
121 | """
122 | my_friends_info_dict = {}
123 | temp_save_friends_info_list = []
124 |
125 | temp_one_level_list = os.listdir('./WFACat_data/temp/1')
126 | user_file_list = os.listdir('./WFACat_data/temp/1/' + temp_one_level_list[0])
127 |
128 | for json_file_num in range(0, len(user_file_list)):
129 | json_file_name = './WFACat_data/temp/1/' + temp_one_level_list[0] + '/' + user_file_list[json_file_num]
130 |
131 | """
132 | 遍历每个 json 文件中的每个用户 uid，记录进列表，
133 | 然后将列表作为字典值，键为用户文件名 temp_one_level_list[0]，即研究对象的 uid
134 | """
135 | with open(json_file_name, 'r', encoding='utf-8') as f: # 打开一个 json 文件
136 | # 将 json 文件内容转为字典，注意字典的索引可以是字符串或整数
137 | json_file_to_dict = json.loads(f.read())
138 |
139 | user_num_count = 0
140 | # 开始提取遍历 json 中每个用户的 uid
141 | for user_item in json_file_to_dict['users']:
142 | user_uid = json_file_to_dict['users'][user_num_count]['id']
143 | temp_save_friends_info_list.append(str(user_uid)) # 好友 uid 记录进列表
144 | user_num_count = user_num_count + 1
145 |
146 | # 添加进字典，键是用户文件夹名，值是其好友列表
147 | my_friends_info_dict[temp_one_level_list[0]] = temp_save_friends_info_list
148 | del temp_save_friends_info_list
149 |
150 | return my_friends_info_dict
151 |
152 |
153 | def produce_friends_friendinfo_dict():
154 | """
155 | :return: 无参数。返回一个字典。研究对象的好友的好友信息写入字典。遍历 2 度人脉文件夹中所有 json 文件
156 | """
157 | everybody_friends_info_dict = {}
158 |
159 | """
160 | 遍历 2 度文件夹下的每个用户文件夹
161 | 键为某用户文件夹名 document_file_name，值为其遍历的 json 文件中 uid 组成的列表 temp_save_friends_info_list
162 | """
163 | file_list = os.listdir('./WFACat_data/temp/2/')
164 |
165 | for user_file_num in range(0, len(file_list)):
166 | user_file_name_path = './WFACat_data/temp/2/' + file_list[user_file_num] # 某用户文件夹
167 | user_file_list = os.listdir(user_file_name_path) # 某用户文件夹中 json 列表
168 |
169 | document_file_name = file_list[user_file_num] # file_name 是存用户好友信息 json 的文件夹名，即其 uid
170 |
171 | temp_save_friends_info_list = []
172 |
173 | """
174 | 遍历每个用户文件夹中的每个以数字命名的 json 文件
175 | """
176 | for json_file_num in range(0, len(user_file_list)):
177 | json_file_name = user_file_name_path + '/' + user_file_list[json_file_num]
178 |
179 | """
180 | 遍历每个 json 文件中的每个用户 uid
181 | """
182 | with open(json_file_name, 'r', encoding='utf-8') as f: # 打开一个 json 文件
183 | # 将 json 文件内容转为字典，注意字典的索引可以是字符串或整数
184 | json_file_to_dict = json.loads(f.read())
185 |
186 | user_num_count = 0
187 | # 开始提取遍历 json 中每个用户的 uid、name
188 | for user_item in json_file_to_dict['users']:
189 | user_uid = json_file_to_dict['users'][user_num_count]['id']
190 | temp_save_friends_info_list.append(str(user_uid)) # 好友 uid 记录进列表
191 | user_num_count = user_num_count + 1
192 |
193 | # 添加进字典，键是用户文件夹名，值是其好友列表
194 | everybody_friends_info_dict[document_file_name] = temp_save_friends_info_list
195 | del temp_save_friends_info_list
196 |
197 | return everybody_friends_info_dict
198 |
199 |
200 | def write_all_person_info_in_mysql(db_name_params):
201 | db = pymysql.connect(
202 | host=settings.DB_HOST,
203 | port=int(settings.DB_PORT),
204 | user=settings.DB_USER,
205 | passwd=settings.DB_USER_PASSWORD,
206 | charset=settings.DB_CHARSET,
207 | db=db_name_params)
208 |
209 | cur = db.cursor()
210 | cur.execute('USE ' + db_name_params + ';')
211 |
212 | sum_write_count = 0
213 |
214 | person_writed_list = []
215 | # 因为自己的信息不需要写入数据库。研究对象（自己）的 uid 是 ./WFACat_data/temp/1/XXX 其中的文件夹名 XXX
216 | person_uid = os.listdir('./WFACat_data/temp/1')[0]
217 | person_writed_list.append(person_uid)
218 |
219 | """
220 | 遍历 n 度（n 个）人脉文件夹中所有。此处只遍历 2 次，遍历文件夹 1、2
221 | 想要继续遍历下一层，在本层给出下层遍历需要的本层文件列表
222 | """
223 | level_local = 1
224 |
225 | while level_local <= settings.SET_LEVEL:
226 | next_level = int(level_local) + 1 # 创建下一度人脉文件夹，从文件夹 2 开始创建
227 | next_level_file_path = './WFACat_data/temp/' + str(next_level)
228 | if not os.path.exists(next_level_file_path):
229 | os.makedirs(next_level_file_path)
230 |
231 | level_file_path = './WFACat_data/temp/' + str(level_local) # 某度人脉文件夹
232 | file_list = os.listdir(level_file_path) # 某度文件夹中每个用户文件夹列表
233 |
234 | """
235 | 遍历 n 度文件夹下的每个用户文件夹
236 | """
237 | for user_file_num in range(0, len(file_list)):
238 | user_file_name_path = './WFACat_data/temp/' + str(level_local) + '/' + file_list[user_file_num] # 某用户文件夹
239 | user_file_list = os.listdir(user_file_name_path) # 某用户文件夹中 json 列表
240 |
241 | document_name = file_list[user_file_num]
242 |
243 | """
244 | 遍历每个用户文件夹中的每个以数字命名的 json 文件
245 | """
246 | for json_file_num in range(0, len(user_file_list)):
247 | json_file_name = user_file_name_path + '/' + user_file_list[json_file_num]
248 |
249 | """
250 | 遍历每个 json 文件中的每个用户 uid 等详细信息写入数据库
251 | """
252 | with open(json_file_name, 'r', encoding='utf-8') as f: # 打开一个 json 文件
253 | # 将 json 文件内容转为字典，注意字典的索引可以是字符串或整数
254 | json_file_to_dict = json.loads(f.read())
255 |
256 | user_num = 0
257 | # 开始提取遍历 json 中每个用户的 uid 等详细信息写入数据库
258 | for user_item in json_file_to_dict['users']:
259 | # 提取用户的 uid 等信息
260 | uid = json_file_to_dict['users'][user_num]['id']
261 | name = json_file_to_dict['users'][user_num]['name']
262 | province = json_file_to_dict['users'][user_num]['province']
263 | city = json_file_to_dict['users'][user_num]['city']
264 | location = json_file_to_dict['users'][user_num]['location']
265 | temp_description = json_file_to_dict['users'][user_num]['description']
266 |
267 | # 防止有人在描述里写 SQL 语句导致无法将其写入数据库
268 | description = str(temp_description).replace("\\", ",").replace('\"', '\\"').replace("\'", "\\'")
269 |
270 | url = json_file_to_dict['users'][user_num]['url']
271 | profile_image_url = json_file_to_dict['users'][user_num]['profile_image_url']
272 | profile_url = json_file_to_dict['users'][user_num]['profile_url']
273 | domain = json_file_to_dict['users'][user_num]['domain']
274 | gender = json_file_to_dict['users'][user_num]['gender']
275 | followers_count = json_file_to_dict['users'][user_num]['followers_count']
276 | friends_count = json_file_to_dict['users'][user_num]['friends_count']
277 | statuses_count = json_file_to_dict['users'][user_num]['statuses_count']
278 | video_status_count = json_file_to_dict['users'][user_num]['video_status_count']
279 | favourites_count = json_file_to_dict['users'][user_num]['favourites_count']
280 | created_at = json_file_to_dict['users'][user_num]['created_at']
281 | verified = json_file_to_dict['users'][user_num]['verified']
282 |
283 | """
284 | 存入 status_source 属性值
285 | 判断 status 这个字段书否存在（debug 时显示不存在），说明可能不存在
286 | """
287 | if 'status' in json_file_to_dict['users'][user_num]:
288 | if 'source' in json_file_to_dict['users'][user_num]['status']:
289 | temp_status_source = json_file_to_dict['users'][user_num]['status']['source']
290 | # 正则表达式提取出 标签中的手机客户端型号
291 | pattern_html = re.compile(">(.*?)<")
292 | pattern_txt = re.compile("[a-zA-Z0-9\u4e00-\u9fa5]+")
293 | # 先把 json 中 html 标签中的内容匹配出来，放进 temp_two_status_source（是一个列表）
294 | temp_two_status_source = pattern_html.findall(str(temp_status_source))
295 |
296 | if list(temp_two_status_source):
297 | # json 中取出的匹配出变成列表。此处（不为空的话）将其再次匹配，只要数字、字母、中文。过滤掉特殊字符
298 | temp_three_status_source = temp_two_status_source[0]
299 | # 将再次匹配得到的列表连接起来，变成字符串
300 | status_source = " ".join(pattern_txt.findall(str(temp_three_status_source)))
301 |
302 | # 写入数据库时，输出 log 部分信息
303 | print('log: Uid: ' + str(uid) + ' Name: ' + str(name) +
304 | ' Description:' + str(description) + ' Client: ' + str(status_source))
305 |
306 | cur.execute("UPDATE " + db_name_params + ".peopleinfo SET status_source = '" +
307 | str(status_source) + "' WHERE uid = '" + str(uid) + "';")
308 |
309 | """
310 | 存入 verified 属性值
311 | """
312 | if str(verified) != 'False':
313 | cur.execute("UPDATE " + db_name_params + ".peopleinfo SET verified = '" + str(verified) +
314 | "' WHERE uid = '" + str(uid) + "';")
315 |
316 | """
317 | 将提取的信息剩下的属性值写入数据库
318 | """
319 | sql_write = "UPDATE " + db_name_params + ".peopleinfo SET name = '" + str(name) + \
320 | "', province = '" + str(province) + \
321 | "', city = '" + str(city) + \
322 | "', location = '" + str(location) + \
323 | "', description = \"" + str(description) + \
324 | "\", url = '" + str(url) + \
325 | "', profile_image_url = '" + str(profile_image_url) + \
326 | "', profile_url = '" + str(profile_url) + \
327 | "', domain = '" + str(domain) + \
328 | "', gender = '" + str(gender) + \
329 | "', followers_count = '" + str(followers_count) + \
330 | "', friends_count = '" + str(friends_count) + \
331 | "', statuses_count = '" + str(statuses_count) + \
332 | "', video_status_count = '" + str(video_status_count) + \
333 | "', favourites_count = '" + str(favourites_count) + \
334 | "', created_at = '" + str(created_at) + \
335 | "' WHERE uid = '" + str(uid) + "';"
336 |
337 | # 记录下此用户的详细信息已经写入到数据库
338 | if str(uid) not in person_writed_list:
339 | cur.execute(sql_write)
340 | db.commit()
341 | person_writed_list.append(str(uid))
342 | sum_write_count = sum_write_count + 1
343 |
344 | # json 中的下一个用户
345 | user_num = user_num + 1
346 |
347 | """
348 | 写入此好友（一度好友）互关的好友数
349 | """
350 | if level_local == 2:
351 | total_number = json_file_to_dict['total_number']
352 | cur.execute(
353 | "UPDATE " + db_name_params + ".peopleinfo SET total_number = '" + str(total_number) +
354 | "' WHERE uid = '" + document_name + "';")
355 | db.commit()
356 |
357 | level_local = int(level_local) + 1 # 完成第 n 度人脉 json 文件信息写入
358 |
359 | cur.close()
360 | db.close()
361 | print('Sum write count: ' + str(sum_write_count))
362 |
363 |
364 | if __name__ == '__main__':
365 | warnings.filterwarnings('ignore')
366 |
367 | print('= MySQL analysis =')
368 | print('Make sure mysql service started! Then input database name ~')
369 | DB_NAME = input(
370 | 'Enter database name (weibo user who you want to analysis):')
371 |
372 | judge_and_create_db(DB_NAME)
373 | create_db_table(DB_NAME)
374 |
375 | print('Analysizing, wait...')
376 |
377 | """
378 | 对数据处理，得到两个基本集合，其它结论通过这两个基本集合计算得到
379 | """
380 | print('log: basic data...')
381 | my_friendinfo_dict = produce_my_friendinfo_dict()
382 | friends_friendinfo_dict = produce_friends_friendinfo_dict()
383 |
384 | """
385 | 将每个好友的好友信息列表取出放进一个列表
386 | """
387 | friends_friendinfo_dict_values_list = []
388 | for value in friends_friendinfo_dict.values(): # 遍历字典中的值（多个列表）
389 | friends_friendinfo_dict_values_list.append(value)
390 |
391 | """
392 | 每个好友信息列表逐个取交集（两两取交集）。结果减去自己和自己的好友（一度好友）即是除度为 1 的二度好友
393 | 得到第三个基本集合
394 | 总算法复杂度（比较次数）为排列组合算出来的次数 C n 2 次，n 是一度好友数量
395 | """
396 | print('log: two level friends...')
397 | file_list = os.listdir('./WFACat_data/temp/1')
398 | my_uid = file_list[0]
399 | temp_two_level_useful_friends_list = []
400 |
401 | count = 1
402 | for person in my_friendinfo_dict[my_uid][0:-1]:
403 | for temp_person in my_friendinfo_dict[my_uid][count:]:
404 | temp_set = set(friends_friendinfo_dict[person]) & set(friends_friendinfo_dict[temp_person])
405 | for item in list(temp_set):
406 | temp_two_level_useful_friends_list.append(item)
407 | count = count + 1
408 |
409 | # 用元组去掉重复的
410 | two_level_useful_friends_list = list(set(temp_two_level_useful_friends_list))
411 | two_level_useful_friends_list.remove(my_uid)
412 | # 注意，如果某个一度好友与自己的朋友圈无任何关系，那么他就已经不在其中，不需要再移出一次
413 | for item in my_friendinfo_dict[my_uid]:
414 | if item in two_level_useful_friends_list:
415 | two_level_useful_friends_list.remove(item)
416 |
417 | """
418 | 连接进入数据库
419 | """
420 | print('log: connecting to mysql...')
421 | db = pymysql.connect(
422 | host=settings.DB_HOST,
423 | port=int(settings.DB_PORT),
424 | user=settings.DB_USER,
425 | passwd=settings.DB_USER_PASSWORD,
426 | charset=settings.DB_CHARSET,
427 | db=DB_NAME)
428 |
429 | cur = db.cursor()
430 | cur.execute('USE ' + DB_NAME + ';')
431 |
432 | """
433 | 写入每个好友（一度、二度，全部）uid 信息
434 | """
435 | print('log: every user uid to mysql, may be long time, wait...')
436 | cur.execute('USE ' + DB_NAME + ';')
437 |
438 | uid_saved = []
439 | # 不记录自己
440 | uid_saved.append(my_uid)
441 |
442 | for item in my_friendinfo_dict[my_uid]:
443 | # 一度人脉，标记为 1
444 | cur.execute("INSERT INTO " + DB_NAME +
445 | ".peopleinfo (uid, rel_me) VALUES ('" + item + "', '1');")
446 | db.commit()
447 | uid_saved.append(item)
448 |
449 | for data in friends_friendinfo_dict:
450 | for item in friends_friendinfo_dict[data]:
451 | if item not in uid_saved:
452 | if item in two_level_useful_friends_list:
453 | # 节点度不为 1 的二度人脉，标记为 2
454 | cur.execute("INSERT INTO " + DB_NAME +
455 | ".peopleinfo (uid, rel_me) VALUES ('" + item + "', '2');")
456 | db.commit()
457 | uid_saved.append(item)
458 | else:
459 | # 节点度为 1 的二度人脉，标记为 2.1
460 | cur.execute("INSERT INTO " + DB_NAME +
461 | ".peopleinfo (uid, rel_me) VALUES ('" + item + "', '2.1');")
462 | db.commit()
463 | uid_saved.append(item)
464 |
465 | """
466 | 将每个一度好友的互关列表写入数据库 mutual_follow
467 | """
468 | print('log: mutual_follow...')
469 | for key in friends_friendinfo_dict:
470 | value = ", ".join(friends_friendinfo_dict[key])
471 |
472 | cur.execute("INSERT INTO " + DB_NAME +
473 | ".mutualinfo (uid, mutual_follow) VALUES ('" + str(key) + "', '" + str(value) + "');")
474 | db.commit()
475 |
476 | """
477 | 计算每个一度好友 connect_to_my_friends 写入数据库
478 | 某个好友的好友信息列表与自己的信息列表取交集
479 | """
480 | print('log: connect_to_my_friends...')
481 | for data in my_friendinfo_dict[my_uid]:
482 | # 好友的好友列表和我的好友列表取交集
483 | connect_to_my_friends = set(
484 | friends_friendinfo_dict[data]) & set(
485 | my_friendinfo_dict[my_uid])
486 | temp_content = ", ".join(list(connect_to_my_friends))
487 |
488 | cur.execute(
489 | "UPDATE " + DB_NAME +
490 | ".peopleinfo SET connect_to_my_friends = '" + temp_content +
491 | "', connect_to_my_friends_count = '" + str(len(list(connect_to_my_friends))) +
492 | "' WHERE uid = '" + data + "';")
493 | db.commit()
494 | del temp_content
495 |
496 | """
497 | 计算每个一度好友 connect_to_two_level_friends 写入数据库
498 | 某个好友信息列表与 two_level_useful_friends_list 集合取交集
499 | two_level_useful_friends_list 是所有除去节点度为一的二度好友
500 | """
501 | print('log: connect_to_two_level_friends...')
502 | # 初始化一个字典，记下每个一度好友的二度好友列表（除去节点的度为一的）
503 | friends_two_level_friendinfo_dict = {}
504 |
505 | for data in my_friendinfo_dict[my_uid]:
506 | # 好友的好友列表和二度人脉 two_level_useful_friends_list 取交集
507 | connect_to_two_level_friends = set(
508 | friends_friendinfo_dict[data]) & set(
509 | two_level_useful_friends_list)
510 | temp_content = ", ".join(list(connect_to_two_level_friends))
511 | # 记下每个一度好友的二度好友列表（除去节点的度为一的）
512 | friends_two_level_friendinfo_dict[data] = list(
513 | connect_to_two_level_friends)
514 |
515 | cur.execute(
516 | "UPDATE " + DB_NAME +
517 | ".peopleinfo SET connect_to_two_level_friends = '" + temp_content +
518 | "', connect_to_two_level_friends_count = '" + str(len(list(connect_to_two_level_friends))) +
519 | "' WHERE uid = '" + data + "';")
520 | db.commit()
521 | del temp_content
522 |
523 | """
524 | 计算每个一度好友可以间接认识哪些其它一度好友，为其建立一张表，表属性分别是可以间接认识的人、通过谁，写入数据库
525 | 每个好友的二度人脉好友信息列表（通过字典查到）与其它好友的二度人脉好友信息列表取交集
526 | 集合不为空，表示可以通过集合里的人建立联系。
527 | 总算法复杂度（比较次数）为排列组合算出来的次数 C n 2 次，n 是一度好友数量
528 | """
529 | print('log: maybe_connect_to_friends...')
530 | count = 1
531 | for person in my_friendinfo_dict[my_uid][0:-1]:
532 | for temp_person in my_friendinfo_dict[my_uid][count:]:
533 | two_level_friends_inter_have = set(
534 | friends_two_level_friendinfo_dict[person]) & set(
535 | friends_two_level_friendinfo_dict[temp_person])
536 |
537 | if two_level_friends_inter_have:
538 | cur.execute(
539 | "CREATE TABLE IF NOT EXISTS " + DB_NAME +
540 | ".u" + person +
541 | " ( `uid` CHAR(10) PRIMARY KEY, `by_friends` VARCHAR(5000) NULL, `by_friends_count` INT NULL" + ");"
542 | )
543 | db.commit()
544 |
545 | cur.execute(
546 | "CREATE TABLE IF NOT EXISTS " + DB_NAME +
547 | ".u" + temp_person +
548 | " ( `uid` CHAR(10) PRIMARY KEY, `by_friends` VARCHAR(5000) NULL, `by_friends_count` INT NULL" + ");"
549 | )
550 | db.commit()
551 |
552 | temp_content = ", ".join(list(two_level_friends_inter_have))
553 | cur.execute("INSERT INTO " + DB_NAME +
554 | ".u" + person +
555 | " VALUES ('" + temp_person + "', '" + temp_content +
556 | "', '" + str(len(two_level_friends_inter_have)) + "');")
557 | db.commit()
558 |
559 | cur.execute("INSERT INTO " + DB_NAME +
560 | ".u" + temp_person +
561 | " VALUES ('" + person + "', '" + temp_content +
562 | "', '" + str(len(two_level_friends_inter_have)) + "');")
563 | db.commit()
564 | del temp_content
565 |
566 | count = count + 1
567 |
568 | del my_friendinfo_dict
569 | del friends_friendinfo_dict
570 | del two_level_useful_friends_list
571 | cur.close()
572 | db.close()
573 |
574 | """
575 | 写入所有人的详细信息，除了上面计算的三个属性（因为已经写入）
576 | """
577 | print('log: every user info detail, may be long time, wait...')
578 | write_all_person_info_in_mysql(DB_NAME)
579 | print('Congratulations! All task completed!')
580 | print('You could use \'detail\' to get more info about your weibo user!')
581 |
582 | """
583 | 修复 bug，详见 fixed 模块
584 | """
585 | this_file_path = os.path.abspath(__file__)[:-20]
586 | os.system("\"" + this_file_path + "fixed.py\"")
587 |
--------------------------------------------------------------------------------
/WFACatPro/modules/fixed.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | """
4 | WFACatPro.fixed
5 | ~~~~~~~~~~~~~~~~~~~
6 | This is a fixed module.
7 | """
8 |
9 |
10 | import pymysql
11 |
12 | import settings
13 |
14 |
15 | # 判断表是否存在
16 | def table_exists(cur_param, table_name_param):
17 | sql = "show tables;"
18 | cur_param.execute(sql)
19 | tables = str(cur_param.fetchall())
20 | if table_name_param in tables:
21 | return True #存在返回1
22 | else:
23 | return False #不存在返回0
24 |
25 |
26 | if __name__ == '__main__':
27 | DB_NAME = input("Enter database name that your created to fixed bug ：p")
28 | print("fixing !")
29 | db = pymysql.connect(
30 | host=settings.DB_HOST,
31 | port=int(settings.DB_PORT),
32 | user=settings.DB_USER,
33 | passwd=settings.DB_USER_PASSWORD,
34 | charset=settings.DB_CHARSET,
35 | db=DB_NAME)
36 |
37 | cur = db.cursor()
38 |
39 | """
40 | debug
41 | 某个一度好友可能认识的另外一度好友原理：
42 | 其中用他们的圈内二度好友列表（对研究对象来说的）取交集，
43 | 交集部分连接的两个节点，一个节点是主节点，另一个节点是可能认识的另外的一度好友（对研究对象来说的），
44 |
45 | bug 原因：交集部分连接的两个节点，这两个节点不一定只通过交集部分连接，他们两个自己也可能能直接连接。
46 | 解决：删掉可能认识的一度好友中已经互关的一度人脉。删掉 uxxxxxx 表中 uid 存在于他的 connect_to_my_friends 中的
47 | """
48 | # 遍历研究对象一度人脉
49 | cur.execute("SELECT uid, connect_to_my_friends FROM %s.peopleinfo WHERE rel_me = \'1\' " % (DB_NAME))
50 | result = cur.fetchall()
51 |
52 | for row in result:
53 | connect_to_my_friends_str = str(row[1])
54 | table_name = str(row[0])
55 |
56 | if table_exists(cur, table_name):
57 | cur.execute("SELECT uid FROM %s.u%s" % (DB_NAME, str(row[0])))
58 | u_table_result = cur.fetchall()
59 | for u_table_row in u_table_result:
60 | # 删掉 uxxxxx 表（可能认识的人表）中 uid 存在于他（peopleinfo 表）的 connect_to_my_friends 中的条目
61 | if str(u_table_row[0]) in connect_to_my_friends_str:
62 | cur.execute("DELETE FROM %s.u%s WHERE uid = \'%s\'" % (DB_NAME, str(row[0]), str(u_table_row[0])))
63 | db.commit()
64 |
65 | print("completed !")
--------------------------------------------------------------------------------
/WFACatPro/modules/get_data.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | """
4 | WFACatPro.get_data
5 | ~~~~~~~~~~~~~~~~~~~
6 | This is a get_data module.
7 | """
8 |
9 |
10 | import json
11 | import os
12 | import random
13 | import shutil
14 | import time
15 | import urllib
16 | from urllib.parse import urlparse
17 |
18 | import requests
19 |
20 | import settings
21 |
22 |
23 | # 创建一个列表，用来保存已经下载过的人的好友信息
24 | global uid_json_file_downloaded
25 |
26 |
27 | def get_person_uid(person_name_params):
28 | """
29 | :param: 某个微博用户的用户名
30 | :return: 返回微博用户对应的 uid
31 | """
32 | """
33 | 下载并保存自己的好友信息 person.json 文件（一度人脉）
34 | """
35 | person_info_json_file = requests.get(
36 | get_person_info_json_file_url(person_name_params))
37 | if person_info_json_file.status_code == requests.codes.ok:
38 | file_size = (len(person_info_json_file.content) / 1024) # 计算文件大小并输出
39 | print('%s %.1f KB downloaded OK!' % ('person.json', file_size))
40 | else:
41 | person_info_json_file.raise_for_status()
42 |
43 | with open('./WFACat_data/temp/person.json', 'wb') as person_json_file_saved: # 保存下载的文件到硬盘
44 | person_json_file_saved.write(person_info_json_file.content)
45 | person_json_file_saved.close()
46 |
47 | """
48 | 提取 person 的 uid
49 | """
50 | with open('./WFACat_data/temp/person.json', 'r', encoding='utf-8') as person_json_file:
51 | json_file_trans = json.loads(
52 | person_json_file.read()) # 将 json 文件内容转为字典
53 | # 注意字典的索引可以是字符串或整数
54 | person_uid_local = json_file_trans['cards'][1]['card_group'][0]['user']['id']
55 | person_json_file.close()
56 |
57 | return person_uid_local
58 |
59 |
60 | def get_person_info_json_file_url(person_name_params):
61 | """
62 | :param: 某个微博用户名称，在 settings 中的 person_name
63 | :return: 返回获取某个用户信息 json 文件的下载 url
64 | """
65 |
66 | # 需要研究的用户微博名字，经过两次编码，以便能在参数中使用
67 | temp_person_name = urllib.parse.quote(person_name_params)
68 | encoded_person_name = urllib.parse.quote(temp_person_name)
69 |
70 | # 拼接出 url
71 | url_local = 'https://api.weibo.cn/2/cardlist?' + \
72 | 'aid=' + str(params['aid'])[2:-2] + \
73 | '&c=' + str(params['c'])[2:-2] + \
74 | '&containerid=100103type%253D3%2526q%253D' + encoded_person_name + \
75 | '&count=10' + \
76 | '&from=' + str(params['from'])[2:-2] + \
77 | '&gsid=' + str(params['gsid'])[2:-2] + \
78 | '&i=' + str(params['i'])[2:-2] + \
79 | '&lang=' + str(params['lang'])[2:-2] + \
80 | '&page=' + str(params['page'])[2:-2] + \
81 | '&s=' + str(params['s'])[2:-2] + \
82 | '&ua=' + str(params['ua'])[2:-2] + \
83 | '&v_p=' + str(params['v_p'])[2:-2]
84 |
85 | return url_local
86 |
87 |
88 | def get_data_url(uid_params, page_params):
89 | """
90 | :param uid_params: 某个微博用户的 uid
91 | :param page_params: 页数。通过循环增加页数，下载到全部互关好友 json 文件
92 | :return: 返回获取某个用户的互关好友列表 json 文件的下载 url
93 | """
94 |
95 | params['count'] = settings.EACH_FOLLOWER_COUNT
96 | params['uid'] = uid_params
97 | params['page'] = page_params
98 |
99 | # 拼接出 url
100 | url_local = 'https://api.weibo.cn/2/friendships/bilateral?' + \
101 | 'aid=' + str(params['aid'])[2:-2] + \
102 | '&c=' + str(params['c'])[2:-2] + \
103 | '&count=' + str(params['count']) + \
104 | '&from=' + str(params['from'])[2:-2] + \
105 | '&gsid=' + str(params['gsid'])[2:-2] + \
106 | '&i=' + str(params['i'])[2:-2] + \
107 | '&lang=' + str(params['lang'])[2:-2] + \
108 | '&page=' + str(params['page']) + \
109 | '&real_relationships=' + str(params['real_relationships'])[2:-2] + \
110 | '&s=' + str(params['s'])[2:-2] + \
111 | '&trim_status' + str(params['trim_status'])[2:-2] + \
112 | '&ua=' + str(params['ua'])[2:-2] + \
113 | '&uid=' + str(params['uid']) + \
114 | '&v_p=' + str(params['v_p'])[2:-2]
115 | return url_local
116 |
117 |
118 | def download_user_json_file(user_uid_params, user_json_file_saved_path_params):
119 | """
120 | :param user_uid_params: 某一个用户的 uid
121 | :param user_json_file_saved_path_params: 将下载的此用户好友所有 json 文件保存的位置
122 | :return: 无返回值
123 | """
124 | user_uid = user_uid_params
125 | user_json_file_saved_path = user_json_file_saved_path_params
126 |
127 | page_count = 1
128 | while True:
129 | friends_info_json_file = requests.get(
130 | get_data_url(user_uid, page_count)
131 | )
132 |
133 | if friends_info_json_file.status_code == requests.codes.ok:
134 | file_size = (len(friends_info_json_file.content) / 1024) # 计算文件大小并输出
135 | print('%s/%s.json %.1f KB downloaded OK!' % (
136 | user_json_file_saved_path, str(page_count), file_size))
137 | else:
138 | friends_info_json_file.raise_for_status()
139 |
140 | user_json_file_name = user_json_file_saved_path + '/' + str(page_count) + '.json'
141 |
142 | with open(user_json_file_name, 'wb') as f: # 保存下载的文件到硬盘
143 | f.write(friends_info_json_file.content)
144 |
145 | with open(user_json_file_name, 'r', encoding='utf8') as f: # 如果不含有用户信息，则这个 json 文件是最后一个不要的
146 | json_file_to_dict = json.loads(f.read())
147 |
148 | if json_file_to_dict['users']: # 判断如果 users 下有用户信息则下载下一页，没有就跳出不下载下一页的 json 文件
149 | page_count = page_count + 1
150 | time.sleep(random.randint(1, 3)) # 随机停几秒，请求下一页的 json 文件
151 | continue
152 | else:
153 | break
154 |
155 | user_json_file_name_last = user_json_file_saved_path + '/' + str(page_count) + '.json'
156 | os.remove(user_json_file_name_last) # 删除最后一个下载的用户信息为空的 json 文件
157 | print(user_json_file_name_last + ' deleted ~')
158 |
159 | # 记录此用户的好友信息 json 文件已下载
160 | uid_json_file_downloaded.append(user_uid)
161 |
162 |
163 | def download_over_one_level_user_json_file():
164 | """
165 | :return: 无返回值。下载 n 度人脉全部用户的好友信息的 json 文件（默认一度人脉的信息文件已下载好）
166 | """
167 | if not os.path.exists('./WFACat_data/temp/1'):
168 | print('ERROR! ./WFACat_data/temp/1 not exists!')
169 |
170 | # 初始化人脉深度为 1，因为获得第一层人脉信息（以自己为种子）是预先下载，不使用此下载函数下载
171 | level_local = 1
172 |
173 | """
174 | 遍历 n 个人脉文件夹
175 | 想要继续遍历下一层，在本层给出下层遍历需要的本层文件列表
176 | """
177 | while level_local < settings.SET_LEVEL:
178 | next_level = int(level_local) + 1 # 创建下一度人脉文件夹，从文件夹 2 开始创建
179 | next_level_file_path = './WFACat_data/temp/' + str(next_level)
180 | if not os.path.exists(next_level_file_path):
181 | os.makedirs(next_level_file_path)
182 |
183 | level_file_path = './WFACat_data/temp/' + str(level_local) # 某度人脉文件夹
184 | file_list = os.listdir(level_file_path) # 某度文件夹中每个用户文件夹列表
185 |
186 | """
187 | 遍历 n 度文件夹下的每个用户文件夹
188 | """
189 | for user_file_num in range(0, len(file_list)):
190 | user_file_name_path = './WFACat_data/temp/' + str(level_local) + '/' + file_list[user_file_num] # 某用户文件夹
191 | user_file_list = os.listdir(user_file_name_path) # 某用户文件夹中 json 列表
192 |
193 | """
194 | 遍历每个用户文件夹中的每个以数字命名的 json 文件
195 | """
196 | for json_file_num in range(0, len(user_file_list)):
197 | json_file_name = user_file_name_path + '/' + user_file_list[json_file_num]
198 |
199 | """
200 | 遍历每个 json 文件中的每个用户 uid，并下载其好友信息的 json 文件
201 | """
202 | with open(json_file_name, 'r', encoding='utf-8') as f: # 打开一个 json 文件
203 | # 将 json 文件内容转为字典，注意字典的索引可以是字符串或整数
204 | json_file_to_dict = json.loads(f.read())
205 |
206 | user_num_count = 0
207 | # 开始提取遍历 json 中每个用户的 uid
208 | for user_item in json_file_to_dict['users']:
209 | user_uid = json_file_to_dict['users'][user_num_count]['id']
210 | user_num_count = user_num_count + 1
211 |
212 | # 判断是否已下载过该用户好友 json 文件
213 | if user_uid not in uid_json_file_downloaded:
214 | time.sleep(random.randint(5, settings.TIME_PAUSE_MAX)) # 随机暂停几秒，数字越大越安全
215 | # 拼好用于存放其好友 json 文件的文件夹，以其 uid 命名
216 | user_json_file_saved_path = next_level_file_path + '/' + str(user_uid)
217 | os.makedirs(user_json_file_saved_path)
218 |
219 | download_user_json_file(user_uid, user_json_file_saved_path)
220 |
221 | level_local = int(level_local) + 1 # 完成第 n 度人脉 json 文件下载
222 |
223 |
224 | if __name__ == '__main__':
225 | # 创建一个列表，用来保存已经下载过的人的好友信息
226 | uid_json_file_downloaded = []
227 |
228 | print('= get data =')
229 |
230 | # 拆解设置的 privacy_url
231 | parsed = urlparse(settings.PRIVACY_URL)
232 | # 拆解请求参数
233 | params = urllib.parse.parse_qs(parsed.query)
234 |
235 | # 创建存放每个以用户 uid 命名的文件夹
236 | if os.path.exists('./WFACat_data/temp'):
237 | shutil.rmtree('./WFACat_data/temp')
238 | os.makedirs('./WFACat_data/temp')
239 | else:
240 | os.makedirs('./WFACat_data/temp')
241 |
242 | # 获取到 settings 中 person_name 微博用户对应的 uid
243 | person_uid = get_person_uid(settings.PERSON_NAME)
244 | print("%s\'s uid: %s" % (settings.PERSON_NAME, str(person_uid)))
245 |
246 | """
247 | 根据 settings 中 person_name 微博用户对应的 uid
248 | 下载并保存一度人脉信息的 json 文件。
249 | 文件夹 1 中以其 uid 命名的文件夹，其中以数字 1 开始，若干个 json 文件（互关好友数多需要请求几个 page）
250 | """
251 | if not os.path.exists('./WFACat_data/temp/1'):
252 | os.makedirs('./WFACat_data/temp/1')
253 |
254 | # 创建研究对象用户文件夹，用于存放互关好友信息的 json 文件（json 以数字命名）
255 | user_json_file_saved_path = './WFACat_data/temp/1/' + str(person_uid)
256 | os.makedirs(user_json_file_saved_path)
257 |
258 | if person_uid not in uid_json_file_downloaded:
259 | download_user_json_file(person_uid, user_json_file_saved_path)
260 |
261 | # 下载 n 度人脉全部用户的好友信息的 json 文件，一度人脉的文件已下载好
262 | download_over_one_level_user_json_file()
263 |
--------------------------------------------------------------------------------
/WFACatPro/modules/help.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | """
4 | WFACatPro.help
5 | ~~~~~~~~~~~~~~~~~~~
6 | This is a help module.
7 | """
8 |
9 |
10 | def print_help_info():
11 | print('= User manual =')
12 | print('You could analysis your weibo friends\' network with Gephi and MySQL software.')
13 | print('Using this python programme to produce csv file, then use Gephi. And you could only use MySQL module or Tools.')
14 | print('Make sure run get command to get basic data first !')
15 | print('- csv: First, get data. Second, analysing data and produce csv file. ')
16 | print('- MySQL: First, get data. Second, analysing data and store to mysql. ')
17 | print('- Tools: Many tools to get info about weibo user. ')
18 | print('By Marlous')
19 | print('')
20 | print('Some commands:')
21 | print('help')
22 | print('conf (settings info)')
23 | print('get (get data from internet)')
24 | print('tocsv (produce csv file)')
25 | print('tomysql (analysis data and store to MySQL (start MySQL service first!))')
26 | print('mysqld (Look some detail info about user deep analysis in MySQL)')
27 | print('tool (All about one user info)')
28 | print('quit')
29 |
30 |
31 | if __name__ == '__main__':
32 | print_help_info()
33 |
--------------------------------------------------------------------------------
/WFACatPro/modules/mysql_query.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | """
4 | WFACatPro.mysql_query
5 | ~~~~~~~~~~~~~~~~~~~
6 | This is a mysql query module.
7 | """
8 |
9 |
10 | import traceback
11 |
12 | import pymysql
13 |
14 | import settings
15 |
16 |
17 | global DB_NAME_QUERY
18 |
19 |
20 | """
21 | 查询
22 | """
23 | # 通过微博用户名查某用户基本信息
24 | def query_basic_info_by_name():
25 | name_params = input('Enter name: ')
26 |
27 | try:
28 | cur.execute("SELECT * FROM %s.peopleinfo WHERE name = \'%s\'" % (DB_NAME_QUERY, name_params))
29 | result = cur.fetchone()
30 | row = result
31 |
32 | uid = str(row[0])
33 | name = str(row[1])
34 | rel_me = str(row[2])
35 | connect_to_my_friends = str(row[3])
36 | if row[4] is None:
37 | connect_to_my_friends_count = '0'
38 | else:
39 | connect_to_my_friends_count = str(row[4])
40 | location = str(row[9])
41 | description = str(row[10])
42 | url = str(row[11])
43 | domain = str(row[14])
44 | gender = str(row[15])
45 | followers_count = str(row[16])
46 | friends_count = str(row[17])
47 | statuses_count = str(row[18])
48 | favourites_count = str(row[20])
49 | created_at = str(row[21])
50 | verified = str(row[22])
51 | total_number = str(row[23])
52 | status_source = str(row[24])
53 |
54 | # 把 connect_to_my_friends_count 的 uid 列表中的每个查出 name，组成 has_one_level_friends 名字列表
55 | if connect_to_my_friends_count != '0':
56 | if connect_to_my_friends_count == '1':
57 | user_uid = connect_to_my_friends
58 | cur.execute("SELECT name FROM %s.peopleinfo WHERE uid = \'%s\'" % (DB_NAME_QUERY, str(user_uid)))
59 | user_name_row = cur.fetchone()
60 | has_one_level_friends = str(user_name_row[0])
61 | else:
62 | temp_has_one_level_friends = []
63 | for user_uid in connect_to_my_friends.split(', '):
64 | cur.execute("SELECT name FROM %s.peopleinfo WHERE uid = \'%s\'" % (DB_NAME_QUERY, str(user_uid)))
65 | user_name_row = cur.fetchone()
66 | temp_has_one_level_friends.append(str(user_name_row[0]))
67 | has_one_level_friends = ', '.join(temp_has_one_level_friends)
68 | elif connect_to_my_friends_count == '0':
69 | has_one_level_friends = ''
70 |
71 | print("# uid: %s / name: %s / rel me: %s / has one level friends: %s / has one level friends count: %s "
72 | "/ location: %s / description: %s / url: %s / domain: %s / gender: %s / fans count: %s "
73 | "/ follow count: %s / statuses_count: %s / favourites_count: %s / created_at: %s / verified: %s "
74 | "/ total friends number: %s / client: %s"
75 | % (uid, name, rel_me, has_one_level_friends, connect_to_my_friends_count, location,
76 | description, url, domain, gender, followers_count, friends_count, statuses_count,
77 | favourites_count, created_at, verified, total_number, status_source))
78 |
79 | except Exception:
80 | print('ERROR! Unable to fetch data')
81 | traceback.print_exc()
82 |
83 |
84 | # 通过 uid 查某用户基本信息
85 | # SELECT * FROM DB_NAME_QUERY.peopleinfo WHERE uid = 'uid_params';
86 | def query_basic_info_by_uid():
87 | uid_params = input('Enter uid: ')
88 |
89 | try:
90 | cur.execute("SELECT * FROM %s.peopleinfo WHERE uid = \'%s\'" % (DB_NAME_QUERY, uid_params))
91 | result = cur.fetchone()
92 | row = result
93 |
94 | uid = str(row[0])
95 | name = str(row[1])
96 | rel_me = str(row[2])
97 | connect_to_my_friends = str(row[3])
98 | if row[4] is None:
99 | connect_to_my_friends_count = '0'
100 | else:
101 | connect_to_my_friends_count = str(row[4])
102 | location = str(row[9])
103 | description = str(row[10])
104 | url = str(row[11])
105 | domain = str(row[14])
106 | gender = str(row[15])
107 | followers_count = str(row[16])
108 | friends_count = str(row[17])
109 | statuses_count = str(row[18])
110 | favourites_count = str(row[20])
111 | created_at = str(row[21])
112 | verified = str(row[22])
113 | total_number = str(row[23])
114 | status_source = str(row[24])
115 |
116 | # 把 connect_to_my_friends_count 的 uid 列表中的每个查出 name，组成 has_one_level_friends 名字列表
117 | if connect_to_my_friends_count != '0':
118 | if connect_to_my_friends_count == '1':
119 | user_uid = connect_to_my_friends
120 | cur.execute("SELECT name FROM %s.peopleinfo WHERE uid = \'%s\'" % (DB_NAME_QUERY, str(user_uid)))
121 | user_name_row = cur.fetchone()
122 | has_one_level_friends = str(user_name_row[0])
123 | else:
124 | temp_has_one_level_friends = []
125 | for user_uid in connect_to_my_friends.split(', '):
126 | cur.execute("SELECT name FROM %s.peopleinfo WHERE uid = \'%s\'" % (DB_NAME_QUERY, str(user_uid)))
127 | user_name_row = cur.fetchone()
128 | temp_has_one_level_friends.append(str(user_name_row[0]))
129 | has_one_level_friends = ', '.join(temp_has_one_level_friends)
130 | elif connect_to_my_friends_count == '0':
131 | has_one_level_friends = ''
132 |
133 | print("# uid: %s / name: %s / rel me: %s / has one level friends: %s / has one level friends count: %s "
134 | "/ location: %s / description: %s / url: %s / domain: %s / gender: %s / fans count: %s "
135 | "/ follow count: %s / statuses_count: %s / favourites_count: %s / created_at: %s / verified: %s "
136 | "/ total friends number: %s / client: %s"
137 | % (uid, name, rel_me, has_one_level_friends, connect_to_my_friends_count, location,
138 | description, url, domain, gender, followers_count, friends_count, statuses_count,
139 | favourites_count, created_at, verified, total_number, status_source))
140 |
141 | except Exception:
142 | print('ERROR! Unable to fetch data')
143 | traceback.print_exc()
144 |
145 |
146 | # 通过微博用户名查某用户的互关好友列表
147 | def query_friend_mutual_user():
148 | name_params = input('Enter name: ')
149 |
150 | try:
151 | cur.execute("SELECT * FROM %s.peopleinfo WHERE name = \'%s\'" % (DB_NAME_QUERY, name_params))
152 | result = cur.fetchone()
153 | row = result
154 |
155 | uid_params = str(row[0])
156 |
157 | cur.execute("SELECT * FROM %s.mutualinfo WHERE uid = \'%s\'" % (DB_NAME_QUERY, uid_params))
158 | result = cur.fetchone()
159 | row = result
160 |
161 | temp_mutual_follow = []
162 | for user_uid in row[1].split(', '):
163 | cur.execute("SELECT name FROM %s.peopleinfo WHERE uid = \'%s\'" % (DB_NAME_QUERY, str(user_uid)))
164 | row = cur.fetchone()
165 | temp_mutual_follow.append(str(row)[2:][:-3])
166 |
167 | mutual_follow_name = ', '.join(temp_mutual_follow)
168 |
169 | print("%s" % (mutual_follow_name))
170 |
171 | except Exception:
172 | print('ERROR! Unable to fetch data')
173 | traceback.print_exc()
174 |
175 |
176 | # 通过微博用户名查某一度好友能通过圈内二度好友认识的一度好友
177 | def query_one_level_friend_probably_one_level():
178 | name_params = input('Enter name: ')
179 |
180 | try:
181 | cur.execute("SELECT * FROM %s.peopleinfo WHERE name = \'%s\'" % (DB_NAME_QUERY, name_params))
182 | result = cur.fetchone()
183 | row = result
184 |
185 | uid_params = str(row[0])
186 |
187 | cur.execute("SELECT * FROM %s.u%s" % (DB_NAME_QUERY, uid_params))
188 | result = cur.fetchall()
189 |
190 | for row in result:
191 | cur.execute("SELECT * FROM %s.peopleinfo WHERE uid = \'%s\'" % (DB_NAME_QUERY, str(row[0])))
192 | row_people = cur.fetchone()
193 | people_name = row_people[1]
194 |
195 | friend_people_list = []
196 | for by_friend_uid in row[1].split(', '):
197 | cur.execute("SELECT * FROM %s.peopleinfo WHERE uid = \'%s\'" % (DB_NAME_QUERY, by_friend_uid))
198 | row_friend_people = cur.fetchone()
199 | each_people_name = row_friend_people[1]
200 | friend_people_list.append(str(each_people_name))
201 | friend_people_name = ', '.join(friend_people_list)
202 |
203 | print("Probably people: %s via: %s count %d" % (people_name, friend_people_name, row[2]))
204 |
205 | except Exception:
206 | print('ERROR! Unable to fetch data')
207 | traceback.print_exc()
208 |
209 |
210 | # 所有一度好友信息
211 | def query_all_one_level_friend():
212 | try:
213 | cur.execute("SELECT * FROM %s.peopleinfo WHERE rel_me = \'1\'" % (DB_NAME_QUERY))
214 | result = cur.fetchall()
215 |
216 | for row in result:
217 | people_name_my_friend = ""
218 | people_name_two_level_friend = ""
219 |
220 | uid = str(row[0])
221 | name = str(row[1])
222 |
223 | connect_to_my_friends_count = row[4]
224 | connect_to_my_friends = str(row[3])
225 | if connect_to_my_friends_count >= 2:
226 | people_uid_to_name_my_friend_list = []
227 | for people_uid_my_friend in connect_to_my_friends.split(", "):
228 | cur.execute(
229 | "SELECT * FROM %s.peopleinfo WHERE uid = \'%s\'" % (DB_NAME_QUERY, people_uid_my_friend))
230 | row_temp = cur.fetchone()
231 | each_people_name = row_temp[1]
232 | people_uid_to_name_my_friend_list.append(str(each_people_name))
233 | people_name_my_friend = ', '.join(people_uid_to_name_my_friend_list)
234 | elif connect_to_my_friends_count == 1:
235 | cur.execute("SELECT * FROM %s.peopleinfo WHERE uid = \'%s\'" % (DB_NAME_QUERY, connect_to_my_friends))
236 | row_temp = cur.fetchone()
237 | one_people_name = row_temp[1]
238 | people_name_my_friend = one_people_name
239 |
240 | connect_to_two_level_friends_count = row[6]
241 | connect_to_two_level_friends = str(row[5])
242 | if connect_to_two_level_friends_count >= 2:
243 | people_uid_to_name_two_level_friend_list = []
244 | for people_uid_two_level_friend in connect_to_two_level_friends.split(", "):
245 | cur.execute(
246 | "SELECT * FROM %s.peopleinfo WHERE uid = \'%s\'" % (DB_NAME_QUERY, people_uid_two_level_friend))
247 | row_temp_2 = cur.fetchone()
248 | each_people_name_2 = row_temp_2[1]
249 | people_uid_to_name_two_level_friend_list.append(str(each_people_name_2))
250 | people_name_two_level_friend = ', '.join(people_uid_to_name_two_level_friend_list)
251 | elif connect_to_my_friends_count == 1:
252 | cur.execute("SELECT * FROM %s.peopleinfo WHERE uid = \'%s\'" % (DB_NAME_QUERY, connect_to_two_level_friends))
253 | row_temp_2 = cur.fetchone()
254 | one_people_name_2 = row_temp_2[1]
255 | people_name_two_level_friend = one_people_name_2
256 |
257 | location = str(row[9])
258 | description = str(row[10])
259 | gender = str(row[15])
260 | followers_count = str(row[16])
261 | friends_count = str(row[17])
262 | statuses_count = str(row[18])
263 | favourites_count = str(row[20])
264 | created_at = str(row[21])
265 | verified = str(row[22])
266 | total_number = str(row[23])
267 | status_source = str(row[24])
268 |
269 | print("-----------------------------------------------------------------------------")
270 | print("uid: %s / name: %s / acquire one level people: %s / acquire one level people count: %d / \
271 | acquire two level people: %s / acquire two level people count: %d / \
272 | location: %s / description: %s / gender: %s / followers count: %s / friends count: %s / \
273 | statuses count: %s / favourites count: %s / created at: %s / verified: %s / \
274 | total friends number: %s / client: %s"
275 | % (uid, name, people_name_my_friend, connect_to_my_friends_count,
276 | people_name_two_level_friend, connect_to_two_level_friends_count,
277 | location, description, gender, followers_count, friends_count,
278 | statuses_count, favourites_count, created_at, verified,
279 | total_number, status_source))
280 |
281 | except Exception:
282 | print('ERROR! Unable to fetch data')
283 | traceback.print_exc()
284 |
285 |
286 | """
287 | 统计
288 | """
289 | # 总体概况：总人数、一度好友数、圈内二度好友数、圈外二度好友数、二度好友数
290 | def statistic_person_count():
291 | try:
292 | cur.execute("SELECT COUNT(rel_me) FROM %s.peopleinfo" % (DB_NAME_QUERY))
293 | row = cur.fetchone()
294 | total_people = row[0]
295 | print("总人数: %d" % (total_people))
296 |
297 | cur.execute("SELECT COUNT(rel_me) FROM %s.peopleinfo WHERE rel_me = \'1\'" % (DB_NAME_QUERY))
298 | row = cur.fetchone()
299 | one_level_people = row[0]
300 | one_level_people_proportion = (one_level_people / total_people) * 100
301 | print("一度好友数: %d / 占比: %.2f%%" % (one_level_people, one_level_people_proportion))
302 |
303 | cur.execute("SELECT COUNT(rel_me) FROM %s.peopleinfo WHERE rel_me = \'2\'" % (DB_NAME_QUERY))
304 | row = cur.fetchone()
305 | two_level_in_people = row[0]
306 | two_level_in_people_proportion = (two_level_in_people / total_people) * 100
307 | print("圈内二度好友数（与研究对象一度好友相关的二度好友）: %d / 占比: %.2f%%"
308 | % (two_level_in_people, two_level_in_people_proportion))
309 |
310 | two_level_out_people = total_people - two_level_in_people
311 | two_level_out_people_proportion = (two_level_out_people / total_people) * 100
312 | print("圈外二度好友数（与研究对象一度好友无关的二度好友）: %d / 占比: %.2f%%"
313 | % (two_level_out_people, two_level_out_people_proportion))
314 |
315 | two_level_people = total_people - one_level_people
316 | two_level_people_proportion = two_level_people / total_people
317 | print("二度好友数: %d / 占比: %.2f%%"
318 | % (two_level_people, two_level_people_proportion))
319 |
320 | except Exception:
321 | print('ERROR! Unable to fetch data')
322 | traceback.print_exc()
323 |
324 |
325 | # 一度好友中与其他一度好友互关最多的人（排序）、与圈内二度好友互关最多的人
326 | def mutual_follow_count_sort():
327 | try:
328 | print("一度好友中与其他一度好友互关最多的人:")
329 | cur.execute("SELECT uid, name, connect_to_my_friends_count FROM %s.peopleinfo WHERE rel_me = \'1\' "
330 | "ORDER BY connect_to_my_friends_count DESC;"
331 | % (DB_NAME_QUERY))
332 | result = cur.fetchall()
333 | for row in result:
334 | uid = row[0]
335 | name = row[1]
336 | connect_to_my_friends_count = row[2]
337 | print("uid: %s / name: %s / connect to one level count: %d"
338 | % (str(uid), str(name), connect_to_my_friends_count))
339 |
340 | print("----------------------------------")
341 | print("一度好友中与其他圈内二度好友互关最多的人:")
342 | cur.execute("SELECT uid, name, connect_to_two_level_friends_count FROM %s.peopleinfo WHERE rel_me = \'1\' "
343 | "ORDER BY connect_to_two_level_friends_count DESC;"
344 | % (DB_NAME_QUERY))
345 | result = cur.fetchall()
346 | for row in result:
347 | uid = row[0]
348 | name = row[1]
349 | connect_to_two_level_friends_count = row[2]
350 | print("uid: %s / name: %s / connect to two level count: %d"
351 | % (str(uid), str(name), connect_to_two_level_friends_count))
352 |
353 | except Exception:
354 | print('ERROR! Unable to fetch data')
355 | traceback.print_exc()
356 |
357 |
358 | # 一度好友/圈内二度好友/二度好友中认证情况统计
359 | def statistic_verified():
360 | try:
361 | cur.execute("SELECT COUNT(verified = \'True\') FROM %s.peopleinfo WHERE rel_me = \'1\' " % (DB_NAME_QUERY))
362 | row = cur.fetchone()
363 | print("一度好友认证数量：%d" % (row[0]))
364 |
365 | cur.execute("SELECT COUNT(verified = \'True\') FROM %s.peopleinfo WHERE rel_me = \'2\' " % (DB_NAME_QUERY))
366 | row = cur.fetchone()
367 | print("圈内二度好友认证数量：%d" % (row[0]))
368 |
369 | cur.execute("SELECT COUNT(verified = \'True\') FROM %s.peopleinfo WHERE rel_me = \'2.1\' " % (DB_NAME_QUERY))
370 | row = cur.fetchone()
371 | print("圈外二度好友认证数量：%d" % (row[0]))
372 |
373 | except Exception:
374 | print('ERROR! Unable to fetch data')
375 | traceback.print_exc()
376 |
377 |
378 | # 一度好友地理位置统计、性别统计、关注数、粉丝数、状态数、点赞数、微博创建时间、客户端
379 | def statistic_one_level():
380 | try:
381 | cur.execute("SELECT * FROM %s.peopleinfo WHERE rel_me = \'1\' " % (DB_NAME_QUERY))
382 | result = cur.fetchall()
383 |
384 | location_dict = {}
385 | gender_dict = {}
386 | created_dict = {}
387 | client_dict = {}
388 |
389 | for row in result:
390 | # 地理位置
391 | if row[9] in location_dict.keys():
392 | location_dict[row[9]] = location_dict[row[9]] + 1
393 | else:
394 | location_dict[row[9]] = 1
395 |
396 | # 性别
397 | if row[15] in gender_dict.keys():
398 | gender_dict[row[15]] = int(gender_dict[row[15]]) + 1
399 | else:
400 | gender_dict[row[15]] = 1
401 |
402 | # 微博创建时间
403 | created = row[21][-4:]
404 | if created in created_dict.keys():
405 | created_dict[created] = int(created_dict[created]) + 1
406 | else:
407 | created_dict[created] = 1
408 |
409 | # 客户端
410 | if row[24] is not None:
411 | if row[24] in client_dict.keys():
412 | client_dict[row[24]] = int(client_dict[row[24]]) + 1
413 | else:
414 | client_dict[row[24]] = 1
415 |
416 | location_top_key = max(location_dict, key=lambda x: location_dict[x])
417 | created_top_key = max(created_dict, key=lambda x: created_dict[x])
418 | client_top_key = max(client_dict, key=lambda x: client_dict[x])
419 |
420 | # 关注数
421 | cur.execute("SELECT name, followers_count FROM %s.peopleinfo WHERE rel_me = \'1\' "
422 | "ORDER BY followers_count DESC;"
423 | % (DB_NAME_QUERY))
424 | row = cur.fetchone()
425 | name_followers_count = row[0]
426 | followers_count = row[1]
427 |
428 | # 粉丝数
429 | cur.execute("SELECT name, friends_count FROM %s.peopleinfo WHERE rel_me = \'1\' "
430 | "ORDER BY friends_count DESC;"
431 | % (DB_NAME_QUERY))
432 | row = cur.fetchone()
433 | name_friends_count = row[0]
434 | friends_count = row[1]
435 |
436 | # 状态数
437 | cur.execute("SELECT name, statuses_count FROM %s.peopleinfo WHERE rel_me = \'1\' "
438 | "ORDER BY statuses_count DESC;"
439 | % (DB_NAME_QUERY))
440 | row = cur.fetchone()
441 | name_statuses_count = row[0]
442 | statuses_count = row[1]
443 |
444 | # 点赞数
445 | cur.execute("SELECT name, favourites_count FROM %s.peopleinfo WHERE rel_me = \'1\' "
446 | "ORDER BY favourites_count DESC;"
447 | % (DB_NAME_QUERY))
448 | row = cur.fetchone()
449 | name_favourites_count = row[0]
450 | favourites_count = row[1]
451 |
452 | # 互关好友总数
453 | cur.execute("SELECT name, total_number FROM %s.peopleinfo WHERE rel_me = \'1\' "
454 | "ORDER BY total_number DESC;"
455 | % (DB_NAME_QUERY))
456 | row = cur.fetchone()
457 | name_total_number = row[0]
458 | total_number = row[1]
459 |
460 | print("------ 一度好友 -------")
461 | print("地理位置最多的：%s / 性别统计：男 %d 人，女 %d 人 / "
462 | "关注数最多的：%s %d / 粉丝数最多的：%s %d / "
463 | "状态数最多的：%s %d / 点赞数最多的：%s %d / "
464 | "微博创建时间统计：%s / "
465 | "互关好友总数最多的：%s %d / 客户端最多的：%s"
466 | % (str(location_top_key), int(gender_dict['f']), int(gender_dict['m']),
467 | str(name_followers_count), int(followers_count), str(name_friends_count), int(friends_count),
468 | str(name_statuses_count), int(statuses_count), str(name_favourites_count), int(favourites_count),
469 | str(created_top_key),
470 | str(name_total_number), int(total_number), str(client_top_key)))
471 |
472 | except Exception:
473 | print('ERROR! Unable to fetch data')
474 | traceback.print_exc()
475 |
476 |
477 | # 圈内二度好友地理位置统计、性别统计、关注数、粉丝数、状态数、点赞数、微博创建时间、客户端
478 | def statistic_inner_two_level():
479 | try:
480 | cur.execute("SELECT * FROM %s.peopleinfo WHERE rel_me = \'2\' " % (DB_NAME_QUERY))
481 | result = cur.fetchall()
482 |
483 | location_dict = {}
484 | gender_dict = {}
485 | created_dict = {}
486 | client_dict = {}
487 |
488 | for row in result:
489 | # 地理位置
490 | if row[9] in location_dict.keys():
491 | location_dict[row[9]] = location_dict[row[9]] + 1
492 | else:
493 | location_dict[row[9]] = 1
494 |
495 | # 性别
496 | if row[15] in gender_dict.keys():
497 | gender_dict[row[15]] = int(gender_dict[row[15]]) + 1
498 | else:
499 | gender_dict[row[15]] = 1
500 |
501 | # 微博创建时间
502 | created = row[21][-4:]
503 | if created in created_dict.keys():
504 | created_dict[created] = int(created_dict[created]) + 1
505 | else:
506 | created_dict[created] = 1
507 |
508 | # 客户端
509 | if row[24] is not None:
510 | if row[24] in client_dict.keys():
511 | client_dict[row[24]] = int(client_dict[row[24]]) + 1
512 | else:
513 | client_dict[row[24]] = 1
514 |
515 | location_top_key = max(location_dict, key=lambda x: location_dict[x])
516 | created_top_key = max(created_dict, key=lambda x: created_dict[x])
517 | client_top_key = max(client_dict, key=lambda x: client_dict[x])
518 |
519 | # 关注数
520 | cur.execute("SELECT name, followers_count FROM %s.peopleinfo WHERE rel_me = \'2\' "
521 | "ORDER BY followers_count DESC;"
522 | % (DB_NAME_QUERY))
523 | row = cur.fetchone()
524 | name_followers_count = row[0]
525 | followers_count = row[1]
526 |
527 | # 粉丝数
528 | cur.execute("SELECT name, friends_count FROM %s.peopleinfo WHERE rel_me = \'2\' "
529 | "ORDER BY friends_count DESC;"
530 | % (DB_NAME_QUERY))
531 | row = cur.fetchone()
532 | name_friends_count = row[0]
533 | friends_count = row[1]
534 |
535 | # 状态数
536 | cur.execute("SELECT name, statuses_count FROM %s.peopleinfo WHERE rel_me = \'2\' "
537 | "ORDER BY statuses_count DESC;"
538 | % (DB_NAME_QUERY))
539 | row = cur.fetchone()
540 | name_statuses_count = row[0]
541 | statuses_count = row[1]
542 |
543 | # 点赞数
544 | cur.execute("SELECT name, favourites_count FROM %s.peopleinfo WHERE rel_me = \'2\' "
545 | "ORDER BY favourites_count DESC;"
546 | % (DB_NAME_QUERY))
547 | row = cur.fetchone()
548 | name_favourites_count = row[0]
549 | favourites_count = row[1]
550 |
551 | print("------ 圈内二度好友 -------")
552 | print("地理位置最多的：%s / 性别统计：男 %d 人，女 %d 人 / "
553 | "关注数最多的：%s %d / 粉丝数最多的：%s %d / "
554 | "状态数最多的：%s %d / 点赞数最多的：%s %d / "
555 | "微博创建时间统计：%s / "
556 | "客户端最多的：%s"
557 | % (str(location_top_key), int(gender_dict['f']), int(gender_dict['m']),
558 | str(name_followers_count), int(followers_count), str(name_friends_count), int(friends_count),
559 | str(name_statuses_count), int(statuses_count), str(name_favourites_count), int(favourites_count),
560 | str(created_top_key),
561 | str(client_top_key)))
562 |
563 | except Exception:
564 | print('ERROR! Unable to fetch data')
565 | traceback.print_exc()
566 |
567 |
568 | # 二度好友地理位置统计、性别统计、关注数、粉丝数、状态数、点赞数、微博创建时间、互关好友总数、客户端
569 | def statistic_three_level():
570 | try:
571 | cur.execute("SELECT * FROM %s.peopleinfo WHERE rel_me = \'2.1\' " % (DB_NAME_QUERY))
572 | result = cur.fetchall()
573 |
574 | location_dict = {}
575 | gender_dict = {}
576 | created_dict = {}
577 | client_dict = {}
578 |
579 | for row in result:
580 | # 地理位置
581 | if row[9] in location_dict.keys():
582 | location_dict[row[9]] = location_dict[row[9]] + 1
583 | else:
584 | location_dict[row[9]] = 1
585 |
586 | # 性别
587 | if row[15] in gender_dict.keys():
588 | gender_dict[row[15]] = int(gender_dict[row[15]]) + 1
589 | else:
590 | gender_dict[row[15]] = 1
591 |
592 | # 微博创建时间
593 | created = row[21][-4:]
594 | if created in created_dict.keys():
595 | created_dict[created] = int(created_dict[created]) + 1
596 | else:
597 | created_dict[created] = 1
598 |
599 | # 客户端
600 | if row[24] is not None:
601 | if row[24] in client_dict.keys():
602 | client_dict[row[24]] = int(client_dict[row[24]]) + 1
603 | else:
604 | client_dict[row[24]] = 1
605 |
606 | location_top_key = max(location_dict, key=lambda x: location_dict[x])
607 | created_top_key = max(created_dict, key=lambda x: created_dict[x])
608 | client_top_key = max(client_dict, key=lambda x: client_dict[x])
609 |
610 | # 关注数
611 | cur.execute("SELECT name, followers_count FROM %s.peopleinfo WHERE rel_me = \'2.1\' "
612 | "ORDER BY followers_count DESC;"
613 | % (DB_NAME_QUERY))
614 | row = cur.fetchone()
615 | name_followers_count = row[0]
616 | followers_count = row[1]
617 |
618 | # 粉丝数
619 | cur.execute("SELECT name, friends_count FROM %s.peopleinfo WHERE rel_me = \'2.1\' "
620 | "ORDER BY friends_count DESC;"
621 | % (DB_NAME_QUERY))
622 | row = cur.fetchone()
623 | name_friends_count = row[0]
624 | friends_count = row[1]
625 |
626 | # 状态数
627 | cur.execute("SELECT name, statuses_count FROM %s.peopleinfo WHERE rel_me = \'2.1\' "
628 | "ORDER BY statuses_count DESC;"
629 | % (DB_NAME_QUERY))
630 | row = cur.fetchone()
631 | name_statuses_count = row[0]
632 | statuses_count = row[1]
633 |
634 | # 点赞数
635 | cur.execute("SELECT name, favourites_count FROM %s.peopleinfo WHERE rel_me = \'2.1\' "
636 | "ORDER BY favourites_count DESC;"
637 | % (DB_NAME_QUERY))
638 | row = cur.fetchone()
639 | name_favourites_count = row[0]
640 | favourites_count = row[1]
641 |
642 | print("------ 二度好友 -------")
643 | print("地理位置最多的：%s / 性别统计：男 %d 人，女 %d 人 / "
644 | "关注数最多的：%s %d / 粉丝数最多的：%s %d / "
645 | "状态数最多的：%s %d / 点赞数最多的：%s %d / "
646 | "微博创建时间统计：%s / "
647 | "客户端最多的：%s"
648 | % (str(location_top_key), int(gender_dict['f']), int(gender_dict['m']),
649 | str(name_followers_count), int(followers_count), str(name_friends_count), int(friends_count),
650 | str(name_statuses_count), int(statuses_count), str(name_favourites_count), int(favourites_count),
651 | str(created_top_key),
652 | str(client_top_key)))
653 |
654 | except Exception:
655 | print('ERROR! Unable to fetch data')
656 | traceback.print_exc()
657 |
658 |
659 | # 所有用户微博创建时间统计
660 | def statistic_created():
661 | try:
662 | cur.execute("SELECT created_at FROM %s.peopleinfo" % (DB_NAME_QUERY))
663 | result = cur.fetchall()
664 |
665 | created_at_dict = {}
666 |
667 | for row in result:
668 | if str(row[0])[26:] in created_at_dict.keys():
669 | created_at_dict[str(row[0])[26:]] = created_at_dict[str(row[0])[26:]] + 1
670 | else:
671 | created_at_dict[str(row[0])[26:]] = 1
672 |
673 | created_at_sorted_list = sorted(created_at_dict, key=created_at_dict.get, reverse=True)
674 |
675 | for item in created_at_sorted_list:
676 | print("创建时间：%s 数量：%d" % (str(item), created_at_dict[item]))
677 |
678 | except Exception:
679 | print('ERROR! Unable to fetch data')
680 | traceback.print_exc()
681 |
682 |
683 | """
684 | 推测
685 | """
686 | # 某微博用户（研究对象）成长的城市、久居的城市
687 | def location_probably():
688 | try:
689 | cur.execute("SELECT * FROM %s.peopleinfo WHERE rel_me = \'1\' " % (DB_NAME_QUERY))
690 | result = cur.fetchall()
691 |
692 | location_dict = {}
693 |
694 | for row in result:
695 | # 地理位置
696 | if row[9] in location_dict.keys():
697 | location_dict[row[9]] = location_dict[row[9]] + 1
698 | else:
699 | location_dict[row[9]] = 1
700 |
701 | location_list_sorted = sorted(location_dict, key=location_dict.get, reverse=True)
702 |
703 | location_list = []
704 | count = 0
705 | for item in location_list_sorted:
706 | location_list.append(item)
707 | count = count + 1
708 | if count == 7:
709 | break
710 |
711 | print("可能的出生成长、久居的城市，按可能性排序: %s" % (', '.join(location_list)))
712 |
713 | except Exception:
714 | print('ERROR! Unable to fetch data')
715 | traceback.print_exc()
716 |
717 |
718 | if __name__ == '__main__':
719 | print('= mysql query =')
720 |
721 | db = pymysql.connect(
722 | host=settings.DB_HOST,
723 | port=int(settings.DB_PORT),
724 | user=settings.DB_USER,
725 | passwd=settings.DB_USER_PASSWORD,
726 | charset=settings.DB_CHARSET)
727 | cur = db.cursor()
728 |
729 | cur.execute('SHOW DATABASES;')
730 | # 这里列表得到的是一个个元组，先遍历列表，再遍历列表中每项的元组
731 | db_list_raw = list(cur.fetchall())
732 | db_list_handle = []
733 | for list_item in db_list_raw:
734 | for item in list_item:
735 | db_list_handle.append(item)
736 | db_list = ', '.join(db_list_handle)
737 | print(db_list)
738 | DB_NAME_QUERY = input('Please enter DB name your created: ')
739 |
740 | print('= 注释 =')
741 | print('分为一度好友（直接与研究对象互关的）；'
742 | '圈内二度好友（二度好友中能关联至少互关两个一度好友的）；圈外二度好友（二度好友中只互关一个一度好友的）')
743 | print('= 查询 =')
744 | print('1 通过微博用户名查某用户基本信息')
745 | print('2 通过 uid 查某用户基本信息')
746 | print('3 通过微博用户名查某用户的互关好友列表')
747 | print('4 通过微博用户名查某一度好友能通过圈内二度好友认识的一度好友')
748 | print('5 所有一度好友信息')
749 | print('= 统计 =')
750 | print('6 总体概况：总人数、一度好友数、圈内二度好友数、圈外二度好友数、二度好友数')
751 | print('7 一度好友中与其他一度好友互关最多的人（排序）、与圈内二度好友互关最多的人')
752 | print('8 一度好友/圈内二度好友/二度好友中认证情况统计')
753 | print('9 一度好友地理位置统计、性别统计、关注数、粉丝数、状态数、点赞数、微博创建时间、互关好友总数、客户端')
754 | print('10 圈内二度好友地理位置统计、性别统计、关注数、粉丝数、状态数、点赞数、微博创建时间、客户端')
755 | print('11 二度好友地理位置统计、性别统计、关注数、粉丝数、状态数、点赞数、微博创建时间、客户端')
756 | print('12 所有用户微博创建时间统计')
757 | print('= 推测 =')
758 | print('13 可能的出生成长、久居的城市，按可能性排序。/ '
759 | '研究对象年龄较小的，排名靠最前的为出生成长的城市可能性最大；'
760 | '研究对象年龄较大的，排名靠最前的为久居的城市可能性最大）')
761 | print('= 退出 =')
762 | print('q quit mysql query module')
763 |
764 | while True:
765 | value = input('Enter number to select function: ')
766 | if value not in ('1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13'):
767 | value = input('Enter number to select function: ')
768 |
769 | if value == '1':
770 | query_basic_info_by_name()
771 |
772 | if value == '2':
773 | query_basic_info_by_uid()
774 |
775 | if value == '3':
776 | query_friend_mutual_user()
777 |
778 | if value == '4':
779 | query_one_level_friend_probably_one_level()
780 |
781 | if value == '5':
782 | query_all_one_level_friend()
783 |
784 | if value == '6':
785 | statistic_person_count()
786 |
787 | if value == '7':
788 | mutual_follow_count_sort()
789 |
790 | if value == '8':
791 | statistic_verified()
792 |
793 | if value == '9':
794 | statistic_one_level()
795 |
796 | if value == '10':
797 | statistic_inner_two_level()
798 |
799 | if value == '11':
800 | statistic_three_level()
801 |
802 | if value == '12':
803 | statistic_created()
804 |
805 | if value == '13':
806 | location_probably()
807 |
808 | if value == 'q':
809 | cur.close()
810 | db.close()
811 | exit(1)
812 |
--------------------------------------------------------------------------------
/WFACatPro/modules/settings.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | """
4 | WFACatPro.settings
5 | ~~~~~~~~~~~~~~~~~~~
6 | This is a setting module.
7 | """
8 |
9 |
10 | import json
11 | import os
12 |
13 |
14 | """
15 | 登陆手机微博客户端，打开抓包软件，点击所要查的用户的主页，点击粉丝 -> 点击相互关注，关闭抓包软件，
16 | 找出 含有 friendships 关键字的数据包，导出原始连接填入 privacy_url 后的值。
17 | """
18 | global PRIVACY_URL
19 |
20 | """
21 | 所要研究的微博用户名
22 | """
23 | global PERSON_NAME
24 |
25 | """
26 | 设置要获取的最大相互关注好友数量。
27 | 单页只能获得 200，（json 文件），可以通过构造参数的页数来请求全部的
28 | 如果某个好友（研究对象的好友）的互关数超过 200 人，那么会导致此好友的相关分析信息可能会不准确！
29 | 不建议修改。
30 | """
31 | global EACH_FOLLOWER_COUNT
32 |
33 | """
34 | 设置所要分析的人脉深度，建议研究二度人脉。深度增加，请求数据次数增加。
35 | 不建议修改。
36 | """
37 | global SET_LEVEL
38 |
39 |
40 | """
41 | MySQL 相关参数
42 | """
43 | global DB_HOST
44 | global DB_PORT
45 | global DB_USER
46 | global DB_USER_PASSWORD
47 | global DB_CHARSET
48 |
49 |
50 | """
51 | 在 get_data.py 中下载某用户好友信息的随机最大的暂停时间（范围从 5 到设置的秒数）
52 | """
53 | global TIME_PAUSE_MAX
54 | TIME_PAUSE_MAX = 15
55 |
56 |
57 | def print_config_info():
58 | with open('./WFACat_data/config.json', 'r', encoding='utf-8') as config_file:
59 | config_file_handle = json.loads(config_file.read())
60 | privacy_url_local = config_file_handle['PRIVACY_URL']
61 | person_name_local = config_file_handle['PERSON_NAME']
62 | each_follower_count_local = config_file_handle['EACH_FOLLOWER_COUNT']
63 | set_level_local = config_file_handle['SET_LEVEL']
64 | db_host_local = config_file_handle['DB_HOST']
65 | db_port_local = config_file_handle['DB_PORT']
66 | db_user_local = config_file_handle['DB_USER']
67 | db_user_password_local = config_file_handle['DB_USER_PASSWORD']
68 | db_charset_local = config_file_handle['DB_CHARSET']
69 |
70 | print('= Settings =')
71 | print('PRIVACY_URL: ' + privacy_url_local)
72 | print('PERSON_NAME: ' + person_name_local)
73 | print('EACH_FOLLOWER_COUNT: ' + str(each_follower_count_local))
74 | print('SET_LEVEL: ' + str(set_level_local))
75 | print('DB_HOST: ' + db_host_local)
76 | print('DB_PORT: ' + str(db_port_local))
77 | print('DB_USER: ' + db_user_local)
78 | print('DB_USER_PASSWORD: ' + db_user_password_local)
79 | print('DB_CHARSET: ' + db_charset_local)
80 |
81 |
82 | def write_config_file_privacy_url(privacy_url_params):
83 | if os.path.isfile('./WFACat_data/config.json'):
84 | config_file_in = open('./WFACat_data/config.json', 'r', encoding='utf-8')
85 | config_file_handle_one = json.loads(config_file_in.read())
86 | config_file_in.close()
87 |
88 | config_file_handle_one['PRIVACY_URL'] = privacy_url_params
89 | config_file_handle_two = json.dumps(config_file_handle_one)
90 |
91 | config_file_out = open('./WFACat_data/config.json', 'w', encoding='utf-8')
92 | config_file_out.write(config_file_handle_two)
93 | config_file_out.close()
94 | else:
95 | print('config.json not found!')
96 |
97 |
98 | def write_config_file_person_name(person_name_params):
99 | if os.path.isfile('./WFACat_data/config.json'):
100 | config_file_in = open('./WFACat_data/config.json', 'r', encoding='utf-8')
101 | config_file_handle_one = json.loads(config_file_in.read())
102 | config_file_in.close()
103 |
104 | config_file_handle_one['PERSON_NAME'] = person_name_params
105 | config_file_handle_two = json.dumps(config_file_handle_one)
106 |
107 | config_file_out = open('./WFACat_data/config.json', 'w', encoding='utf-8')
108 | config_file_out.write(config_file_handle_two)
109 | config_file_out.close()
110 | else:
111 | print('config.json not found!')
112 |
113 |
114 | def write_config_file_db_info(db_host_params, db_port_params, db_user_params, db_charset_params):
115 | if os.path.isfile('./WFACat_data/config.json'):
116 | config_file_in = open('./WFACat_data/config.json', 'r', encoding='utf-8')
117 | config_file_handle_one = json.loads(config_file_in.read())
118 | config_file_in.close()
119 |
120 | config_file_handle_one['DB_HOST'] = db_host_params
121 | config_file_handle_one['DB_PORT'] = db_port_params
122 | config_file_handle_one['DB_USER'] = db_user_params
123 | config_file_handle_one['DB_CHARSET'] = db_charset_params
124 | config_file_handle_two = json.dumps(config_file_handle_one)
125 |
126 | config_file_out = open('./WFACat_data/config.json', 'w', encoding='utf-8')
127 | config_file_out.write(config_file_handle_two)
128 | config_file_out.close()
129 | else:
130 | print('config.json not found!')
131 |
132 |
133 | def write_config_file_db_passwd(db_user_password_params):
134 | if os.path.isfile('./WFACat_data/config.json'):
135 | config_file_in = open('./WFACat_data/config.json', 'r', encoding='utf-8')
136 | config_file_handle_one = json.loads(config_file_in.read())
137 | config_file_in.close()
138 |
139 | config_file_handle_one['DB_USER_PASSWORD'] = db_user_password_params
140 | config_file_handle_two = json.dumps(config_file_handle_one)
141 |
142 | config_file_out = open('./WFACat_data/config.json', 'w', encoding='utf-8')
143 | config_file_out.write(config_file_handle_two)
144 | config_file_out.close()
145 | else:
146 | print('config.json not found!')
147 |
148 |
149 | """
150 | 读取配置文件
151 | """
152 | if os.path.isfile('./WFACat_data/config.json'):
153 | with open('./WFACat_data/config.json', 'r', encoding='utf-8') as config_file:
154 | config_file_handle = json.loads(config_file.read())
155 | PRIVACY_URL = config_file_handle['PRIVACY_URL']
156 | PERSON_NAME = config_file_handle['PERSON_NAME']
157 | EACH_FOLLOWER_COUNT = int(config_file_handle['EACH_FOLLOWER_COUNT'])
158 | SET_LEVEL = config_file_handle['SET_LEVEL']
159 | DB_HOST = config_file_handle['DB_HOST']
160 | DB_PORT = int(config_file_handle['DB_PORT'])
161 | DB_USER = config_file_handle['DB_USER']
162 | DB_USER_PASSWORD = config_file_handle['DB_USER_PASSWORD']
163 | DB_CHARSET = config_file_handle['DB_CHARSET']
164 |
165 |
166 | if __name__ == '__main__':
167 | print_config_info()
168 |
169 | values = input('Do you want to modify settings ?[Y/N]')
170 | while values not in ('Y', 'N'):
171 | values = input('Please enter Y or N:')
172 |
173 | if values == 'Y':
174 | select_one = input(
175 | 'Setting privacy url (should use weibo account to analysis data packages on your mobile phone.[Y/N])')
176 | while select_one not in ('Y', 'N'):
177 | select_one = input('Please enter Y or N:')
178 | if select_one == 'Y':
179 | privacy_url_local = input('Enter privacy url: ')
180 | write_config_file_privacy_url(privacy_url_local)
181 |
182 | select_two = input(
183 | 'Setting weibo name (who you want to analysis).[Y/N]')
184 | while select_two not in ('Y', 'N'):
185 | select_two = input('Please enter Y or N:')
186 | if select_two == 'Y':
187 | person_name_local = input('Enter weibo name: ')
188 | write_config_file_person_name(person_name_local)
189 |
190 | select_three = input(
191 | 'Setting DB host, port, user, charset (Default: localhost, 3306, root, utf8mb4).[Y/N]')
192 | while select_three not in ('Y', 'N'):
193 | select_three = input('Please enter Y or N:')
194 | if select_three == 'Y':
195 | db_host_local = input('Enter host: ')
196 | db_port_local = input('Enter port: ')
197 | db_user_local = input('Enter user: ')
198 | db_charset_local = input('Enter charset:')
199 | write_config_file_db_info(
200 | db_host_local,
201 | db_port_local,
202 | db_user_local,
203 | db_charset_local)
204 |
205 | select_four = input('Setting DB user password).[Y/N]')
206 | while select_four not in ('Y', 'N'):
207 | select_four = input('Please enter Y or N:')
208 | if select_four == 'Y':
209 | db_user_password_local = input('Enter user password: ')
210 | write_config_file_db_passwd(db_user_password_local)
211 |
212 | print_config_info()
213 | exit()
214 |
215 | elif values == 'N':
216 | print('Settings no changed !')
217 | exit()
218 |
--------------------------------------------------------------------------------
/WFACatPro/modules/tools.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | """
4 | WFACatPro.tools
5 | ~~~~~~~~~~~~~~~~~~~
6 | This is a tools module.
7 | """
8 |
9 |
10 | import json
11 | import os
12 | import random
13 | import shutil
14 | import time
15 | import urllib
16 | from urllib.parse import urlparse
17 |
18 | import requests
19 |
20 | import settings
21 |
22 |
23 | def get_person_uid(person_name_params):
24 | """
25 | :param: 某个微博用户的用户名
26 | :return: 返回微博用户对应的 uid
27 | """
28 | """
29 | 下载并保存研究对象的好友信息 person.json 文件（一度人脉）
30 | """
31 | person_info_json_file = requests.get(
32 | get_person_info_json_file_url(person_name_params))
33 | if person_info_json_file.status_code == requests.codes.ok:
34 | pass
35 | else:
36 | person_info_json_file.raise_for_status()
37 |
38 | with open('./WFACat_data/tools_query/person.json', 'wb') as person_json_file_saved: # 保存下载的文件到硬盘
39 | person_json_file_saved.write(person_info_json_file.content)
40 |
41 | """
42 | 提取 person 的 uid
43 | """
44 | with open('./WFACat_data/tools_query/person.json', 'r', encoding='utf-8') as person_json_file:
45 | json_file_trans = json.loads(
46 | person_json_file.read()) # 将 json 文件内容转为字典
47 | # 注意字典的索引可以是字符串或整数
48 | person_uid_local = json_file_trans['cards'][1]['card_group'][0]['user']['id']
49 |
50 | return person_uid_local
51 |
52 |
53 | def get_data_url(uid_params, page_params):
54 | """
55 | :param uid_params: 某个微博用户的 uid
56 | :param page_params: 页数。通过循环增加页数，下载到全部互关好友 json 文件
57 | :return: 返回获取某个用户的互关好友列表 json 文件的下载 url
58 | """
59 |
60 | params['count'] = settings.EACH_FOLLOWER_COUNT
61 | params['uid'] = uid_params
62 | params['page'] = page_params
63 |
64 | # 拼接出 url
65 | url_local = 'https://api.weibo.cn/2/friendships/bilateral?' + \
66 | 'aid=' + str(params['aid'])[2:-2] + \
67 | '&c=' + str(params['c'])[2:-2] + \
68 | '&count=' + str(params['count']) + \
69 | '&from=' + str(params['from'])[2:-2] + \
70 | '&gsid=' + str(params['gsid'])[2:-2] + \
71 | '&i=' + str(params['i'])[2:-2] + \
72 | '&lang=' + str(params['lang'])[2:-2] + \
73 | '&page=' + str(params['page']) + \
74 | '&real_relationships=' + str(params['real_relationships'])[2:-2] + \
75 | '&s=' + str(params['s'])[2:-2] + \
76 | '&trim_status' + str(params['trim_status'])[2:-2] + \
77 | '&ua=' + str(params['ua'])[2:-2] + \
78 | '&uid=' + str(params['uid']) + \
79 | '&v_p=' + str(params['v_p'])[2:-2]
80 | return url_local
81 |
82 |
83 | def get_person_info_json_file_url(person_name_params):
84 | """
85 | :param: 某个微博用户名称，在 settings 中的 person_name
86 | :return: 返回获取某个用户信息 json 文件的下载 url
87 | """
88 |
89 | # 需要研究的用户微博名字，经过两次编码，以便能在参数中使用
90 | temp_person_name = urllib.parse.quote(person_name_params)
91 | encoded_person_name = urllib.parse.quote(temp_person_name)
92 |
93 | # 拼接出 url
94 | url_local = 'https://api.weibo.cn/2/cardlist?' + \
95 | 'aid=' + str(params['aid'])[2:-2] + \
96 | '&c=' + str(params['c'])[2:-2] + \
97 | '&containerid=100103type%253D3%2526q%253D' + encoded_person_name + \
98 | '&count=10' + \
99 | '&from=' + str(params['from'])[2:-2] + \
100 | '&gsid=' + str(params['gsid'])[2:-2] + \
101 | '&i=' + str(params['i'])[2:-2] + \
102 | '&lang=' + str(params['lang'])[2:-2] + \
103 | '&page=' + str(params['page'])[2:-2] + \
104 | '&s=' + str(params['s'])[2:-2] + \
105 | '&ua=' + str(params['ua'])[2:-2] + \
106 | '&v_p=' + str(params['v_p'])[2:-2]
107 |
108 | return url_local
109 |
110 |
111 | def get_person_detail_info_json_file_url(person_uid_params):
112 | """
113 | :param: 某个微博用户名称，在 settings 中的 person_name
114 | :return: 返回获取某个用户信息 json 文件的下载 url
115 | """
116 |
117 | # 拼接出 url
118 | url_local = 'https://api.weibo.cn/2/users/show?' + \
119 | 'aid=' + str(params['aid'])[2:-2] + \
120 | '&c=' + str(params['c'])[2:-2] + \
121 | '&from=' + str(params['from'])[2:-2] + \
122 | '&gsid=' + str(params['gsid'])[2:-2] + \
123 | '&has_extend=1&has_profile=1' + \
124 | '&i=' + str(params['i'])[2:-2] + \
125 | '&lang=' + str(params['lang'])[2:-2] + \
126 | '&s=' + str(params['s'])[2:-2] + \
127 | '&ua=' + str(params['ua'])[2:-2] + \
128 | '&uid=' + str(person_uid_params) + \
129 | '&v_p=' + str(params['v_p'])[2:-2]
130 |
131 | return url_local
132 |
133 |
134 | def download_user_json_file(person_uid_params):
135 | """
136 | :param: 某个微博用户的用户名
137 | :return: 返回微博用户对应的 uid
138 | """
139 | """
140 | 下载并保存研究对象的好友信息 person.json 文件（一度人脉）
141 | """
142 | person_info_json_file = requests.get(
143 | get_person_detail_info_json_file_url(person_uid_params))
144 | if person_info_json_file.status_code == requests.codes.ok:
145 | pass
146 | else:
147 | person_info_json_file.raise_for_status()
148 |
149 | with open('./WFACat_data/tools_query/person.json', 'wb') as person_json_file_saved: # 保存下载的文件到硬盘
150 | person_json_file_saved.write(person_info_json_file.content)
151 |
152 |
153 | def download_user_friend_json_file(user_uid_params, user_json_file_saved_path_params):
154 | """
155 | :param user_uid_params: 某一个用户的 uid
156 | :param user_json_file_saved_path_params: 将下载的此用户好友所有 json 文件保存的位置
157 | :return: 无返回值
158 | """
159 | user_uid = user_uid_params
160 | user_json_file_saved_path = user_json_file_saved_path_params
161 |
162 | page_count = 1
163 | while True:
164 | friends_info_json_file = requests.get(
165 | get_data_url(user_uid, page_count)
166 | )
167 |
168 | if friends_info_json_file.status_code == requests.codes.ok:
169 | pass
170 | else:
171 | friends_info_json_file.raise_for_status()
172 |
173 | user_json_file_name = user_json_file_saved_path + '/' + str(page_count) + '.json'
174 |
175 | with open(user_json_file_name, 'wb') as f: # 保存下载的文件到硬盘
176 | f.write(friends_info_json_file.content)
177 |
178 | with open(user_json_file_name, 'r', encoding='utf8') as f: # 如果不含有用户信息，则这个 json 文件是最后一个不要的
179 | json_file_to_dict = json.loads(f.read())
180 |
181 | if json_file_to_dict['users']: # 判断如果 users 下有用户信息则下载下一页，没有就跳出不下载下一页的 json 文件
182 | page_count = page_count + 1
183 | time.sleep(random.randint(1, 3)) # 随机停几秒，请求下一页的 json 文件
184 | continue
185 | else:
186 | break
187 |
188 | user_json_file_name_last = user_json_file_saved_path + '/' + str(page_count) + '.json'
189 | os.remove(user_json_file_name_last) # 删除最后一个下载的用户信息为空的 json 文件
190 |
191 |
192 | def read_and_print_person_info():
193 | """
194 | 所要查询的用户详细信息输出
195 | """
196 | ability_tags = ' '
197 | birthday = ' '
198 | email = ' '
199 |
200 | with open('./WFACat_data/tools_query/person.json', 'r', encoding='utf-8') as f:
201 | json_to_dict = json.loads(f.read()) # 将 json 文件内容转为字典
202 | # 注意字典的索引可以是字符串或整数
203 | uid = json_to_dict['id']
204 | name = json_to_dict['name']
205 | location = json_to_dict['location']
206 | description = json_to_dict['description']
207 | gender = json_to_dict['gender']
208 | followers_count = json_to_dict['followers_count']
209 | friends_count = json_to_dict['friends_count']
210 | statuses_count = json_to_dict['statuses_count']
211 | favourites_count = json_to_dict['favourites_count']
212 | created_at = json_to_dict['created_at']
213 | constellation = json_to_dict['constellation']
214 | verified_reason = json_to_dict['verified_reason']
215 | if 'ability_tags' in json_to_dict:
216 | ability_tags = json_to_dict['ability_tags']
217 | if 'birthday' in json_to_dict:
218 | birthday = json_to_dict['birthday']
219 | if 'email' in json_to_dict:
220 | email = json_to_dict['email']
221 |
222 | print('======== person info ========')
223 | print(person_name + ':')
224 | print('- uid: ' + str(uid))
225 | print('- name: ' + str(name))
226 | print('- location: ' + str(location))
227 | print('- description: ' + str(description))
228 | print('- gender: ' + str(gender))
229 | print('- followers_count: ' + str(followers_count))
230 | print('- friends_count: ' + str(friends_count))
231 | print('- statuses_count: ' + str(statuses_count))
232 | print('- favourites_count: ' + str(favourites_count))
233 | print('- created_at: ' + str(created_at))
234 | if verified_reason:
235 | print('- verified_reason: ' + str(verified_reason))
236 | if constellation:
237 | print('- constellation: ' + str(constellation))
238 | if ability_tags != ' ':
239 | print('- ability_tags: ' + str(ability_tags))
240 | if birthday != ' ':
241 | print('- birthday: ' + str(birthday))
242 | if email != ' ':
243 | print('- email: ' + str(email))
244 |
245 |
246 | def read_and_print_person_mutual_info(person_uid_params):
247 | print('======== person mutual list ========')
248 |
249 | count = 0
250 |
251 | for item in os.listdir(user_json_file_saved_path):
252 | with open('./WFACat_data/tools_query/' + str(person_uid_params) + '/' + item, 'r', encoding='utf8') as f:
253 | json_to_dict = json.loads(f.read())
254 |
255 | user_num = 0
256 | for user_item in json_to_dict['users']:
257 | uid = json_to_dict['users'][user_num]['id']
258 | name = json_to_dict['users'][user_num]['name']
259 | location = json_to_dict['users'][user_num]['location']
260 | description = json_to_dict['users'][user_num]['description']
261 | gender = json_to_dict['users'][user_num]['gender']
262 | followers_count = json_to_dict['users'][user_num]['followers_count']
263 | friends_count = json_to_dict['users'][user_num]['friends_count']
264 | statuses_count = json_to_dict['users'][user_num]['statuses_count']
265 | created_at = json_to_dict['users'][user_num]['created_at'][-4:]
266 |
267 | user_num = user_num + 1
268 |
269 | print('uid: ' + str(uid) + ' / name: ' + str(name) + ' / location: ' + str(location) +
270 | ' / description: ' + str(description) + ' / gender : ' + str(gender) +
271 | ' / followers_count: ' + str(followers_count) + ' / friends_count: ' + str(friends_count) +
272 | ' / statuses_count: ' + str(statuses_count) + ' / created_at: ' + str(created_at))
273 | print('')
274 |
275 | count = count + 1
276 |
277 | print('Friends sum count: ' + str(count))
278 |
279 |
280 | if __name__ == '__main__':
281 | print('= tools =')
282 |
283 | # 拆解设置的 privacy_url
284 | parsed = urlparse(settings.PRIVACY_URL)
285 | # 拆解请求参数
286 | params = urllib.parse.parse_qs(parsed.query)
287 |
288 | # 创建存放每个以用户 uid 命名的文件夹
289 | if os.path.exists('./WFACat_data/tools_query'):
290 | shutil.rmtree('./WFACat_data/tools_query')
291 | os.makedirs('./WFACat_data/tools_query')
292 | else:
293 | os.makedirs('./WFACat_data/tools_query')
294 |
295 | # 用户输入所要查的微博用户名，得到其用户基本信息，并得到 uid
296 | person_name = input('Enter weibo user name that you want find: ')
297 | print('log: wait...')
298 | person_uid = get_person_uid(person_name)
299 |
300 | # 下载查询的某用户详细信息的 json 文件
301 | download_user_json_file(person_uid)
302 |
303 | # 创建研究对象用户文件夹，用于存放互关好友信息的 json 文件（json 以数字命名），并下载相关文件
304 | user_json_file_saved_path = './WFACat_data/tools_query/' + str(person_uid)
305 | os.makedirs(user_json_file_saved_path)
306 |
307 | download_user_friend_json_file(person_uid, user_json_file_saved_path)
308 |
309 | print('1 person info detail')
310 | print('2 person mutual friends info list')
311 | print('q exit tools module')
312 |
313 | while True:
314 | cmd = input('Enter to select:')
315 | if cmd == '1':
316 | read_and_print_person_info()
317 | elif cmd == '2':
318 | read_and_print_person_mutual_info(person_uid)
319 | elif cmd == 'q':
320 | exit()
321 |
--------------------------------------------------------------------------------
/WFACatPro/modules/version.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | """
4 | WFACatPro.version
5 | ~~~~~~~~~~~~~~~~~~~
6 | This is a version info module.
7 | """
8 |
9 |
10 | def print_version_info():
11 | print('###################################################################')
12 | print('Title: ' + 'WFACat: weibo friends deep analysis')
13 | print('Description: ' + 'Python friends\' network analysis for every one.')
14 | print('URL: ' + 'https://github.com/Marlous/WFACat')
15 | print('Version: ' + 'v4.1')
16 | print('Author: ' + 'Marlous')
17 | print('Author E-mail: ' + 'Goonecat@foxmail.com')
18 | print('License: ' + 'GPL v2.0 License')
19 | print('Copyright: ' + 'Copyright 2019 Marlous')
20 | print('####################################################################')
21 |
22 |
23 | if __name__ == '__main__':
24 | print_version_info()
25 |
--------------------------------------------------------------------------------
/docs/软件架构文档.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Marlous/WFACat/cc717dc0fa9b596507575258ac55a62c03afcb29/docs/软件架构文档.PNG
--------------------------------------------------------------------------------