6 |
7 |
8 |
9 | 文章列表
10 |
11 |
60 |
61 | {% endblock %}
--------------------------------------------------------------------------------
/wechatspider/templatetags/paginator.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | __author__ = 'yijingping'
3 | from copy import copy
4 | from django import template
5 |
6 | register = template.Library()
7 |
8 |
9 | def paginator(context, paginated_object_list, adjacent_pages=2):
10 | """
11 | To be used in conjunction with the object_list generic view.
12 |
13 | Adds pagination context variables for use in displaying first, adjacent and
14 | last page links in addition to those created by the object_list generic
15 | view.
16 |
17 | """
18 | page = paginated_object_list.number
19 | pages = paginated_object_list.paginator.num_pages
20 | page_numbers = [n for n in \
21 | range(page - adjacent_pages, page + adjacent_pages + 1) \
22 | if n > 0 and n <= pages]
23 | return {
24 | 'params': context.get('params'),
25 | #'hits': context['hits'],
26 | 'results_per_page': paginated_object_list.paginator.per_page,
27 | 'page': page,
28 | 'pages': pages,
29 | 'page_numbers': page_numbers,
30 | 'next': page + 1,
31 | 'previous': page - 1,
32 | 'has_next': paginated_object_list.has_next(),
33 | 'has_previous': paginated_object_list.has_previous(),
34 | 'show_first': 1 not in page_numbers,
35 | 'show_last': pages not in page_numbers,
36 | 'count': paginated_object_list.paginator.count,
37 | 'start_index': paginated_object_list.start_index,
38 | 'end_index': paginated_object_list.end_index
39 |
40 | }
41 |
42 | register.inclusion_tag('paginator.html', takes_context=True)(paginator)
43 |
44 |
45 | @register.filter
46 | def update_page(params, page):
47 | res = copy((params or {}))
48 | res['page'] = page
49 | return res
50 |
51 | @register.filter
52 | def update_status(params, value):
53 | res = copy((params or {}))
54 | res['status'] = value
55 | return res
56 |
57 | @register.filter
58 | def gen_get_params(params):
59 | res = []
60 | for k, v in params.iteritems():
61 | res.append('%s=%s' % (k, v))
62 |
63 | return '?' + '&'.join(res)
64 |
65 | @register.filter
66 | def remove_key(params, key):
67 | res = copy((params or {}))
68 | res.pop(key, None)
69 | return res
70 |
--------------------------------------------------------------------------------
/wechat/management/commands/checkproxies.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | __author__ = 'yijingping'
3 | import time
4 | from django.core.management.base import BaseCommand
5 | from wechat.models import Proxy
6 | from wechat.util import check_proxy, check_wechat
7 |
8 |
9 | class Command(BaseCommand):
10 | help = 'check proxies'
11 |
12 | def handle(self, *args, **options):
13 | while True:
14 | #self.check_all_proxies()
15 | self.check_wechat_proxies()
16 | time.sleep(60)
17 |
18 | def check_all_proxies(self):
19 | # 检测新代理
20 | qs1 = Proxy.objects.filter(status=Proxy.STATUS_NEW)
21 | # 检测成功代理
22 | qs2 = Proxy.objects.filter(status=Proxy.STATUS_SUCCESS)
23 | # 检测失败代理
24 | qs3 = Proxy.objects.filter(status=Proxy.STATUS_FAIL, retry__lt=3)
25 | for qs in [qs1, qs2, qs3]:
26 | for item in qs:
27 | has_exception, proxy_detected, time_diff = check_proxy(item.host, item.port)
28 | if has_exception or not proxy_detected:
29 | item.status = Proxy.STATUS_FAIL
30 | item.retry += 1
31 | item.save()
32 | else:
33 | item.status = Proxy.STATUS_SUCCESS
34 | item.speed = time_diff * 1000
35 | item.retry = 0
36 | item.save()
37 |
38 | def check_wechat_proxies(self):
39 | # 删除无效代理
40 | qs3 = Proxy.objects.filter(kind=Proxy.KIND_DOWNLOAD, status=Proxy.STATUS_FAIL, retry__gte=1).delete()
41 | # 检测新代理
42 | qs1 = Proxy.objects.filter(kind=Proxy.KIND_DOWNLOAD, status=Proxy.STATUS_NEW)
43 | # 检测成功代理
44 | qs2 = Proxy.objects.filter(kind=Proxy.KIND_DOWNLOAD, status=Proxy.STATUS_SUCCESS)
45 | # 检测失败代理
46 | #qs3 = Proxy.objects.filter(kind=Proxy.KIND_DOWNLOAD, status=Proxy.STATUS_FAIL, retry__lt=1)
47 | for qs in [qs1, qs2]:
48 | for item in qs:
49 | has_exception, proxy_detected, time_diff = check_wechat(item.host, item.port)
50 | if has_exception or not proxy_detected:
51 | item.delete()
52 | else:
53 | item.status = Proxy.STATUS_SUCCESS
54 | item.speed = time_diff * 1000
55 | item.retry = 0
56 | item.save()
57 |
--------------------------------------------------------------------------------
/bin/scheduler.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | __author__ = 'yijingping'
3 | # 加载django环境
4 | import sys
5 | import os
6 | reload(sys)
7 | sys.setdefaultencoding("utf-8")
8 | sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
9 | os.environ['DJANGO_SETTINGS_MODULE'] = 'wechatspider.settings'
10 | import django
11 | django.setup()
12 |
13 | import json
14 | from wechat.models import Wechat, Word
15 | from django.conf import settings
16 | import logging
17 | logger = logging.getLogger()
18 | from datetime import datetime, timedelta
19 | import time
20 | from wechatspider.util import get_redis
21 | from wechat.constants import KIND_NORMAL, KIND_KEYWORD
22 |
23 |
24 | class Scheduler(object):
25 | def run(self):
26 | r = get_redis()
27 | if settings.CRAWLER_DEBUG:
28 | r.delete(settings.CRAWLER_CONFIG["downloader"])
29 |
30 | while True:
31 | now = datetime.now()
32 | # 获取要抓取的公众号
33 | wechats = Wechat.objects.filter(frequency__gt=0, next_crawl_time__lt=now, status=Wechat.STATUS_DEFAULT).order_by('-id')
34 | for item in wechats:
35 | data = {
36 | 'kind': KIND_NORMAL,
37 | 'wechat_id': item.id,
38 | 'wechatid': item.wechatid
39 | }
40 |
41 | r.lpush(settings.CRAWLER_CONFIG["downloader"], json.dumps(data))
42 |
43 | # 更新index_rule
44 | item.next_crawl_time = now + timedelta(minutes=item.frequency)
45 | #item.next_crawl_time = now + timedelta(seconds=item.frequency)
46 | item.save()
47 |
48 | logging.debug(data)
49 |
50 | # 获取要抓取的关键词
51 | keywords = Word.objects.filter(frequency__gt=0, next_crawl_time__lt=now).order_by('-id')
52 | for item in keywords:
53 | data = {
54 | 'kind': KIND_KEYWORD,
55 | 'word': item.text
56 | }
57 |
58 | r.lpush(settings.CRAWLER_CONFIG["downloader"], json.dumps(data))
59 |
60 | # 更新index_rule
61 | item.next_crawl_time = now + timedelta(minutes=item.frequency)
62 | #item.next_crawl_time = now + timedelta(seconds=item.frequency)
63 | item.save()
64 |
65 | logging.debug(data)
66 |
67 | time.sleep(1)
68 |
69 | if __name__ == '__main__':
70 | scheduler = Scheduler()
71 | scheduler.run()
72 |
--------------------------------------------------------------------------------
/supervisord.conf:
--------------------------------------------------------------------------------
1 | #[program:wechatspider.0fenbei.com]
2 | #command=/usr/bin/python /var/www/0fenbei/wechat-spider/manage.py runserver 0.0.0.0:8090
3 | #umask=022
4 | #user=ripple
5 | #startsecs=0
6 | #stopwaitsecs=0
7 | #autostart=true
8 | #autorestart=true
9 | #stdout_logfile=/var/log/wechatspider/wechatspider.stdout.log
10 | #stderr_logfile=/var/log/wechatspider/wechatspider.stderr.log
11 | #stopsignal=KILL
12 | #killasgroup=true
13 |
14 | [program:uwsgi-wechatspider.0fenbei.com]
15 | command=/usr/bin/uwsgi --ini /var/www/0fenbei/wechat-spider/uwsgi.ini
16 | directory=/var/www/0fenbei/wechat-spider
17 | umask=022
18 | user=ripple
19 | startsecs=0
20 | stopwaitsecs=0
21 | autostart=true
22 | autorestart=true
23 | stdout_logfile=/var/log/bowenpay/wechatspider.stdout.log
24 | stderr_logfile=/var/log/bowenpay/wechatspider.stderr.log
25 | stopsignal=QUIT
26 | killasgroup=true
27 |
28 | [program:wechatspider_scheduler]
29 | command=/usr/bin/python /var/www/0fenbei/wechat-spider/bin/scheduler.py
30 | umask=022
31 | user=ripple
32 | startsecs=0
33 | stopwaitsecs=0
34 | autostart=true
35 | autorestart=true
36 | stdout_logfile=/var/log/wechatspider/wechatspider_scheduler.stdout.log
37 | stderr_logfile=/var/log/wechatspider/wechatspider_scheduler.stderr.log
38 | stopsignal=KILL
39 | killasgroup=true
40 |
41 | [program:wechatspider_downloader]
42 | command=/usr/bin/python /var/www/0fenbei/wechat-spider/bin/downloader.py
43 | umask=022
44 | user=root
45 | startsecs=0
46 | stopwaitsecs=0
47 | autostart=true
48 | autorestart=true
49 | stdout_logfile=/var/log/wechatspider/wechatspider_downloader.stdout.log
50 | stderr_logfile=/var/log/wechatspider/wechatspider_downloader.stderr.log
51 | stopsignal=KILL
52 | killasgroup=true
53 | process_name=%(process_num)s
54 | numprocs=2
55 |
56 | [program:wechatspider_extractor]
57 | command=/usr/bin/python /var/www/0fenbei/wechat-spider/bin/extractor.py
58 | umask=022
59 | user=ripple
60 | startsecs=0
61 | stopwaitsecs=0
62 | autostart=true
63 | autorestart=true
64 | stdout_logfile=/var/log/wechatspider/wechatspider_extractor.stdout.log
65 | stderr_logfile=/var/log/wechatspider/wechatspider_extractor.stderr.log
66 | stopsignal=KILL
67 | killasgroup=true
68 | process_name=%(process_num)s
69 | numprocs=2
70 |
71 | [program:wechatspider_processor]
72 | command=/usr/bin/python /var/www/0fenbei/wechat-spider/bin/processor.py
73 | umask=022
74 | user=ripple
75 | startsecs=0
76 | stopwaitsecs=0
77 | autostart=true
78 | autorestart=true
79 | stdout_logfile=/var/log/wechatspider/wechatspider_processor.stdout.log
80 | stderr_logfile=/var/log/wechatspider/wechatspider_processor.stderr.log
81 | stopsignal=KILL
82 | killasgroup=true
83 |
84 |
--------------------------------------------------------------------------------
/wechatspider/templates/base.html:
--------------------------------------------------------------------------------
1 | {% load staticfiles %}
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 | {% block title %}
12 |
微信爬虫
13 |
14 |
15 | {% endblock %}
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 | {% block header_css %}
26 |
27 | {% endblock %}
28 |
29 |
30 |
34 |
35 |
36 |
37 | {% block nav %}
38 | {% include "nav.html" %}
39 | {% endblock %}
40 | {% block main %}
41 |
42 |
43 | {% for message in messages %}
44 |
45 |
46 | {{ message | safe }}
47 |
48 | {% endfor %}
49 | {% block main-content %}
50 |
51 | {% endblock %}
52 |
53 |
54 | {% endblock %}
55 |
56 | {% block footer %}
57 | {% include "footer.html" %}
58 | {% endblock %}
59 |
60 |
61 |
62 |
63 |
64 |
65 | {% block footer-js %}
66 |
67 | {% endblock %}
68 |
73 |
74 |
75 |
--------------------------------------------------------------------------------
/wechat/templates/wechat/topic_list.html:
--------------------------------------------------------------------------------
1 | {% extends "base.html" %}
2 | {% load paginator %}
3 | {% block main-content %}
4 |
5 |
6 |
20 |
21 |
22 | 文章列表
23 |
24 |
25 |
26 |
27 |
28 | | #ID |
29 | 头像 |
30 | 标题 |
31 | 原文链接 |
32 | 公众号 |
33 | 时间 |
34 | 阅读数 |
35 | 点赞数 |
36 | 是否可用 |
37 | 操作 |
38 |
39 |
40 |
41 | {% for item in topics %}
42 | {% with wechat=item.wechat %}
43 |
44 | | {{ item.id}} |
45 |  |
46 | {{ item.title}} |
47 | |
48 | {{ wechat.name}} |
49 | {{ item.publish_time}} |
50 | {{ item.read_num }} |
51 | {{ item.like_num }} |
52 | {{ item.available }} |
53 |
54 | {% if wechat.status == 2 %}
55 | 公众号已删除
56 | {% else %}
57 | 删除公众号
58 | {% endif %}
59 | |
60 |
61 | {% endwith %}
62 | {% endfor %}
63 |
64 |
65 |
66 |
69 |
70 |
71 |
72 |
73 | {% endblock %}
74 |
--------------------------------------------------------------------------------
/wechatspider/static/web/js/qiniu-custom.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Created by yijingping on 15/11/12.
3 | */
4 | function getQiniuUploader(options){
5 | var uploader_option = {
6 | runtimes: 'html5,flash,html4', //上传模式,依次退化
7 | browse_button: options.browse_button, //上传选择的点选按钮，**必需**
8 | uptoken_url: '/qiniu/uptoken/', //Ajax请求upToken的Url，**强烈建议设置**（服务端提供）
9 | // downtoken_url: '/downtoken',
10 | // Ajax请求downToken的Url，私有空间时使用,JS-SDK将向该地址POST文件的key和domain,服务端返回的JSON必须包含`url`字段，`url`值为该文件的下载地址
11 | // uptoken : '
', //若未指定uptoken_url,则必须指定 uptoken ,uptoken由其他程序生成
12 | // unique_names: true, // 默认 false，key为文件名。若开启该选项，SDK会为每个文件自动生成key（文件名）
13 | // save_key: true, // 默认 false。若在服务端生成uptoken的上传策略中指定了 `sava_key`，则开启，SDK在前端将不对key进行任何处理
14 | domain: 'http://7oxfjw.com1.z0.glb.clouddn.com/', //bucket 域名，下载资源时用到，**必需**
15 | get_new_uptoken: false, //设置上传文件的时候是否每次都重新获取新的token
16 | container: options.container, //上传区域DOM ID，默认是browser_button的父元素，
17 | max_file_size: '100mb', //最大文件体积限制
18 | flash_swf_url: 'js/plupload/Moxie.swf', //引入flash,相对路径
19 | max_retries: 3, //上传失败最大重试次数
20 | dragdrop: true, //开启可拖曳上传
21 | drop_element: options.drop_element, //拖曳上传区域元素的ID，拖曳文件或文件夹后可触发上传
22 | chunk_size: '4mb', //分块上传时，每片的体积
23 | auto_start: true, //选择文件后自动上传，若关闭需要自己绑定事件触发上传,
24 | //x_vars : {
25 | // 自定义变量，参考http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html
26 | // 'time' : function(up,file) {
27 | // var time = (new Date()).getTime();
28 | // do something with 'time'
29 | // return time;
30 | // },
31 | // 'size' : function(up,file) {
32 | // var size = file.size;
33 | // do something with 'size'
34 | // return size;
35 | // }
36 | //},
37 | init: {
38 | 'FilesAdded': function(up, files) {
39 | plupload.each(files, function(file) {
40 | // 文件添加进队列后,处理相关的事情
41 | });
42 | },
43 | 'BeforeUpload': function(up, file) {
44 | // 每个文件上传前,处理相关的事情
45 | },
46 | 'UploadProgress': function(up, file) {
47 | // 每个文件上传时,处理相关的事情
48 | },
49 | 'FileUploaded': options.FileUploaded,
50 | 'Error': function(up, err, errTip) {
51 | //上传出错时,处理相关的事情
52 | },
53 | 'UploadComplete': function() {
54 | //队列文件处理完毕后,处理相关的事情
55 | },
56 | 'Key': function(up, file) {
57 | // 若想在前端对每个文件的key进行个性化处理，可以配置该函数
58 | // 该配置必须要在 unique_names: false , save_key: false 时才生效
59 |
60 | var key = ""+new Date().getTime();
61 | // do something with key here
62 | return key
63 | }
64 | }
65 | };
66 | var uploader = Qiniu.uploader(uploader_option);
67 | // domain 为七牛空间（bucket)对应的域名，选择某个空间后，可通过"空间设置->基本设置->域名设置"查看获取
68 |
69 | // uploader 为一个plupload对象，继承了所有plupload的方法，参考http://plupload.com/docs
70 | }
--------------------------------------------------------------------------------
/wechat/util.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | __author__ = 'yijingping'
3 | import time
4 | import urllib2
5 | import requests
6 | from lxml.etree import tostring
7 | from itertools import chain
8 |
9 | ip_check_url = 'http://api.ipify.org'
10 | user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0'
11 | socket_timeout = 3
12 |
13 |
14 | # Get real public IP address
15 | def get_real_pip():
16 | req = urllib2.Request(ip_check_url)
17 | req.add_header('User-agent', user_agent)
18 | conn = urllib2.urlopen(req)
19 | page = conn.read()
20 | conn.close()
21 | return page
22 |
23 | # Set global variable containing "real" public IP address
24 | real_pip = get_real_pip()
25 |
26 |
27 | def check_proxy(host, port):
28 | try:
29 | # Build opener
30 | proxy_handler = urllib2.ProxyHandler({'http': '%s:%s' % (host, port)})
31 | opener = urllib2.build_opener(proxy_handler)
32 | opener.addheaders = [('User-agent', user_agent)]
33 | urllib2.install_opener(opener)
34 |
35 | # Build, time, and execute request
36 | req = urllib2.Request(ip_check_url)
37 | time_start = time.time()
38 | conn = urllib2.urlopen(req, timeout=socket_timeout)
39 | time_end = time.time()
40 | detected_pip = conn.read()
41 | conn.close()
42 |
43 | # Calculate request time
44 | time_diff = time_end - time_start
45 |
46 | # Check if proxy is detected
47 | if detected_pip == real_pip:
48 | proxy_detected = False
49 | else:
50 | proxy_detected = True
51 |
52 | # Catch exceptions
53 | except urllib2.HTTPError, e:
54 | print "ERROR: Code ", e.code
55 | return (True, False, 999)
56 | except Exception, detail:
57 | print "ERROR: ", detail
58 | return (True, False, 999)
59 |
60 | # Return False if no exceptions, proxy_detected=True if proxy detected
61 | return (False, proxy_detected, time_diff)
62 |
63 |
64 | def check_wechat(host, port):
65 | try:
66 | time_start = time.time()
67 | # Build opener
68 | proxies = {
69 | 'http': 'http://%s:%s' % (host, port)
70 | }
71 | headers = {
72 | 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36'
73 | }
74 | rsp = requests.get("http://weixin.sogou.com/weixin",
75 | params={"type": 1, "query": "金融"},
76 | proxies=proxies, headers=headers,
77 | timeout=1
78 | )
79 | rsp.close()
80 | time_end = time.time()
81 |
82 | # Calculate request time
83 | time_diff = time_end - time_start
84 |
85 | #print rsp.content
86 | # Check if proxy is detected
87 | if '金融的相关微信公众号' in rsp.content:
88 | proxy_detected = True
89 | print rsp.content
90 | else:
91 | proxy_detected = False
92 |
93 | # Catch exceptions
94 | except urllib2.HTTPError, e:
95 | print "ERROR: Code ", e.code
96 | return (True, False, 999)
97 | except Exception, detail:
98 | print "ERROR: ", detail
99 | return (True, False, 999)
100 |
101 | # Return False if no exceptions, proxy_detected=True if proxy detected
102 | return (False, proxy_detected, time_diff)
103 |
104 |
105 | def stringify_children(node):
106 | return "".join([x for x in node.itertext()])
107 |
--------------------------------------------------------------------------------
/bin/downloader.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # 加载django环境
3 | import sys
4 | import os
5 | reload(sys)
6 | sys.setdefaultencoding('utf8')
7 | sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
8 | os.environ['DJANGO_SETTINGS_MODULE'] = 'wechatspider.settings'
9 | import django
10 | django.setup()
11 |
12 | import time
13 | import json
14 | from random import randint
15 | from django.conf import settings
16 | from wechatspider.util import get_redis
17 | from wechat.proxies import MysqlProxyBackend
18 | from wechat.downloaders import SeleniumDownloaderBackend
19 | from wechat.constants import KIND_HISTORY, KIND_DETAIL, KIND_KEYWORD
20 | import logging
21 | logger = logging.getLogger()
22 |
23 | CRAWLER_CONFIG = settings.CRAWLER_CONFIG
24 | CRAWLER_GLOBAL_LIMIT_SPEED = settings.CRAWLER_GLOBAL_LIMIT_SPEED
25 |
26 |
27 | class Downloader(object):
28 | def __init__(self):
29 | self.redis = get_redis()
30 |
31 | def get_proxy(self):
32 | return MysqlProxyBackend()
33 |
34 | def check_limit_speed(self):
35 | proxy = self.get_proxy()
36 | key = '%s:%s' % (CRAWLER_CONFIG['global_limit_speed'], proxy)
37 | if self.redis.exists(key):
38 | return True, proxy
39 | else:
40 | self.redis.psetex(key, CRAWLER_GLOBAL_LIMIT_SPEED, CRAWLER_GLOBAL_LIMIT_SPEED)
41 | return False, proxy
42 |
43 | def run(self):
44 | r = self.redis
45 | if settings.CRAWLER_DEBUG:
46 | r.delete(CRAWLER_CONFIG["downloader"])
47 | while True:
48 | try:
49 | resp_data = r.brpop(settings.CRAWLER_CONFIG["downloader"])
50 | except Exception as e:
51 | print e
52 | continue
53 |
54 | try:
55 | data = json.loads(resp_data[1])
56 |
57 | logger.debug(data)
58 | is_limited, proxy = self.check_limit_speed()
59 | if is_limited:
60 | print '# 被限制, 放回去, 下次下载'
61 | time.sleep(1) # 休息一秒, 延迟放回去的时间
62 | r.lpush(CRAWLER_CONFIG["downloader"], resp_data[1])
63 | else:
64 | print '# 未被限制,可以下载'
65 | # 处理文章的函数,用于回调. 每下载一篇, 处理一篇
66 | def process_topic(topic):
67 | if topic.get('kind', None) == KIND_DETAIL:
68 | item_data = topic
69 | elif topic.get('kind', None) == KIND_KEYWORD:
70 | item_data = topic
71 | else:
72 | item_data = topic
73 | item_data["wechat_id"] = data["wechat_id"]
74 | r.lpush(CRAWLER_CONFIG["extractor"], json.dumps(item_data))
75 | logger.debug(item_data)
76 |
77 | with SeleniumDownloaderBackend(proxy=proxy) as browser:
78 | if data.get('kind') == KIND_DETAIL:
79 | res = browser.download_wechat_topic_detail(data, process_topic)
80 | elif data.get('kind') == KIND_HISTORY:
81 | #res = browser.download_wechat_history(data, process_topic)
82 | pass
83 | elif data.get('kind') == KIND_KEYWORD:
84 | res = browser.download_wechat_keyword(data, process_topic)
85 | else:
86 | res = browser.download_wechat(data, process_topic)
87 |
88 | time.sleep(randint(1, 5))
89 | except Exception as e:
90 | print e
91 | raise
92 |
93 |
94 | if __name__ == '__main__':
95 | downloader = Downloader()
96 | downloader.run()
97 |
--------------------------------------------------------------------------------
/wechatspider/static/bootstrap-datepicker/css/bootstrap-datepicker.min.css.map:
--------------------------------------------------------------------------------
1 | {"version":3,"sources":["less/datepicker.less","build/build.less"],"names":[],"mappings":"AAAA,YACC,QAAA,ICsBC,sBAAA,IACG,mBAAA,IACK,cAAA,IDnBT,UAAA,IAHC,mBACA,MAAA,MAGC,2BACD,UAAA,IACY,4CACX,MAAA,MAGD,qBACA,IAAA,EACA,KAAA,EACC,4BACA,QAAA,GACA,QAAA,aACA,YAAA,IAAA,MAAA,YACA,aAAA,IAAA,MAAA,YACA,cAAA,IAAA,MAAA,KACA,WAAA,EACA,oBAAA,eACA,SAAA,SAEA,2BACA,QAAA,GACA,QAAA,aACA,YAAA,IAAA,MAAA,YACA,aAAA,IAAA,MAAA,YACA,cAAA,IAAA,MAAA,KACA,WAAA,EACA,SAAA,SAEuB,mDAAY,KAAA,IACZ,kDAAY,KAAA,IACX,oDAAW,MAAA,IACX,mDAAW,MAAA,IACV,qDAAU,IAAA,KACV,oDAAU,IAAA,KACb,kDACtB,OAAA,KACA,cAAA,EACA,WAAA,IAAA,MAAA,KAEsB,iDACtB,OAAA,KACA,cAAA,EACA,WAAA,IAAA,MAAA,KAGF,gBACC,QAAA,KAED,kBACC,OAAA,EACA,sBAAA,KACA,oBAAA,KACA,mBAAA,KACA,iBAAA,KACA,gBAAA,KACA,YAAA,KAED,eAAI,eACH,WAAA,OACA,MAAA,KACA,OAAA,KC5CA,sBAAA,IACG,mBAAA,IACK,cAAA,ID6CR,OAAA,KAKA,uCAAI,uCACH,iBAAA,YAKI,oCADA,kCAEJ,WAAA,KACA,OAAA,QAGA,4BADA,4BAEA,MAAA,KAEA,iCACS,uCACT,WAAA,IACA,MAAA,KACA,OAAA,QAEA,oCACA,WAAA,QACA,cAAA,EAEA,8BAEM,uCACS,6CAFT,oCC5CL,iBAAA,QACA,iBAAkB,gDAClB,iBAAkB,+CAClB,iBAA2E,8DAC3E,iBAAkB,mDAClB,iBAAkB,8CAClB,iBAAkB,2CAClB,kBAAA,SACA,OAAA,2GAfF,aAAA,QAAA,QAAA,QACA,aAAA,eAAA,eAAA,gBAPA,OAAQ,0DD8DP,MAAA,KCzEmB,qCAAU,uCAAV,8CAAU,gDAApB,8CAAT,6CAAmB,oDAAU,sDAApB,oDAAT,mDAAyC,uDAAA,iDAAhC,qCAAT,oCAAmB,2CAAU,6CAApB,2CAAT,0CAAyC,8CAAA,wCACxC,iBAAA,QAGD,qCAAA,8CADA,8CACA,oDADA,oDAAA,qCACA,2CADA,2CAEC,iBAAA,UDsEW,0CAEZ,MAAA,KAEa,2CACb,MAAA,KAEA,8BAEM,uCACS,6CAFT,oCAGN,WAAA,KC/FD,sBAAA,EACG,mBAAA,EACK,cAAA,EDgGD,oCAEM,6CACS,mDAFT,0CClEX,iBAAA,QACA,iBAAkB,gDAClB,iBAAkB,+CAClB,iBAA2E,8DAC3E,iBAAkB,mDAClB,iBAAkB,8CAClB,iBAAkB,2CAClB,kBAAA,SACA,OAAA,2GAfF,aAAA,QAAA,QAAA,QACA,aAAA,eAAA,eAAA,gBAPA,OAAQ,0DApBR,sBAAA,EACG,mBAAA,EACK,cAAA,EAOY,2CAAU,6CAAV,oDAAU,sDAApB,oDAAT,mDAAmB,0DAAU,4DAApB,0DAAT,yDAAyC,6DAAA,uDAAhC,2CAAT,0CAAmB,iDAAU,mDAApB,iDAAT,gDAAyC,oDAAA,8CACxC,iBAAA,QAGD,2CAAA,oDADA,oDACA,0DADA,0DAAA,2CACA,iDADA,iDAEC,iBAAA,UD4FD,iCAES,0CACS,gDAFT,uCC1ER,iBAAA,QACA,iBAAkB,6CAClB,iBAAkB,4CAClB,iBAA2E,2DAC3E,iBAAkB,gDAClB,iBAAkB,2CAClB,iBAAkB,wCAClB,kBAAA,SACA,OAAA,2GAfF,aAAA,KAAA,KAAA,QACA,aAAA,eAAA,eAAA,gBAPA,OAAQ,0DD2FP,MAAA,KACA,YAAA,EAAA,KAAA,EAAA,gBCvGmB,wCAAU,0CAAV,iDAAU,mDAApB,iDAAT,gDAAmB,uDAAU,yDAApB,uDAAT,sDAAyC,0DAAA,oDAAhC,wCAAT,uCAAmB,8CAAU,gDAApB,8CAAT,6CAAyC,iDAAA,2CACxC,iBAAA,KAGD,wCAAA,iDADA,iDACA,uDADA,uDAAA,wCACA,8CADA,8CAEC,iBAAA,ODoGD,+BAEO,wCACS,8CAFT,qCClFN,iBAAA,QACA,iBAAkB,0CAClB,iBAAkB,yCAClB,iBAA2E,wDAC3E,iBAAkB,6CAClB,iBAAkB,wCAClB,iBAAkB,qCAClB,kBAAA,SACA,OAAA,2GAfF,aAAA,KAAA,KAAA,QACA,aAAA,eAAA,eAAA,gBAPA,OAAQ,0DDmGP,MAAA,KACA,YAAA,EAAA,KAAA,EAAA,gBC/GmB,sCAAU,wCAAV,+CAAU,iDAApB,+CAAT,8CAAmB,qDAAU,uDAApB,qDAAT,oDAAyC,wDAAA,kDAAhC,sCAAT,qCAAmB,4CAAU,8CAApB,4CAAT,2CAAyC,+CAAA,yCACxC,iBAAA,KAGD,sCAAA,+CADA,+CACA,qDADA,qDAAA,sCACA,4CADA,4CAEC,iBAAA,OD4GF,6BACC,QAAA,MACA,MAAA,IACA,OAAA,KACA,YAAA,KACA,MAAA,KACA,OAAA,GACA,OAAA,QCjID,sBAAA,IACG,mBAAA,IACK,cAAA,IDkIN,qCADA,mCAEA,WAAA,KAEA,sCACS,4CACT,WAAA,IACA,MAAA,KACA,OAAA,QAEA,oCAEO,6CACS,mDAFT,0CC7GP,iBAAA,QACA,iBAAkB,0CAClB,iBAAkB,yCAClB,iBAA2E,wDAC3E,iBAAkB,6CAClB,iBAAkB,wCAClB,iBAAkB,qCAClB,kBAAA,SACA,OAAA,2GAfF,aAAA,KAAA,KAAA,QACA,aAAA,eAAA,eAAA,gBAPA,OAAQ,0DD8HN,MAAA,KACA,YAAA,EAAA,KAAA,EAAA,gBC1IkB,2CAAU,6CAAV,oDAAU,sDAApB,oDAAT,mDAAmB,0DAAU,4DAApB,0DAAT,yDAAyC,6DAAA,uDAAhC,2CAAT,0CAAmB,iDAAU,mDAApB,iDAAT,gDAAyC,oDAAA,8CACxC,iBAAA,KAGD,2CAAA,oDADA,oDACA,0DADA,0DAAA,2CACA,iDADA,iDAEC,iBAAA,ODwIA,iCADA,iCAEA,MAAA,KAKH,+BACC,MAAA,MAGD,+BAEA,kBADA,kBAES,wBACR,OAAA,QACC,qCAAA,wBAAA,wBAAA,8BACA,WAAA,KAKF,gBACC,UAAA,KACA,MAAA,KACA,QAAA,EAAA,IAAA,EAAA,IACA,eAAA,OAKM,2BAAA,4BACN,OAAA,QAEA,6BAAA,8BACC,WAAA,IAKF,uBACC,WAAA,OAEI,mCChMJ,sBAAA,IAAA,EAAA,EAAA,IACG,mBAAA,IAAA,EAAA,EAAA,IACK,cAAA,IAAA,EAAA,EAAA,IDiMJ,kCCnMJ,sBAAA,EAAA,IAAA,IAAA,EACG,mBAAA,EAAA,IAAA,IAAA,EACK,cAAA,EAAA,IAAA,IAAA,EDoMT,yBACC,QAAA,aACA,MAAA,KACA,UAAA,KACA,OAAA,KACA,QAAA,IAAA,IACA,YAAA,IACA,YAAA,KACA,WAAA,OACA,YAAA,EAAA,IAAA,EAAA,KACA,eAAA,OACA,iBAAA,KACA,OAAA,IAAA,MAAA,KACA,YAAA,KACA,aAAA"}
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | 注意：搜狗微信在2019.10.29下线相关功能过后，该项目废弃 ！！！
2 |
3 | # 微信爬虫
4 | 一个爬取微信公众号文章的爬虫。
5 |
6 | # 微信爬虫的由来
7 | 零分贝是一家帮助中国5000万贫困人口与社会公益组织对接的公司。
8 |
9 | 我们通过国家和地方政府的“建档立卡”系统，获取到了一手的贫困户数据，目前有100万左右，总数为5000万，目前每个月都在增长。
10 |
11 | 为了帮助这部分贫困户对接公益机构，我写了这个微信爬虫，从微信公众号发布的文章中上找出最新的公益项目。
12 |
13 | 这种找项目的方式的可行性，我们还在试验中。
14 |
15 | 起初，为了快速上线，本爬虫的代码是基于我的另一个 [通用爬虫项目](https://github.com/yijingping/unicrawler) 开发的，我也希望任何对本项目感兴趣的人联系我，与我一同改进这个项目。
16 |
17 | 联系方式：在本项目中新建一个issue留言。
18 |
19 | # 界面预览
20 |
21 | 1） 要爬取的微信公众号列表
22 |
23 | 
24 |
25 | 2） 要爬取的文章关键字列表
26 |
27 | 
28 |
29 | 3） 已经爬取的微信文章
30 |
31 | 
32 |
33 | 4） 查看文章，并标记是否可用
34 |
35 | 
36 |
37 | 5） 控制爬取进程数
38 |
39 | 
40 |
41 |
42 | # 安装
43 |
44 | 1）python环境, 检查python的版本，是否为2.7.x，如果不是，安装2.7.6。
45 |
46 | 如果是centos 6.x，升级python2.6到python2.7，参考教程 http://ruiaylin.github.io/2014/12/12/python%20update/
47 |
48 | 如果是centos 7.x，默认就是python2.7,不用升级
49 |
50 | 如果是mac osx，可以使用virtualenv，安装python2.7
51 |
52 | 2）安装依赖包, clone代码
53 | 安装Mysql-python依赖
54 | ```
55 | yum install python-devel mysql-devel gcc
56 | ```
57 |
58 | 安装lxml依赖
59 | ```
60 | yum install libxslt-devel libxml2-devel
61 | ```
62 |
63 | 安装浏览器环境 selenium依赖.(如果是mac环境，仅需安装firefox， 但确保版本是 firefox 36.0，使用最新的版本会报错)
64 | ```
65 | yum install xorg-x11-server-Xvfb
66 | yum upgrade glib2 # 确保glib2版本大于2.42.2，否则firefox启动会报错
67 | yum install firefox # centos下安装最新的firefox版本
68 | ```
69 |
70 | clone代码,安装依赖python库
71 | ```
72 | $ git clone https://github.com/bowenpay/wechat-spider.git
73 | $ cd wechat-spider
74 | $ pip install -r requirements.txt
75 | ```
76 |
77 | 3) 创建mysql数据库
78 |
79 | 创建数据库wechatspider，默认采用utf8编码。（如果系统支持，可以采用utf8mb4，以兼容emoji字符）
80 |
81 | ```
82 | mysql> CREATE DATABASE `wechatspider` CHARACTER SET utf8;
83 | ```
84 |
85 | 4) 安装和运行Redis
86 |
87 | ```shell
88 | $ wget http://download.redis.io/releases/redis-2.8.3.tar.gz
89 | $ tar xzvf redis-2.8.3.tar.gz
90 | $ cd redis-2.8.3
91 | $ make
92 | $ make install
93 | $ redis-server
94 | ```
95 |
96 | 5) 更新配置文件local_settings
97 |
98 | 在 wechatspider 目录下,添加 `local_settings.py` 文件,配置如下:
99 | ```
100 | # -*- coding: utf-8 -*-
101 |
102 | SECRET_KEY="xxxxxx"
103 |
104 | CRAWLER_DEBUG = True
105 |
106 | # aliyun oss2, 可以将图片和视频存储到阿里云，也可以选择不存储，爬取速度会更快。 默认不存储。
107 | #OSS2_ENABLE = True
108 | #OSS2_CONFIG = {
109 | # "ACCESS_KEY_ID": "XXXXXXXXXXXXXX",
110 | # "ACCESS_KEY_SECRET": "YYYYYYYYYYYYYYYYYYYYYY",
111 | # "ENDPOINT": "",
112 | # "BUCKET_DOMAIN": "oss-cn-hangzhou.aliyuncs.com",
113 | # "BUCKET_NAME": "XXXXX",
114 | # "IMAGES_PATH": "images/",
115 | # "VIDEOS_PATH": "videos/",
116 | # "CDN_DOMAIN": "XXXXXX.oss-cn-hangzhou.aliyuncs.com"
117 | #}
118 | # mysql 数据库配置
119 | DATABASES = {
120 | 'default': {
121 | 'ENGINE': 'django.db.backends.mysql',
122 | 'HOST': '127.0.0.1',
123 | 'NAME': 'wechatspider',
124 | 'USER': 'root',
125 | 'PASSWORD': '',
126 | 'OPTIONS':{
127 | 'charset': 'utf8mb4',
128 | },
129 | }
130 | }
131 | # redis配置,用于消息队列和k-v存储
132 | REDIS_OPTIONS = {
133 | 'host': 'localhost',
134 | 'port': 6379,
135 | 'password': '',
136 | 'db': 4
137 | }
138 |
139 | ```
140 |
141 | 6) 初始化表
142 | ```
143 | $ python manage.py migrate
144 | ```
145 |
146 | 7）启动网站
147 |
148 | ```
149 | python manage.py runserver 0.0.0.0:8001
150 | ```
151 | 访问 http://localhost:8001/。
152 |
153 |
154 | 6) 创建超级管理员账号,访问后台，并配置要爬取的公众号和关键字
155 | ```
156 | python manage.py createsuperuser
157 | ```
158 |
159 |
160 | 8）启动爬虫
161 |
162 | ```shell
163 | $ python bin/scheduler.py
164 | $ python bin/downloader.py
165 | $ python bin/extractor.py
166 | $ python bin/processor.py
167 | ```
168 |
169 | 以上步骤执行成功，并能爬取文章后。可以参考以下部分配置生产环境。
170 |
171 | # 部署nginx
172 | 前期先用nginx将域名www.mydomain.com转发到8001端口。
173 |
174 | # 部署supervisor脚本
175 | 参考文件 `supervisord.conf`
176 |
177 | # 部署crontab脚本
178 | 参考文件 `crontab`
179 |
180 | # 系统使用文档
181 |
182 |
183 | # API接口文档
184 |
185 |
--------------------------------------------------------------------------------
/wechatspider/static/bootstrap-datepicker/css/bootstrap-datepicker.standalone.min.css.map:
--------------------------------------------------------------------------------
1 | {"version":3,"sources":["less/datepicker.less","build/build.less","build/build_standalone.less"],"names":[],"mappings":"AAAA,YACC,QAAA,ICsBC,sBAAA,IACG,mBAAA,IACK,cAAA,IDnBT,UAAA,IAHC,mBACA,MAAA,MAGC,2BACD,UAAA,IACY,4CACX,MAAA,MAGD,qBACA,IAAA,EACA,KAAA,EACC,4BACA,QAAA,GACA,QAAA,aACA,YAAA,IAAA,MAAA,YACA,aAAA,IAAA,MAAA,YACA,cAAA,IAAA,MAAA,KACA,WAAA,EACA,oBAAA,eACA,SAAA,SAEA,2BACA,QAAA,GACA,QAAA,aACA,YAAA,IAAA,MAAA,YACA,aAAA,IAAA,MAAA,YACA,cAAA,IAAA,MAAA,KACA,WAAA,EACA,SAAA,SAEuB,mDAAY,KAAA,IACZ,kDAAY,KAAA,IACX,oDAAW,MAAA,IACX,mDAAW,MAAA,IACV,qDAAU,IAAA,KACV,oDAAU,IAAA,KACb,kDACtB,OAAA,KACA,cAAA,EACA,WAAA,IAAA,MAAA,KAEsB,iDACtB,OAAA,KACA,cAAA,EACA,WAAA,IAAA,MAAA,KAGF,gBACC,QAAA,KAED,kBACC,OAAA,EACA,sBAAA,KACA,oBAAA,KACA,mBAAA,KACA,iBAAA,KACA,gBAAA,KACA,YAAA,KAED,eAAI,eACH,WAAA,OACA,MAAA,KACA,OAAA,KC5CA,sBAAA,IACG,mBAAA,IACK,cAAA,ID6CR,OAAA,KAKA,uCAAI,uCACH,iBAAA,YAKI,oCADA,kCAEJ,WAAA,KACA,OAAA,QAGA,4BADA,4BAEA,MAAA,KAEA,iCACS,uCACT,WAAA,IACA,MAAA,KACA,OAAA,QAEA,oCACA,WAAA,QACA,cAAA,EAEA,8BAEM,uCACS,6CAFT,oCC5CL,iBAAA,QACA,iBAAkB,gDAClB,iBAAkB,+CAClB,iBAA2E,8DAC3E,iBAAkB,mDAClB,iBAAkB,8CAClB,iBAAkB,2CAClB,kBAAA,SACA,OAAA,2GAfF,aAAA,QAAA,QAAA,QACA,aAAA,eAAA,eAAA,gBAPA,OAAQ,0DD8DP,MAAA,KCzEmB,qCAAU,uCAAV,8CAAU,gDAApB,8CAAT,6CAAmB,oDAAU,sDAApB,oDAAT,mDAAyC,uDAAA,iDAAhC,qCAAT,oCAAmB,2CAAU,6CAApB,2CAAT,0CAAyC,8CAAA,wCACxC,iBAAA,QAGD,qCAAA,8CADA,8CACA,oDADA,oDAAA,qCACA,2CADA,2CAEC,iBAAA,UDsEW,0CAEZ,MAAA,KAEa,2CACb,MAAA,KAEA,8BAEM,uCACS,6CAFT,oCAGN,WAAA,KC/FD,sBAAA,EACG,mBAAA,EACK,cAAA,EDgGD,oCAEM,6CACS,mDAFT,0CClEX,iBAAA,QACA,iBAAkB,gDAClB,iBAAkB,+CAClB,iBAA2E,8DAC3E,iBAAkB,mDAClB,iBAAkB,8CAClB,iBAAkB,2CAClB,kBAAA,SACA,OAAA,2GAfF,aAAA,QAAA,QAAA,QACA,aAAA,eAAA,eAAA,gBAPA,OAAQ,0DApBR,sBAAA,EACG,mBAAA,EACK,cAAA,EAOY,2CAAU,6CAAV,oDAAU,sDAApB,oDAAT,mDAAmB,0DAAU,4DAApB,0DAAT,yDAAyC,6DAAA,uDAAhC,2CAAT,0CAAmB,iDAAU,mDAApB,iDAAT,gDAAyC,oDAAA,8CACxC,iBAAA,QAGD,2CAAA,oDADA,oDACA,0DADA,0DAAA,2CACA,iDADA,iDAEC,iBAAA,UD4FD,iCAES,0CACS,gDAFT,uCC1ER,iBAAA,QACA,iBAAkB,6CAClB,iBAAkB,4CAClB,iBAA2E,2DAC3E,iBAAkB,gDAClB,iBAAkB,2CAClB,iBAAkB,wCAClB,kBAAA,SACA,OAAA,2GAfF,aAAA,KAAA,KAAA,QACA,aAAA,eAAA,eAAA,gBAPA,OAAQ,0DD2FP,MAAA,KACA,YAAA,EAAA,KAAA,EAAA,gBCvGmB,wCAAU,0CAAV,iDAAU,mDAApB,iDAAT,gDAAmB,uDAAU,yDAApB,uDAAT,sDAAyC,0DAAA,oDAAhC,wCAAT,uCAAmB,8CAAU,gDAApB,8CAAT,6CAAyC,iDAAA,2CACxC,iBAAA,KAGD,wCAAA,iDADA,iDACA,uDADA,uDAAA,wCACA,8CADA,8CAEC,iBAAA,ODoGD,+BAEO,wCACS,8CAFT,qCClFN,iBAAA,QACA,iBAAkB,0CAClB,iBAAkB,yCAClB,iBAA2E,wDAC3E,iBAAkB,6CAClB,iBAAkB,wCAClB,iBAAkB,qCAClB,kBAAA,SACA,OAAA,2GAfF,aAAA,KAAA,KAAA,QACA,aAAA,eAAA,eAAA,gBAPA,OAAQ,0DDmGP,MAAA,KACA,YAAA,EAAA,KAAA,EAAA,gBC/GmB,sCAAU,wCAAV,+CAAU,iDAApB,+CAAT,8CAAmB,qDAAU,uDAApB,qDAAT,oDAAyC,wDAAA,kDAAhC,sCAAT,qCAAmB,4CAAU,8CAApB,4CAAT,2CAAyC,+CAAA,yCACxC,iBAAA,KAGD,sCAAA,+CADA,+CACA,qDADA,qDAAA,sCACA,4CADA,4CAEC,iBAAA,OD4GF,6BACC,QAAA,MACA,MAAA,IACA,OAAA,KACA,YAAA,KACA,MAAA,KACA,OAAA,GACA,OAAA,QCjID,sBAAA,IACG,mBAAA,IACK,cAAA,IDkIN,qCADA,mCAEA,WAAA,KAEA,sCACS,4CACT,WAAA,IACA,MAAA,KACA,OAAA,QAEA,oCAEO,6CACS,mDAFT,0CC7GP,iBAAA,QACA,iBAAkB,0CAClB,iBAAkB,yCAClB,iBAA2E,wDAC3E,iBAAkB,6CAClB,iBAAkB,wCAClB,iBAAkB,qCAClB,kBAAA,SACA,OAAA,2GAfF,aAAA,KAAA,KAAA,QACA,aAAA,eAAA,eAAA,gBAPA,OAAQ,0DD8HN,MAAA,KACA,YAAA,EAAA,KAAA,EAAA,gBC1IkB,2CAAU,6CAAV,oDAAU,sDAApB,oDAAT,mDAAmB,0DAAU,4DAApB,0DAAT,yDAAyC,6DAAA,uDAAhC,2CAAT,0CAAmB,iDAAU,mDAApB,iDAAT,gDAAyC,oDAAA,8CACxC,iBAAA,KAGD,2CAAA,oDADA,oDACA,0DADA,0DAAA,2CACA,iDADA,iDAEC,iBAAA,ODwIA,iCADA,iCAEA,MAAA,KAKH,+BACC,MAAA,MAGD,+BAEA,kBADA,kBAES,wBACR,OAAA,QACC,qCAAA,wBAAA,wBAAA,8BACA,WAAA,KAKF,gBACC,UAAA,KACA,MAAA,KACA,QAAA,EAAA,IAAA,EAAA,IACA,eAAA,OAKM,2BAAA,4BACN,OAAA,QAEA,6BAAA,8BACC,WAAA,IAKF,uBACC,WAAA,OAEI,mCChMJ,sBAAA,IAAA,EAAA,EAAA,IACG,mBAAA,IAAA,EAAA,EAAA,IACK,cAAA,IAAA,EAAA,EAAA,IDiMJ,kCCnMJ,sBAAA,EAAA,IAAA,IAAA,EACG,mBAAA,EAAA,IAAA,IAAA,EACK,cAAA,EAAA,IAAA,IAAA,EDoMT,yBACC,QAAA,aACA,MAAA,KACA,UAAA,KACA,OAAA,KACA,QAAA,IAAA,IACA,YAAA,IACA,YAAA,KACA,WAAA,OACA,YAAA,EAAA,IAAA,EAAA,KACA,eAAA,OACA,iBAAA,KACA,OAAA,IAAA,MAAA,KACA,YAAA,KACA,aAAA,KE3MC,0BACC,SAAA,SACA,IAAA,KACA,KAAA,EACA,QAAA,KACA,MAAA,KACA,QAAA,KACA,UAAA,MACA,WAAA,KACA,iBAAA,KACA,OAAA,IAAA,MAAA,KACA,OAAA,IAAA,MAAA,eDpBF,sBAAA,IACG,mBAAA,IACK,cAAA,ICDR,mBAAA,EAAA,IAAA,KAAA,eACG,gBAAA,EAAA,IAAA,KAAA,eACK,WAAA,EAAA,IAAA,KAAA,eAoBN,wBAAA,YACG,qBAAA,QACK,gBAAA,YACR,oBAAA,IACA,qBAAA,IAGA,MAAA,KACA,UAAA,KACA,YAAA,KAII,iCAAJ,iCAAI,6BAAJ,6BACE,QAAA,IAAA"}
--------------------------------------------------------------------------------
/wechatspider/static/bootstrap-datepicker/css/bootstrap-datepicker3.min.css.map:
--------------------------------------------------------------------------------
1 | {"version":3,"sources":["less/datepicker3.less","build/build3.less"],"names":[],"mappings":"AAAA,YACC,cAAA,IAIA,UAAA,IAHC,mBACA,MAAA,MAGC,2BACD,UAAA,IACY,4CACX,MAAA,MAGD,qBACA,IAAA,EACA,KAAA,EACA,QAAA,IACC,4BACA,QAAA,GACA,QAAA,aACA,YAAA,IAAA,MAAA,YACA,aAAA,IAAA,MAAA,YACA,cAAA,IAAA,MAAA,gBACA,WAAA,EACA,oBAAA,eACA,SAAA,SAEA,2BACA,QAAA,GACA,QAAA,aACA,YAAA,IAAA,MAAA,YACA,aAAA,IAAA,MAAA,YACA,cAAA,IAAA,MAAA,KACA,WAAA,EACA,SAAA,SAEuB,mDAAY,KAAA,IACZ,kDAAY,KAAA,IACX,oDAAW,MAAA,IACX,mDAAW,MAAA,IACV,qDAAU,IAAA,KACV,oDAAU,IAAA,KACb,kDACtB,OAAA,KACA,cAAA,EACA,WAAA,IAAA,MAAA,gBAEsB,iDACtB,OAAA,KACA,cAAA,EACA,WAAA,IAAA,MAAA,KAGF,gBACC,QAAA,KAED,kBACC,OAAA,EACA,sBAAA,KACA,oBAAA,KACA,mBAAA,KACA,iBAAA,KACA,gBAAA,KACA,YAAA,KAEC,wBAAI,wBACH,WAAA,OACA,MAAA,KACA,OAAA,KACA,cAAA,IACA,OAAA,KAOF,uCAAI,uCACH,iBAAA,YAKA,4BADA,4BAEA,MAAA,KAEI,kCACJ,gCACA,WAAA,KACA,OAAA,QAEA,iCACS,uCACT,WAAA,IACA,MAAA,KACA,OAAA,QAEA,oCC9DD,MAAA,KACA,iBAAA,QACA,aAAA,QD+DC,cAAA,EC5DA,0CADA,0CAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,0CACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,2CADA,2CAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,iDADA,iDADA,iDAEA,iDADA,iDADA,iDAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,mDADA,mDADA,mDAEA,oDADA,oDADA,oDAEA,6DADA,6DADA,6DAGC,iBAAA,QACI,aAAA,QD+BN,4CACA,WAAA,QAGA,6CACS,oDACT,WAAA,QACA,MAAA,KAGD,8BC7ED,MAAA,KACA,iBAAA,QACA,aAAA,QAGC,oCADA,oCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,oCACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,qCADA,qCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,2CADA,2CADA,2CAEA,2CADA,2CADA,2CAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,6CADA,6CADA,6CAEA,8CADA,8CADA,8CAEA,uDADA,uDADA,uDAGC,iBAAA,QACI,aAAA,QD6CN,sCACA,WAAA,QAGA,uCACS,8CACT,WAAA,QACA,MAAA,KAGD,8BC3FD,MAAA,KACA,iBAAA,KACA,aAAA,KD4FC,cAAA,ECzFA,oCADA,oCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,oCACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,qCADA,qCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,2CADA,2CADA,2CAEA,2CADA,2CADA,2CAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,6CADA,6CADA,6CAEA,8CADA,8CADA,8CAEA,uDADA,uDADA,uDAGC,iBAAA,KACI,aAAA,KD4DN,sCACA,WAAA,QAGA,uCACS,8CACT,WAAA,KACA,MAAA,KAGK,0CC1GP,MAAA,KACA,iBAAA,QACA,aAAA,QAGC,gDADA,gDAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,gDACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,iDADA,iDAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,uDADA,uDADA,uDAEA,uDADA,uDADA,uDAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,yDADA,yDADA,yDAEA,0DADA,0DADA,0DAEA,mEADA,mEADA,mEAGC,iBAAA,QACI,aAAA,QD0EN,kDACA,WAAA,QAGA,mDACS,0DACT,WAAA,QACA,MAAA,KAGK,oCCxHP,MAAA,KACA,iBAAA,QACA,aAAA,QAGC,0CADA,0CAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,0CACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,2CADA,2CAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,iDADA,iDADA,iDAEA,iDADA,iDADA,iDAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,mDADA,mDADA,mDAEA,oDADA,oDADA,oDAEA,6DADA,6DADA,6DAGC,iBAAA,QACI,aAAA,QDwFN,6CACS,oDACT,WAAA,QACA,MAAA,KAGD,iCACS,6CCnIV,MAAA,KACA,iBAAA,KACA,aAAA,KDmIC,YAAA,EAAA,KAAA,EAAA,gBChIA,uCAAA,mDADA,mDAAA,uCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,mDAAA,uCACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,wCAAA,oDADA,oDAAA,wCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,8CADA,8CADA,8CAEA,0DADA,0DADA,0DAEA,0DADA,0DADA,0DAEA,8CADA,8CADA,8CAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,gDADA,gDADA,gDAEA,4DADA,4DADA,4DAEA,6DADA,6DADA,6DAEA,iDADA,iDADA,iDAEA,0DAAA,sEADA,sEADA,sEACA,0DADA,0DAGC,iBAAA,KACI,aAAA,KDmGP,+BACO,2CCxIR,MAAA,KACA,iBAAA,QACA,aAAA,QDwIC,YAAA,EAAA,KAAA,EAAA,gBCrIA,qCAAA,iDADA,iDAAA,qCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,iDAAA,qCACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,sCAAA,kDADA,kDAAA,sCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,4CADA,4CADA,4CAEA,wDADA,wDADA,wDAEA,wDADA,wDADA,wDAEA,4CADA,4CADA,4CAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,8CADA,8CADA,8CAEA,0DADA,0DADA,0DAEA,2DADA,2DADA,2DAEA,+CADA,+CADA,+CAEA,wDAAA,oEADA,oEADA,oEACA,wDADA,wDAGC,iBAAA,QACI,aAAA,QDwGR,6BACC,QAAA,MACA,MAAA,IACA,OAAA,KACA,YAAA,KACA,MAAA,KACA,OAAA,GACA,OAAA,QACA,cAAA,IAEC,qCADA,mCAEA,WAAA,KAEA,sCACS,4CACT,WAAA,IACA,MAAA,KACA,OAAA,QAEA,oCAEO,6CACS,mDAFT,0CChKT,MAAA,KACA,iBAAA,QACA,aAAA,QDkKE,YAAA,EAAA,KAAA,EAAA,gBC/JD,mDADA,mDACA,yDADA,yDACA,0CADA,0CACA,gDADA,gDAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,mDAAA,yDAAA,0CAAA,gDACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,2CAAA,oDADA,oDACA,0DADA,0DAAA,2CACA,iDADA,iDAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,iDADA,iDADA,iDAEA,0DADA,0DADA,0DAEA,0DADA,0DADA,0DAEA,gEADA,gEADA,gEAEA,gEADA,gEADA,gEAEA,iDADA,iDADA,iDAEA,uDADA,uDADA,uDAEA,uDADA,uDADA,uDAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,4DADA,4DADA,4DAEA,mDADA,mDADA,mDAEA,kEADA,kEADA,kEAEA,mEADA,mEADA,mEAEA,6DADA,6DADA,6DAEA,yDADA,yDADA,yDAEA,0DADA,0DADA,0DAEA,oDADA,oDADA,oDAEA,sEADA,sEADA,sEAEA,4EADA,4EADA,4EAEA,6DADA,6DADA,6DAEA,mEADA,mEADA,mEAGC,iBAAA,QACI,aAAA,QDmIN,iCADA,iCAEA,MAAA,KAKH,+BACC,MAAA,MAGD,+BAEA,kBADA,kBAES,wBACR,OAAA,QACC,qCAAA,wBAAA,wBAAA,8BACA,WAAA,KAKF,gBACC,UAAA,KACA,MAAA,KACA,QAAA,EAAA,IAAA,EAAA,IACA,eAAA,OAGgB,qCACjB,OAAA,QAED,iBACC,MAAA,KACA,uBACC,WAAA,OAEI,mCACJ,cAAA,IAAA,EAAA,EAAA,IAEI,kCACJ,cAAA,EAAA,IAAA,IAAA,EAED,oCACC,MAAA,KACA,UAAA,KACA,QAAA,IAAA,IACA,YAAA,WACA,YAAA,EAAA,IAAA,EAAA,KACA,aAAA,IAAA,EACA,YAAA,KACA,aAAA"}
--------------------------------------------------------------------------------
/wechatspider/static/bootstrap-datepicker/css/bootstrap-datepicker3.standalone.min.css.map:
--------------------------------------------------------------------------------
1 | {"version":3,"sources":["less/datepicker3.less","build/build3.less","build/build_standalone3.less"],"names":[],"mappings":"AAAA,YACC,cAAA,IAIA,UAAA,IAHC,mBACA,MAAA,MAGC,2BACD,UAAA,IACY,4CACX,MAAA,MAGD,qBACA,IAAA,EACA,KAAA,EACA,QAAA,IACC,4BACA,QAAA,GACA,QAAA,aACA,YAAA,IAAA,MAAA,YACA,aAAA,IAAA,MAAA,YACA,cAAA,IAAA,MAAA,gBACA,WAAA,EACA,oBAAA,eACA,SAAA,SAEA,2BACA,QAAA,GACA,QAAA,aACA,YAAA,IAAA,MAAA,YACA,aAAA,IAAA,MAAA,YACA,cAAA,IAAA,MAAA,KACA,WAAA,EACA,SAAA,SAEuB,mDAAY,KAAA,IACZ,kDAAY,KAAA,IACX,oDAAW,MAAA,IACX,mDAAW,MAAA,IACV,qDAAU,IAAA,KACV,oDAAU,IAAA,KACb,kDACtB,OAAA,KACA,cAAA,EACA,WAAA,IAAA,MAAA,gBAEsB,iDACtB,OAAA,KACA,cAAA,EACA,WAAA,IAAA,MAAA,KAGF,gBACC,QAAA,KAED,kBACC,OAAA,EACA,sBAAA,KACA,oBAAA,KACA,mBAAA,KACA,iBAAA,KACA,gBAAA,KACA,YAAA,KAEC,wBAAI,wBACH,WAAA,OACA,MAAA,KACA,OAAA,KACA,cAAA,IACA,OAAA,KAOF,uCAAI,uCACH,iBAAA,YAKA,4BADA,4BAEA,MAAA,KAEI,kCACJ,gCACA,WAAA,KACA,OAAA,QAEA,iCACS,uCACT,WAAA,IACA,MAAA,KACA,OAAA,QAEA,oCC9DD,MAAA,KACA,iBAAA,QACA,aAAA,QD+DC,cAAA,EC5DA,0CADA,0CAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,0CACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,2CADA,2CAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,iDADA,iDADA,iDAEA,iDADA,iDADA,iDAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,mDADA,mDADA,mDAEA,oDADA,oDADA,oDAEA,6DADA,6DADA,6DAGC,iBAAA,QACI,aAAA,QD+BN,4CACA,WAAA,QAGA,6CACS,oDACT,WAAA,QACA,MAAA,KAGD,8BC7ED,MAAA,KACA,iBAAA,QACA,aAAA,QAGC,oCADA,oCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,oCACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,qCADA,qCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,2CADA,2CADA,2CAEA,2CADA,2CADA,2CAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,6CADA,6CADA,6CAEA,8CADA,8CADA,8CAEA,uDADA,uDADA,uDAGC,iBAAA,QACI,aAAA,QD6CN,sCACA,WAAA,QAGA,uCACS,8CACT,WAAA,QACA,MAAA,KAGD,8BC3FD,MAAA,KACA,iBAAA,KACA,aAAA,KD4FC,cAAA,ECzFA,oCADA,oCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,oCACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,qCADA,qCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,2CADA,2CADA,2CAEA,2CADA,2CADA,2CAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,6CADA,6CADA,6CAEA,8CADA,8CADA,8CAEA,uDADA,uDADA,uDAGC,iBAAA,KACI,aAAA,KD4DN,sCACA,WAAA,QAGA,uCACS,8CACT,WAAA,KACA,MAAA,KAGK,0CC1GP,MAAA,KACA,iBAAA,QACA,aAAA,QAGC,gDADA,gDAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,gDACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,iDADA,iDAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,uDADA,uDADA,uDAEA,uDADA,uDADA,uDAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,yDADA,yDADA,yDAEA,0DADA,0DADA,0DAEA,mEADA,mEADA,mEAGC,iBAAA,QACI,aAAA,QD0EN,kDACA,WAAA,QAGA,mDACS,0DACT,WAAA,QACA,MAAA,KAGK,oCCxHP,MAAA,KACA,iBAAA,QACA,aAAA,QAGC,0CADA,0CAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,0CACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,2CADA,2CAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,iDADA,iDADA,iDAEA,iDADA,iDADA,iDAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,mDADA,mDADA,mDAEA,oDADA,oDADA,oDAEA,6DADA,6DADA,6DAGC,iBAAA,QACI,aAAA,QDwFN,6CACS,oDACT,WAAA,QACA,MAAA,KAGD,iCACS,6CCnIV,MAAA,KACA,iBAAA,KACA,aAAA,KDmIC,YAAA,EAAA,KAAA,EAAA,gBChIA,uCAAA,mDADA,mDAAA,uCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,mDAAA,uCACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,wCAAA,oDADA,oDAAA,wCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,8CADA,8CADA,8CAEA,0DADA,0DADA,0DAEA,0DADA,0DADA,0DAEA,8CADA,8CADA,8CAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,gDADA,gDADA,gDAEA,4DADA,4DADA,4DAEA,6DADA,6DADA,6DAEA,iDADA,iDADA,iDAEA,0DAAA,sEADA,sEADA,sEACA,0DADA,0DAGC,iBAAA,KACI,aAAA,KDmGP,+BACO,2CCxIR,MAAA,KACA,iBAAA,QACA,aAAA,QDwIC,YAAA,EAAA,KAAA,EAAA,gBCrIA,qCAAA,iDADA,iDAAA,qCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,iDAAA,qCACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,sCAAA,kDADA,kDAAA,sCAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,4CADA,4CADA,4CAEA,wDADA,wDADA,wDAEA,wDADA,wDADA,wDAEA,4CADA,4CADA,4CAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,8CADA,8CADA,8CAEA,0DADA,0DADA,0DAEA,2DADA,2DADA,2DAEA,+CADA,+CADA,+CAEA,wDAAA,oEADA,oEADA,oEACA,wDADA,wDAGC,iBAAA,QACI,aAAA,QDwGR,6BACC,QAAA,MACA,MAAA,IACA,OAAA,KACA,YAAA,KACA,MAAA,KACA,OAAA,GACA,OAAA,QACA,cAAA,IAEC,qCADA,mCAEA,WAAA,KAEA,sCACS,4CACT,WAAA,IACA,MAAA,KACA,OAAA,QAEA,oCAEO,6CACS,mDAFT,0CChKT,MAAA,KACA,iBAAA,QACA,aAAA,QDkKE,YAAA,EAAA,KAAA,EAAA,gBC/JD,mDADA,mDACA,yDADA,yDACA,0CADA,0CACA,gDADA,gDAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAEL,mDAAA,yDAAA,0CAAA,gDACC,MAAA,KACA,iBAAA,QACI,aAAA,QAGL,2CAAA,oDADA,oDACA,0DADA,0DAAA,2CACA,iDADA,iDAEC,MAAA,KACA,iBAAA,QACI,aAAA,QAIH,iDADA,iDADA,iDAEA,0DADA,0DADA,0DAEA,0DADA,0DADA,0DAEA,gEADA,gEADA,gEAEA,gEADA,gEADA,gEAEA,iDADA,iDADA,iDAEA,uDADA,uDADA,uDAEA,uDADA,uDADA,uDAGC,MAAA,KACA,iBAAA,QACI,aAAA,QAQL,4DADA,4DADA,4DAEA,mDADA,mDADA,mDAEA,kEADA,kEADA,kEAEA,mEADA,mEADA,mEAEA,6DADA,6DADA,6DAEA,yDADA,yDADA,yDAEA,0DADA,0DADA,0DAEA,oDADA,oDADA,oDAEA,sEADA,sEADA,sEAEA,4EADA,4EADA,4EAEA,6DADA,6DADA,6DAEA,mEADA,mEADA,mEAGC,iBAAA,QACI,aAAA,QDmIN,iCADA,iCAEA,MAAA,KAKH,+BACC,MAAA,MAGD,+BAEA,kBADA,kBAES,wBACR,OAAA,QACC,qCAAA,wBAAA,wBAAA,8BACA,WAAA,KAKF,gBACC,UAAA,KACA,MAAA,KACA,QAAA,EAAA,IAAA,EAAA,IACA,eAAA,OAGgB,qCACjB,OAAA,QAED,iBACC,MAAA,KACA,uBACC,WAAA,OAEI,mCACJ,cAAA,IAAA,EAAA,EAAA,IAEI,kCACJ,cAAA,EAAA,IAAA,IAAA,EAED,oCACC,MAAA,KACA,UAAA,KACA,QAAA,IAAA,IACA,YAAA,WACA,YAAA,EAAA,IAAA,EAAA,KACA,aAAA,IAAA,EACA,YAAA,KACA,aAAA,KElOC,0BACC,SAAA,SACA,IAAA,KACA,KAAA,EACA,QAAA,KACA,QAAA,KACA,MAAA,KACA,UAAA,MACA,WAAA,KACA,iBAAA,KACA,OAAA,IAAA,MAAA,KACA,OAAA,IAAA,MAAA,gBACA,cAAA,IApBF,mBAAA,EAAA,IAAA,KAAA,iBACG,gBAAA,EAAA,IAAA,KAAA,iBACK,WAAA,EAAA,IAAA,KAAA,iBAoBN,wBAAA,YACG,qBAAA,QACK,gBAAA,YAGR,MAAA,KACA,UAAA,KACA,YAAA,WAII,iCAAJ,iCAAI,6BAAJ,6BACE,QAAA,EAAA"}
--------------------------------------------------------------------------------
/docs/api.md:
--------------------------------------------------------------------------------
1 | 接口文档
2 | ==========
3 |
4 | # 添加/修改公众号
5 | 功能: 输入需要爬取的公众号id和爬取频率,如果公众号不存在就添加,如果已经存在就更新爬取频率.同事更新公众号信息.包括头像,描述,二维码等.
6 |
7 | URL: http://127.0.0.1:8090/api/wechat/add/
8 |
9 | 方法: POST
10 |
11 | 参数:
12 | ```
13 | wechatid: 微信id
14 | frequency: 爬取频率, 正整数, 单位是分钟
15 | ```
16 | 返回:
17 | ```
18 | // 已存在更新返回
19 | {
20 | "message": "已更新",
21 | "ret": 0
22 | }
23 | // 不存在保存后返回
24 | {
25 | "message": "已添加",
26 | "ret": 0
27 | }
28 | // 没有找到对应的微信id
29 | {
30 | "message": "公众号不存在",
31 | "ret": 1
32 | }
33 | ```
34 |
35 |
36 | # 添加文章
37 |
38 | 功能: 输入需要爬取的文章URL.将文章加入爬取队列优先爬取.同事获取相关公众号信息,并将爬取频率设置为0
39 |
40 | URL: http://127.0.0.1:8090/api/wechat/topic/add/
41 |
42 | 方法: POST
43 |
44 | 参数:
45 | ```
46 | url: 文章连接,如http://mp.weixin.qq.com/s?__biz=MjM5NDg2NjA4MQ==&mid=402566965&idx=1&sn=616fb1ffa9afc5acc3f4f2a210f6dd83&3rd=MzA3MDU4NTYzMw==&scene=6#rd
47 | ```
48 |
49 | 返回:
50 | ```
51 | // 正确返回
52 | {
53 | 'ret': 0,
54 | 'message': '提交成功,链接已经提交给爬虫,稍后查看爬取结果'
55 | }
56 |
57 | // 错误返回
58 | {
59 | 'ret': 1,
60 | 'message': '提交失败,url必须以 http://mp.weixin.qq.com/ 开头'
61 | }
62 | ```
63 |
64 |
65 |
66 |
67 | # 搜索api
68 |
69 | 功能: 输入公众号名称或者id,返回对应的列表.
70 |
71 | URL: http://127.0.0.1:8090/api/wechat/search?query=财经
72 |
73 | 方法: GET
74 |
75 | 返回:
76 |
77 | ```
78 | {
79 | "data": [
80 | {
81 | "qrcode": "http://img03.sogoucdn.com/app/a/100520105/h3VIR0DEFX5qh1EknyCj",
82 | "wechatid": "LLDS365",
83 | "intro": "本号出租/出售.关注后,即可进入【财经论坛】,和大家互动交流! 专注分享最实用的财经消息以及股票内容!欢迎关注!合作QQ800178778",
84 | "name": "財經",
85 | "avatar": "http://img01.sogoucdn.com/app/a/100520090/oIWsFt3HcMp751wj8bt3PNRmRzHg"
86 | },
87 | {
88 | "qrcode": "http://img03.sogoucdn.com/app/a/100520105/vklgb27EEFBvh38Mnxya",
89 | "wechatid": "zdbank",
90 | "intro": "指点财经每日提供全面真实的财经快讯,发布最新银行理财、保险、资管、信托等正规金融机构发布的产品信息并高效对接理财师,成为投资者一站式理财资讯交互平台.",
91 | "name": "指点财经",
92 | "avatar": "http://img01.sogoucdn.com/app/a/100520090/oIWsFt-wxZs0pb4AGlpIKntt-xxo"
93 | },
94 | {
95 | "qrcode": "http://img03.sogoucdn.com/app/a/100520105/MHVmaV-EomHdh04KnyAU",
96 | "wechatid": "cctvyscj",
97 | "intro": "中央电视台",
98 | "name": "央视财经",
99 | "avatar": "http://img01.sogoucdn.com/app/a/100520090/oIWsFt_cUwbglodLkLT749ZABOt4"
100 | },
101 | {
102 | "qrcode": "http://img03.sogoucdn.com/app/a/100520105/dUOmqSDE0R6uhzHKnxZR",
103 | "wechatid": "tttmoney",
104 | "intro": "原创的财经评论,独立的观察视角,深度的市场剖析. 联系方式:QQ:1527356260;邮箱:1527356260@qq.com.",
105 | "name": "博闻财经",
106 | "avatar": "http://img01.sogoucdn.com/app/a/100520090/oIWsFtza9DUiaCPSyfc_MQO4N5PY"
107 | },
108 | {
109 | "qrcode": "http://img03.sogoucdn.com/app/a/100520105/H0jT3OvEsNXPh-q-nx07",
110 | "wechatid": "lianhuacaijing",
111 | "intro": "莲花APP官微,让你的财富在这里绽放.依托证券时报22年资本市场专业背景,抢先知晓上市公司公告解读、热点事件分析、主题机会挖掘;更有上市公司深度研究及最新机构调研动向.",
112 | "name": "莲花财经",
113 | "avatar": "http://img01.sogoucdn.com/app/a/100520090/oIWsFtxNDf-e3-9SNoKmaZ8B9hcw"
114 | },
115 | {
116 | "qrcode": "http://img03.sogoucdn.com/app/a/100520105/5XVNQmXEd1sIh3QhnyDB",
117 | "wechatid": "sohucaijing2013",
118 | "intro": "搜狐财经,不海量,不枯燥,财经可以很简单!这里有最及时资讯、最独家爆料、最麻辣点评,最权威的专家学者声音.",
119 | "name": "搜狐财经",
120 | "avatar": "http://img01.sogoucdn.com/app/a/100520090/oIWsFt89mqsRNr-HSk72nHeLEUls"
121 | },
122 | {
123 | "qrcode": "http://img03.sogoucdn.com/app/a/100520105/D0wGCVjEpGbbh0lqnxkr",
124 | "wechatid": "bxcjtv",
125 | "intro": "百姓财经栏目是四川广播电视台经济频道2007年强力推出的一档财经类节目.从百姓投资理财的需求出发,以中国股市为着眼点,联手全国一流券商、邀请资深专业人士,以超前的眼光、敏锐的视角、权威的分析、把握大势,...",
126 | "name": "百姓财经",
127 | "avatar": "http://img01.sogoucdn.com/app/a/100520090/oIWsFtzfSD2HictGHcf_wip7ImVU"
128 | },
129 | {
130 | "qrcode": "http://img03.sogoucdn.com/app/a/100520105/tkP59tLEEuxth8OVnxaS",
131 | "wechatid": "njue_xcb_weixin",
132 | "intro": "南京财经大学官方公众平台,传播校园文化,传递校园信息,服务广大师生校友.欢迎关注!",
133 | "name": "南京财经大学",
134 | "avatar": "http://img01.sogoucdn.com/app/a/100520090/oIWsFt2H_o--VPZokXgl-56Mhh1g"
135 | },
136 | {
137 | "qrcode": "http://img03.sogoucdn.com/app/a/100520105/UUgYF4fE-rmBh5Z0nx11",
138 | "wechatid": "bh2056",
139 | "intro": "做有趣、有料、有新意的大众金融平台",
140 | "name": "渤海财经",
141 | "avatar": "http://img01.sogoucdn.com/app/a/100520090/oIWsFtyo_x125073RKHO4j27ZEtE"
142 | },
143 | {
144 | "qrcode": "http://img03.sogoucdn.com/app/a/100520105/7HVuYcHEfv8Bh9ACnyDI",
145 | "wechatid": "wzdsbcf",
146 | "intro": "欢迎关注温州财经,本公众号由温州都市报财经部运营,为广大温都爱理财俱乐部成员提供在线搜索、便民服务、财经新闻,以及在线股票问诊、国企类理财产品服务.",
147 | "name": "温州财经",
148 | "avatar": "http://img01.sogoucdn.com/app/a/100520090/oIWsFt0G2xPqCn-pG1sAAY2n8GI4"
149 | }
150 | ],
151 | "ret": 0
152 | }
153 | ```
--------------------------------------------------------------------------------
/wechat/models.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | __author__ = 'yijingping'
3 | from django.db import models
4 | from datetime import date, datetime, timedelta
5 |
6 | class Wechat(models.Model):
7 | STATUS_DEFAULT = 0
8 | STATUS_DISABLE = 1
9 | STATUS_DELETE = 2
10 | STATUS_CHOICES = (
11 | (STATUS_DEFAULT, '默认'),
12 | (STATUS_DISABLE, '禁用'),
13 | (STATUS_DELETE, '删除')
14 | )
15 | avatar = models.CharField(max_length=500, blank=True, default='', verbose_name='公众号头像')
16 | qrcode = models.CharField(max_length=500, blank=True, default='', verbose_name='二维码')
17 | name = models.CharField(max_length=100, verbose_name='公众号')
18 | wechatid = models.CharField(max_length=100, verbose_name='公众号id', unique=True)
19 | intro = models.TextField(default='', blank=True, verbose_name='简介')
20 | frequency = models.IntegerField(default=0, verbose_name='爬取频率, 单位:分钟')
21 | next_crawl_time = models.DateTimeField(auto_now_add=True, verbose_name='下次爬取时间')
22 | create_time = models.DateTimeField(auto_now_add=True, verbose_name='创建时间')
23 | status = models.IntegerField(default=STATUS_DEFAULT, choices=STATUS_CHOICES, verbose_name="状态")
24 |
25 | def last_day_topics_count(self):
26 | yestoday = date.today() - timedelta(days=1)
27 | yestoday_datetime = datetime.combine(yestoday, datetime.min.time())
28 | return Topic.objects.filter(wechat=self, publish_time__gt=yestoday_datetime).count()
29 |
30 | def last_week_topics_count(self):
31 | last_week = date.today() - timedelta(days=7)
32 | last_week_datetime = datetime.combine(last_week, datetime.min.time())
33 | return Topic.objects.filter(wechat=self, publish_time__gt=last_week_datetime).count()
34 |
35 | def total_topics_count(self):
36 | return Topic.objects.filter(wechat=self).count()
37 |
38 | def __unicode__(self):
39 | return self.name
40 |
41 | class Meta:
42 | verbose_name_plural = "公众号"
43 |
44 |
45 | class Topic(models.Model):
46 | wechat = models.ForeignKey('Wechat', verbose_name='公众号')
47 | uniqueid = models.CharField(unique=True, max_length=100, verbose_name='url的md5值')
48 | words = models.IntegerField(default=0, verbose_name='字数')
49 |
50 | url = models.CharField(max_length=500, default='', verbose_name='文章的url')
51 | avatar = models.CharField(max_length=500, default='', verbose_name='缩略图地址')
52 | title = models.CharField(max_length=200, verbose_name='标题')
53 | origin_title = models.CharField(max_length=200, default='', verbose_name='原文标题')
54 |
55 | abstract = models.TextField(default='', verbose_name='内容简介')
56 | content = models.TextField(default='', verbose_name='文章内容')
57 | source = models.TextField(default='', verbose_name='文章原内容')
58 |
59 | read_num = models.IntegerField(default=0, verbose_name='阅读数')
60 | like_num = models.IntegerField(default=0, verbose_name='点赞数')
61 |
62 | publish_time = models.DateTimeField(db_index=True, verbose_name='发布时间')
63 | create_time = models.DateTimeField(auto_now_add=True, verbose_name='创建时间')
64 | update_time = models.DateTimeField(auto_now=True, verbose_name='更新时间')
65 | available = models.CharField(db_index=True, max_length=100, default='', verbose_name='是否可用')
66 |
67 | def __unicode__(self):
68 | return self.title
69 |
70 | class Meta:
71 | verbose_name_plural = "文章"
72 |
73 |
74 | class Proxy(models.Model):
75 | STATUS_NEW = 0
76 | STATUS_SUCCESS = 1
77 | STATUS_FAIL = 2
78 | STATUS_CHOICES = (
79 | (STATUS_NEW,'未检测'),
80 | (STATUS_SUCCESS,'检测成功'),
81 | (STATUS_FAIL,'检测失败'),
82 | )
83 | KIND_SEARCH = 0
84 | KIND_DOWNLOAD = 1
85 | KIND_CHOICES = (
86 | (KIND_SEARCH, '搜索代理'),
87 | (KIND_DOWNLOAD, '下载代理'),
88 | )
89 | kind = models.IntegerField(default=KIND_DOWNLOAD, choices=KIND_CHOICES, verbose_name="类型")
90 | user = models.CharField(default='', blank=True, max_length=100)
91 | password = models.CharField(default='', blank=True, max_length=100)
92 | host = models.CharField(max_length=100)
93 | port = models.IntegerField(default=80)
94 | speed = models.IntegerField(default=0, verbose_name="连接速度(ms)")
95 | status = models.IntegerField(default=STATUS_NEW, choices=STATUS_CHOICES, verbose_name="状态")
96 | retry = models.IntegerField(default=0, verbose_name="尝试次数")
97 | create_time = models.DateTimeField(auto_now_add=True, verbose_name='创建时间')
98 | update_time = models.DateTimeField(auto_now=True, verbose_name='更新时间')
99 |
100 | class Meta:
101 | verbose_name_plural = "访问代理"
102 |
103 |
104 | class Word(models.Model):
105 | KIND_KEYWORD = 0
106 | #KIND_TOPIC = 1 #
107 | KIND_CHOICES = (
108 | (KIND_KEYWORD, '关键词'),
109 | #(KIND_TOPIC, '话题'),
110 | )
111 | kind = models.IntegerField(default=KIND_KEYWORD, choices=KIND_CHOICES, verbose_name="类型")
112 | text = models.CharField(max_length=100, verbose_name='词')
113 | intro = models.TextField(default='', blank=True, verbose_name='简介')
114 | frequency = models.IntegerField(default=100, verbose_name='爬取频率, 单位:分钟')
115 | next_crawl_time = models.DateTimeField(auto_now_add=True, verbose_name='下次爬取时间')
116 | create_time = models.DateTimeField(auto_now_add=True, verbose_name='创建时间')
117 |
118 | def __unicode__(self):
119 | return '%s %s' % (self.kind, self.text)
120 |
121 | class Meta:
122 | verbose_name_plural = "词"
123 |
--------------------------------------------------------------------------------
/wechat/templates/wechat/index.html:
--------------------------------------------------------------------------------
1 | {% extends "base.html" %}
2 | {% load paginator %}
3 | {% load staticfiles %}
4 | {% block main-content %}
5 |
6 |
7 |
8 |
9 |
10 | 添加公众号
11 |
12 |
13 |
22 |
23 |
24 |
25 |
26 |
41 |
42 |
43 |
44 |
45 |
46 |
48 |
49 |
50 |
51 |
55 |
95 |
96 |
99 |
100 |
101 |
102 |
103 |
6 | 
15 | 
16 | 